diff --git a/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml b/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75fd67e8d3bd394ee0e49f64243aa7753c2e171a --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml @@ -0,0 +1,314 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-mistral-sample_train_2024-08-04-02:10:14 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +save: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +base_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 8192 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-mistral-sample +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32768 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722705032.417279 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt b/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_021032-cd2cg2ui/logs/debug-internal.log b/wandb/run-20240804_021032-cd2cg2ui/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..36e70dbaa7c2869f8350b57f1f198b9016543207 --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/logs/debug-internal.log @@ -0,0 +1,185 @@ +2024-08-04 02:10:32,669 INFO StreamThr :11309 [internal.py:wandb_internal():86] W&B internal server running at pid: 11309, started at: 2024-08-04 02:10:32.417731 +2024-08-04 02:10:32,670 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: status +2024-08-04 02:10:32,672 INFO WriterThread:11309 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb +2024-08-04 02:10:32,673 DEBUG SenderThread:11309 [sender.py:send():382] send: header +2024-08-04 02:10:32,883 DEBUG SenderThread:11309 [sender.py:send():382] send: run +2024-08-04 02:10:33,348 INFO SenderThread:11309 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_021032-cd2cg2ui/files +2024-08-04 02:10:33,348 INFO SenderThread:11309 [sender.py:_start_run_threads():1136] run started: cd2cg2ui with start time 1722705032.417279 +2024-08-04 02:10:33,353 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 02:10:33,354 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: check_version +2024-08-04 02:10:33,438 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 02:10:33,444 DEBUG HandlerThread:11309 [system_info.py:__init__():27] System info init +2024-08-04 02:10:33,444 DEBUG HandlerThread:11309 [system_info.py:__init__():42] System info init done +2024-08-04 02:10:33,444 INFO HandlerThread:11309 [system_monitor.py:start():194] Starting system monitor +2024-08-04 02:10:33,444 INFO SystemMonitor:11309 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 02:10:33,445 INFO HandlerThread:11309 [system_monitor.py:probe():214] Collecting system info +2024-08-04 02:10:33,445 INFO SystemMonitor:11309 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 02:10:33,445 INFO SystemMonitor:11309 [interfaces.py:start():190] Started disk monitoring +2024-08-04 02:10:33,447 INFO SystemMonitor:11309 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 02:10:33,447 INFO SystemMonitor:11309 [interfaces.py:start():190] Started memory monitoring +2024-08-04 02:10:33,448 INFO SystemMonitor:11309 [interfaces.py:start():190] Started network monitoring +2024-08-04 02:10:33,458 DEBUG HandlerThread:11309 [system_info.py:probe():151] Probing system +2024-08-04 02:10:33,460 DEBUG HandlerThread:11309 [system_info.py:_probe_git():136] Probing git +2024-08-04 02:10:33,471 DEBUG HandlerThread:11309 [system_info.py:_probe_git():144] Probing git done +2024-08-04 02:10:33,471 DEBUG HandlerThread:11309 [system_info.py:probe():199] Probing system done +2024-08-04 02:10:33,471 DEBUG HandlerThread:11309 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T17:10:33.458421', 'startedAt': '2024-08-03T17:10:32.395506', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '8192', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample', '--load', '/work/llm_recipes/models/tiny-mistral-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample_train_2024-08-04-02:10:14'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-04 02:10:33,471 INFO HandlerThread:11309 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 02:10:33,471 INFO HandlerThread:11309 [system_monitor.py:probe():227] Publishing system info +2024-08-04 02:10:33,473 INFO HandlerThread:11309 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 02:10:33,478 DEBUG SenderThread:11309 [sender.py:send():382] send: files +2024-08-04 02:10:33,479 INFO SenderThread:11309 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 02:10:33,488 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 02:10:33,488 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 02:10:33,488 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: python_packages +2024-08-04 02:10:33,489 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 02:10:33,490 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: stop_status +2024-08-04 02:10:33,776 DEBUG SenderThread:11309 [sender.py:send():382] send: telemetry +2024-08-04 02:10:34,131 INFO wandb-upload_0:11309 [upload_job.py:push():131] Uploaded file /tmp/tmpcp1trk59wandb/1uhn5dog-wandb-metadata.json +2024-08-04 02:10:34,327 DEBUG SenderThread:11309 [sender.py:send():382] send: exit +2024-08-04 02:10:34,327 INFO SenderThread:11309 [sender.py:send_exit():589] handling exit code: 1 +2024-08-04 02:10:34,327 INFO SenderThread:11309 [sender.py:send_exit():591] handling runtime: 0 +2024-08-04 02:10:34,328 INFO SenderThread:11309 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 02:10:34,329 INFO SenderThread:11309 [sender.py:send_exit():597] send defer +2024-08-04 02:10:34,329 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,329 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-04 02:10:34,329 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,329 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-04 02:10:34,329 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 1 +2024-08-04 02:10:34,329 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,329 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-04 02:10:34,330 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,330 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-04 02:10:34,330 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 2 +2024-08-04 02:10:34,330 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,330 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-04 02:10:34,330 INFO HandlerThread:11309 [system_monitor.py:finish():203] Stopping system monitor +2024-08-04 02:10:34,330 DEBUG SystemMonitor:11309 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 02:10:34,330 INFO HandlerThread:11309 [interfaces.py:finish():202] Joined cpu monitor +2024-08-04 02:10:34,330 DEBUG SystemMonitor:11309 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-04 02:10:34,331 INFO HandlerThread:11309 [interfaces.py:finish():202] Joined disk monitor +2024-08-04 02:10:34,331 DEBUG SystemMonitor:11309 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-04 02:10:34,350 INFO Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt +2024-08-04 02:10:34,350 INFO Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log +2024-08-04 02:10:34,350 INFO Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json +2024-08-04 02:10:34,350 INFO Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json +2024-08-04 02:10:34,365 INFO HandlerThread:11309 [interfaces.py:finish():202] Joined gpu monitor +2024-08-04 02:10:34,366 INFO HandlerThread:11309 [interfaces.py:finish():202] Joined memory monitor +2024-08-04 02:10:34,366 INFO HandlerThread:11309 [interfaces.py:finish():202] Joined network monitor +2024-08-04 02:10:34,366 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,366 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-04 02:10:34,366 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 3 +2024-08-04 02:10:34,366 DEBUG SenderThread:11309 [sender.py:send():382] send: stats +2024-08-04 02:10:34,366 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,367 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-04 02:10:34,367 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,367 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-04 02:10:34,367 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 4 +2024-08-04 02:10:34,367 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,367 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-04 02:10:34,367 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,367 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-04 02:10:34,367 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 5 +2024-08-04 02:10:34,367 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,367 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-04 02:10:34,368 DEBUG SenderThread:11309 [sender.py:send():382] send: summary +2024-08-04 02:10:34,368 INFO SenderThread:11309 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 02:10:34,369 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,369 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-04 02:10:34,369 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 6 +2024-08-04 02:10:34,369 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,369 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-04 02:10:34,369 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,369 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-04 02:10:34,372 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 02:10:34,573 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 7 +2024-08-04 02:10:34,573 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:34,573 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-04 02:10:34,573 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:34,573 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-04 02:10:35,327 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:10:35,350 INFO Thread-12 :11309 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml +2024-08-04 02:10:36,318 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 8 +2024-08-04 02:10:36,318 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:10:36,318 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:36,318 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-04 02:10:36,318 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:36,318 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-04 02:10:36,319 INFO SenderThread:11309 [job_builder.py:build():296] Attempting to build job artifact +2024-08-04 02:10:36,319 INFO SenderThread:11309 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-04 02:10:36,328 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:10:36,333 INFO SenderThread:11309 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-04 02:10:36,342 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 9 +2024-08-04 02:10:36,342 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:10:36,342 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:36,342 DEBUG SenderThread:11309 [sender.py:send():382] send: artifact +2024-08-04 02:10:36,342 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-04 02:10:36,351 INFO Thread-12 :11309 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log +2024-08-04 02:10:37,328 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:10:37,552 INFO wandb-upload_1:11309 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmpteaibpd9 +2024-08-04 02:10:37,910 INFO wandb-upload_0:11309 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp9hfu5wh3 +2024-08-04 02:10:39,216 INFO SenderThread:11309 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MTk4ODAyMA==', 'versionIndex': 2}}} +2024-08-04 02:10:39,216 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:39,216 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-04 02:10:39,216 INFO SenderThread:11309 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-04 02:10:39,352 INFO SenderThread:11309 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_021032-cd2cg2ui/files +2024-08-04 02:10:39,352 INFO SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt requirements.txt +2024-08-04 02:10:39,352 INFO SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml config.yaml +2024-08-04 02:10:39,354 INFO SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json wandb-metadata.json +2024-08-04 02:10:39,354 INFO SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json wandb-summary.json +2024-08-04 02:10:39,355 INFO SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log output.log +2024-08-04 02:10:39,357 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 10 +2024-08-04 02:10:39,357 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:10:39,358 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:39,358 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-04 02:10:39,359 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:39,359 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-04 02:10:39,359 INFO SenderThread:11309 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 02:10:39,788 INFO wandb-upload_0:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml +2024-08-04 02:10:39,856 INFO wandb-upload_1:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt +2024-08-04 02:10:39,931 INFO wandb-upload_3:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log +2024-08-04 02:10:39,937 INFO wandb-upload_2:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json +2024-08-04 02:10:40,137 INFO Thread-11 (_thread_body):11309 [sender.py:transition_state():617] send defer: 11 +2024-08-04 02:10:40,137 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:40,137 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-04 02:10:40,138 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:40,138 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-04 02:10:40,138 INFO SenderThread:11309 [file_pusher.py:join():178] waiting for file pusher +2024-08-04 02:10:40,138 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 12 +2024-08-04 02:10:40,138 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:40,138 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-04 02:10:40,138 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:40,138 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-04 02:10:40,138 INFO SenderThread:11309 [file_stream.py:finish():595] file stream finish called +2024-08-04 02:10:40,324 INFO SenderThread:11309 [file_stream.py:finish():599] file stream finish is done +2024-08-04 02:10:40,324 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 13 +2024-08-04 02:10:40,324 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:40,324 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-04 02:10:40,324 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:40,324 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-04 02:10:40,324 INFO SenderThread:11309 [sender.py:transition_state():617] send defer: 14 +2024-08-04 02:10:40,325 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:10:40,325 DEBUG SenderThread:11309 [sender.py:send():382] send: final +2024-08-04 02:10:40,325 INFO HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-04 02:10:40,325 DEBUG SenderThread:11309 [sender.py:send():382] send: footer +2024-08-04 02:10:40,325 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: defer +2024-08-04 02:10:40,325 INFO SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-04 02:10:40,325 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:10:40,326 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:10:40,326 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:10:40,326 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: server_info +2024-08-04 02:10:40,326 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:10:40,326 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: server_info +2024-08-04 02:10:40,328 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: get_summary +2024-08-04 02:10:40,328 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-04 02:10:40,328 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 02:10:40,329 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: job_info +2024-08-04 02:10:40,492 DEBUG SenderThread:11309 [sender.py:send_request():409] send_request: job_info +2024-08-04 02:10:40,492 INFO MainThread:11309 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-04 02:10:40,492 INFO MainThread:11309 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-04 02:10:40,492 INFO MainThread:11309 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-04 02:10:40,492 DEBUG HandlerThread:11309 [handler.py:handle_request():146] handle_request: shutdown +2024-08-04 02:10:40,492 INFO HandlerThread:11309 [handler.py:finish():869] shutting down handler +2024-08-04 02:10:41,329 INFO WriterThread:11309 [datastore.py:close():296] close: /project/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb +2024-08-04 02:10:41,492 INFO SenderThread:11309 [sender.py:finish():1572] shutting down sender +2024-08-04 02:10:41,492 INFO SenderThread:11309 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 02:10:41,492 INFO SenderThread:11309 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_021032-cd2cg2ui/logs/debug.log b/wandb/run-20240804_021032-cd2cg2ui/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..683f32c3d3ac6b446f665bc34f76a00bf98cf0e4 --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/logs/debug.log @@ -0,0 +1,28 @@ +2024-08-04 02:10:32,410 INFO MainThread:11238 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_setup.py:_flush():76] Configure stats pid to 11238 +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_021032-cd2cg2ui/logs/debug.log +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_021032-cd2cg2ui/logs/debug-internal.log +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_init.py:init():566] calling init triggers +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample_train_2024-08-04-02:10:14', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample', 'save': '/work/llm_recipes/models/tiny-mistral-sample', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 8192, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40} +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_init.py:init():616] starting backend +2024-08-04 02:10:32,411 INFO MainThread:11238 [wandb_init.py:init():620] setting up manager +2024-08-04 02:10:32,416 INFO MainThread:11238 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 02:10:32,416 INFO MainThread:11238 [wandb_init.py:init():628] backend started and connected +2024-08-04 02:10:32,422 INFO MainThread:11238 [wandb_init.py:init():720] updated telemetry +2024-08-04 02:10:32,879 INFO MainThread:11238 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 02:10:33,353 INFO MainThread:11238 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 02:10:33,431 INFO MainThread:11238 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 02:10:33,431 INFO MainThread:11238 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 02:10:33,487 INFO MainThread:11238 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 02:10:33,487 INFO MainThread:11238 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 02:10:33,487 INFO MainThread:11238 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 02:10:33,488 INFO MainThread:11238 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 02:10:33,489 INFO MainThread:11238 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 02:10:41,493 WARNING MsgRouterThr:11238 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb b/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e21b1ef2339aad37107062986494457e50733702 Binary files /dev/null and b/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb differ diff --git a/wandb/run-20240804_035140-nyllt780/files/config.yaml b/wandb/run-20240804_035140-nyllt780/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a61011547861f20bd53ea753ffc37bd16c096aa5 --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-llama-sample_train_2024-08-04-03:51:30 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-llama-sample +save: + desc: null + value: /work/llm_recipes/models/tiny-llama-sample +base_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-llama-sample +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32000 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722711100.510646 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 2048 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 2048 +num_attention_heads: + desc: null + value: 32 +num_hidden_layers: + desc: null + value: 22 +model_architecture: + desc: null + value: LlamaForCausalLM diff --git a/wandb/run-20240804_035140-nyllt780/files/output.log b/wandb/run-20240804_035140-nyllt780/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6425ca0eecdb8bd2667aaf570b91807c42116124 --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/files/output.log @@ -0,0 +1,130 @@ +Created Hugging Face repository with ID koichi12/tiny-llama-sample. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama-sample, skipping model loading +--> Model /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +--> /share/pretrained_lm/meta-llama/TinyLlama_v1.1 has 1100.048384 Million params +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 323200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama-sample, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(32000, 2048) + (layers): ModuleList( + (0-21): 22 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=2048, out_features=2048, bias=False) + (k_proj): Linear(in_features=2048, out_features=256, bias=False) + (v_proj): Linear(in_features=2048, out_features=256, bias=False) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=2048, out_features=5632, bias=False) + (up_proj): Linear(in_features=2048, out_features=5632, bias=False) + (down_proj): Linear(in_features=5632, out_features=2048, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=32000, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5632, + "label_smoothing": 0.0, + "max_position_embeddings": 2048, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 22, + "num_key_value_heads": 4, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32000 +} +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 281, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train + loss: torch.Tensor = model(**batch).loss + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/llama/modeling_llama.py", line 1141, in forward + outputs = self.model( + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/llama/modeling_llama.py", line 908, in forward + cache_position = torch.arange( +RuntimeError: CUDA error: device-side assert triggered +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1. +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. \ No newline at end of file diff --git a/wandb/run-20240804_035140-nyllt780/files/requirements.txt b/wandb/run-20240804_035140-nyllt780/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json b/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4547cc76176086c618b3624a05d221b2503bda50 --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-03T18:51:41.236802", + "startedAt": "2024-08-03T18:51:40.498160", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "--save", + "/work/llm_recipes/models/tiny-llama-sample", + "--load", + "/work/llm_recipes/models/tiny-llama-sample", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-llama-sample", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-llama-sample_train_2024-08-04-03:51:30" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json b/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..93c8ebf5f49580afea0a6178daa5e287c001b844 --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 63}} \ No newline at end of file diff --git a/wandb/run-20240804_035140-nyllt780/logs/debug-internal.log b/wandb/run-20240804_035140-nyllt780/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..66e60451faf14d782a7445cbbf90ba50143af308 --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/logs/debug-internal.log @@ -0,0 +1,215 @@ +2024-08-04 03:51:40,525 INFO StreamThr :12425 [internal.py:wandb_internal():86] W&B internal server running at pid: 12425, started at: 2024-08-04 03:51:40.511089 +2024-08-04 03:51:40,526 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status +2024-08-04 03:51:40,527 INFO WriterThread:12425 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb +2024-08-04 03:51:40,541 DEBUG SenderThread:12425 [sender.py:send():382] send: header +2024-08-04 03:51:40,658 DEBUG SenderThread:12425 [sender.py:send():382] send: run +2024-08-04 03:51:41,127 INFO SenderThread:12425 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_035140-nyllt780/files +2024-08-04 03:51:41,127 INFO SenderThread:12425 [sender.py:_start_run_threads():1136] run started: nyllt780 with start time 1722711100.510646 +2024-08-04 03:51:41,132 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 03:51:41,133 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: check_version +2024-08-04 03:51:41,218 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 03:51:41,224 DEBUG HandlerThread:12425 [system_info.py:__init__():27] System info init +2024-08-04 03:51:41,224 DEBUG HandlerThread:12425 [system_info.py:__init__():42] System info init done +2024-08-04 03:51:41,224 INFO HandlerThread:12425 [system_monitor.py:start():194] Starting system monitor +2024-08-04 03:51:41,224 INFO SystemMonitor:12425 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 03:51:41,224 INFO HandlerThread:12425 [system_monitor.py:probe():214] Collecting system info +2024-08-04 03:51:41,225 INFO SystemMonitor:12425 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 03:51:41,225 INFO SystemMonitor:12425 [interfaces.py:start():190] Started disk monitoring +2024-08-04 03:51:41,226 INFO SystemMonitor:12425 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 03:51:41,227 INFO SystemMonitor:12425 [interfaces.py:start():190] Started memory monitoring +2024-08-04 03:51:41,228 INFO SystemMonitor:12425 [interfaces.py:start():190] Started network monitoring +2024-08-04 03:51:41,236 DEBUG HandlerThread:12425 [system_info.py:probe():151] Probing system +2024-08-04 03:51:41,238 DEBUG HandlerThread:12425 [system_info.py:_probe_git():136] Probing git +2024-08-04 03:51:41,249 DEBUG HandlerThread:12425 [system_info.py:_probe_git():144] Probing git done +2024-08-04 03:51:41,249 DEBUG HandlerThread:12425 [system_info.py:probe():199] Probing system done +2024-08-04 03:51:41,250 DEBUG HandlerThread:12425 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T18:51:41.236802', 'startedAt': '2024-08-03T18:51:40.498160', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', '--save', '/work/llm_recipes/models/tiny-llama-sample', '--load', '/work/llm_recipes/models/tiny-llama-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-llama-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-llama-sample_train_2024-08-04-03:51:30'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-04 03:51:41,250 INFO HandlerThread:12425 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 03:51:41,250 INFO HandlerThread:12425 [system_monitor.py:probe():227] Publishing system info +2024-08-04 03:51:41,302 INFO HandlerThread:12425 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 03:51:41,308 DEBUG SenderThread:12425 [sender.py:send():382] send: files +2024-08-04 03:51:41,308 INFO SenderThread:12425 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 03:51:41,317 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 03:51:41,317 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 03:51:41,317 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 03:51:41,317 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: python_packages +2024-08-04 03:51:41,335 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: stop_status +2024-08-04 03:51:41,618 DEBUG SenderThread:12425 [sender.py:send():382] send: telemetry +2024-08-04 03:51:41,985 INFO wandb-upload_0:12425 [upload_job.py:push():131] Uploaded file /tmp/tmpxkt1klm7wandb/bxmu94ae-wandb-metadata.json +2024-08-04 03:51:42,129 INFO Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:51:42,129 INFO Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/requirements.txt +2024-08-04 03:51:42,129 INFO Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json +2024-08-04 03:51:44,129 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:51:45,608 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:51:48,132 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:51:50,610 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:51:55,611 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:51:56,316 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 03:51:56,317 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: stop_status +2024-08-04 03:51:56,317 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 03:52:01,592 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:06,593 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:11,316 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 03:52:11,317 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: stop_status +2024-08-04 03:52:11,360 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 03:52:12,552 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:13,160 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/config.yaml +2024-08-04 03:52:17,755 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:22,755 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:26,316 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 03:52:26,317 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: stop_status +2024-08-04 03:52:26,360 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 03:52:28,589 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:33,590 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:38,591 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:41,228 DEBUG SystemMonitor:12425 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 03:52:41,230 DEBUG SenderThread:12425 [sender.py:send():382] send: stats +2024-08-04 03:52:41,316 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 03:52:41,317 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: stop_status +2024-08-04 03:52:41,360 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 03:52:43,008 DEBUG SenderThread:12425 [sender.py:send():382] send: config +2024-08-04 03:52:43,008 DEBUG SenderThread:12425 [sender.py:send():382] send: config +2024-08-04 03:52:44,011 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:44,176 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:52:44,726 DEBUG SenderThread:12425 [sender.py:send():382] send: exit +2024-08-04 03:52:44,726 INFO SenderThread:12425 [sender.py:send_exit():589] handling exit code: 1 +2024-08-04 03:52:44,726 INFO SenderThread:12425 [sender.py:send_exit():591] handling runtime: 63 +2024-08-04 03:52:44,741 INFO SenderThread:12425 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 03:52:44,741 INFO SenderThread:12425 [sender.py:send_exit():597] send defer +2024-08-04 03:52:44,742 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,742 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-04 03:52:44,742 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,742 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-04 03:52:44,742 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 1 +2024-08-04 03:52:44,742 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,742 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-04 03:52:44,742 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,742 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-04 03:52:44,742 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 2 +2024-08-04 03:52:44,742 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,742 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-04 03:52:44,742 INFO HandlerThread:12425 [system_monitor.py:finish():203] Stopping system monitor +2024-08-04 03:52:44,743 DEBUG SystemMonitor:12425 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-04 03:52:44,743 INFO HandlerThread:12425 [interfaces.py:finish():202] Joined cpu monitor +2024-08-04 03:52:44,743 DEBUG SystemMonitor:12425 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-04 03:52:44,743 INFO HandlerThread:12425 [interfaces.py:finish():202] Joined disk monitor +2024-08-04 03:52:44,777 INFO HandlerThread:12425 [interfaces.py:finish():202] Joined gpu monitor +2024-08-04 03:52:44,777 INFO HandlerThread:12425 [interfaces.py:finish():202] Joined memory monitor +2024-08-04 03:52:44,777 INFO HandlerThread:12425 [interfaces.py:finish():202] Joined network monitor +2024-08-04 03:52:44,778 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,778 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-04 03:52:44,778 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 3 +2024-08-04 03:52:44,778 DEBUG SenderThread:12425 [sender.py:send():382] send: stats +2024-08-04 03:52:44,778 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,778 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-04 03:52:44,778 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,779 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-04 03:52:44,779 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 4 +2024-08-04 03:52:44,779 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,779 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-04 03:52:44,779 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,779 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-04 03:52:44,779 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 5 +2024-08-04 03:52:44,779 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,779 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-04 03:52:44,779 DEBUG SenderThread:12425 [sender.py:send():382] send: summary +2024-08-04 03:52:44,780 INFO SenderThread:12425 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 03:52:44,780 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,780 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-04 03:52:44,780 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 6 +2024-08-04 03:52:44,780 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,781 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-04 03:52:44,781 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,781 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-04 03:52:44,781 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 7 +2024-08-04 03:52:44,781 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 03:52:44,781 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:44,781 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-04 03:52:44,781 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:44,781 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-04 03:52:45,177 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/config.yaml +2024-08-04 03:52:45,177 INFO Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json +2024-08-04 03:52:45,726 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 03:52:46,178 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:52:47,600 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 8 +2024-08-04 03:52:47,600 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 03:52:47,600 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:47,601 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-04 03:52:47,601 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:47,601 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-04 03:52:47,601 INFO SenderThread:12425 [job_builder.py:build():296] Attempting to build job artifact +2024-08-04 03:52:47,602 INFO SenderThread:12425 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-04 03:52:47,616 INFO SenderThread:12425 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-04 03:52:47,688 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 9 +2024-08-04 03:52:47,689 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:47,689 DEBUG SenderThread:12425 [sender.py:send():382] send: artifact +2024-08-04 03:52:47,689 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-04 03:52:47,727 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 03:52:48,179 INFO Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:52:48,575 INFO SenderThread:12425 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}} +2024-08-04 03:52:48,575 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:48,575 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-04 03:52:48,575 INFO SenderThread:12425 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-04 03:52:49,180 INFO SenderThread:12425 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_035140-nyllt780/files +2024-08-04 03:52:49,180 INFO SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/requirements.txt requirements.txt +2024-08-04 03:52:49,180 INFO SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/config.yaml config.yaml +2024-08-04 03:52:49,181 INFO SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json wandb-metadata.json +2024-08-04 03:52:49,182 INFO SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json wandb-summary.json +2024-08-04 03:52:49,183 INFO SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/output.log output.log +2024-08-04 03:52:49,185 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 10 +2024-08-04 03:52:49,185 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 03:52:49,185 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:49,187 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-04 03:52:49,187 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:49,187 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-04 03:52:49,187 INFO SenderThread:12425 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 03:52:49,580 INFO wandb-upload_0:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/requirements.txt +2024-08-04 03:52:49,719 INFO wandb-upload_1:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/config.yaml +2024-08-04 03:52:49,727 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 03:52:49,727 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 03:52:49,752 INFO wandb-upload_2:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json +2024-08-04 03:52:49,778 INFO wandb-upload_3:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/output.log +2024-08-04 03:52:49,978 INFO Thread-11 (_thread_body):12425 [sender.py:transition_state():617] send defer: 11 +2024-08-04 03:52:49,978 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:49,979 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-04 03:52:49,979 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:49,979 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-04 03:52:49,979 INFO SenderThread:12425 [file_pusher.py:join():178] waiting for file pusher +2024-08-04 03:52:49,979 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 12 +2024-08-04 03:52:49,979 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:49,979 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-04 03:52:49,979 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:49,979 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-04 03:52:49,979 INFO SenderThread:12425 [file_stream.py:finish():595] file stream finish called +2024-08-04 03:52:50,544 INFO SenderThread:12425 [file_stream.py:finish():599] file stream finish is done +2024-08-04 03:52:50,544 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 13 +2024-08-04 03:52:50,545 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:50,545 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-04 03:52:50,545 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:50,545 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-04 03:52:50,545 INFO SenderThread:12425 [sender.py:transition_state():617] send defer: 14 +2024-08-04 03:52:50,545 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer +2024-08-04 03:52:50,545 DEBUG SenderThread:12425 [sender.py:send():382] send: final +2024-08-04 03:52:50,545 INFO HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-04 03:52:50,545 DEBUG SenderThread:12425 [sender.py:send():382] send: footer +2024-08-04 03:52:50,546 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: defer +2024-08-04 03:52:50,546 INFO SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-04 03:52:50,546 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 03:52:50,546 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 03:52:50,546 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 03:52:50,547 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: server_info +2024-08-04 03:52:50,547 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: get_summary +2024-08-04 03:52:50,547 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 03:52:50,547 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-04 03:52:50,547 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: server_info +2024-08-04 03:52:50,548 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 03:52:50,549 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: job_info +2024-08-04 03:52:50,716 DEBUG SenderThread:12425 [sender.py:send_request():409] send_request: job_info +2024-08-04 03:52:50,717 INFO MainThread:12425 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-04 03:52:50,717 INFO MainThread:12425 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-04 03:52:50,717 INFO MainThread:12425 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-04 03:52:50,717 DEBUG HandlerThread:12425 [handler.py:handle_request():146] handle_request: shutdown +2024-08-04 03:52:50,717 INFO HandlerThread:12425 [handler.py:finish():869] shutting down handler +2024-08-04 03:52:51,549 INFO WriterThread:12425 [datastore.py:close():296] close: /project/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb +2024-08-04 03:52:51,717 INFO SenderThread:12425 [sender.py:finish():1572] shutting down sender +2024-08-04 03:52:51,717 INFO SenderThread:12425 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 03:52:51,717 INFO SenderThread:12425 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_035140-nyllt780/logs/debug.log b/wandb/run-20240804_035140-nyllt780/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c04e0f5f8e0d3931f01d9acb9c31d5d8614237f2 --- /dev/null +++ b/wandb/run-20240804_035140-nyllt780/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-04 03:51:40,503 INFO MainThread:12354 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_setup.py:_flush():76] Configure stats pid to 12354 +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_035140-nyllt780/logs/debug.log +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_035140-nyllt780/logs/debug-internal.log +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_init.py:init():566] calling init triggers +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-llama-sample_train_2024-08-04-03:51:30', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-llama-sample', 'save': '/work/llm_recipes/models/tiny-llama-sample', 'base_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-llama-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32000, 'gradient_accumulation_steps': 40} +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_init.py:init():616] starting backend +2024-08-04 03:51:40,504 INFO MainThread:12354 [wandb_init.py:init():620] setting up manager +2024-08-04 03:51:40,509 INFO MainThread:12354 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 03:51:40,510 INFO MainThread:12354 [wandb_init.py:init():628] backend started and connected +2024-08-04 03:51:40,515 INFO MainThread:12354 [wandb_init.py:init():720] updated telemetry +2024-08-04 03:51:40,654 INFO MainThread:12354 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 03:51:41,132 INFO MainThread:12354 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 03:51:41,211 INFO MainThread:12354 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 03:51:41,211 INFO MainThread:12354 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 03:51:41,316 INFO MainThread:12354 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 03:51:41,317 INFO MainThread:12354 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 03:51:41,317 INFO MainThread:12354 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 03:51:41,317 INFO MainThread:12354 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 03:51:41,318 INFO MainThread:12354 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 03:52:43,007 INFO MainThread:12354 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 2048, 'model_type': 'llama', 'max_position_embeddings': 2048, 'num_attention_heads': 32, 'num_hidden_layers': 22, 'model_architecture': 'LlamaForCausalLM'} +2024-08-04 03:52:43,008 INFO MainThread:12354 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-04 03:52:51,718 WARNING MsgRouterThr:12354 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb b/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d90ec97e81090f4490417918113a0cc724a3d69e Binary files /dev/null and b/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb differ diff --git a/wandb/run-20240804_211947-niq3ake5/files/config.yaml b/wandb/run-20240804_211947-niq3ake5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da04d17917542c1fa993c4b3c5ec075e3e5ad554 --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-llama_train_2024-08-04-21:19:16 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-llama +save: + desc: null + value: /work/llm_recipes/models/tiny-llama +base_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 2000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 2000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-llama +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32000 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722773987.17106 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 2048 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 2048 +num_attention_heads: + desc: null + value: 32 +num_hidden_layers: + desc: null + value: 22 +model_architecture: + desc: null + value: LlamaForCausalLM diff --git a/wandb/run-20240804_211947-niq3ake5/files/output.log b/wandb/run-20240804_211947-niq3ake5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9abb53ace2c37c8a4d95ebea31daf244e7fa2440 --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/files/output.log @@ -0,0 +1,135 @@ +Created Hugging Face repository with ID koichi12/tiny-llama. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama, skipping model loading +--> Model /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +--> /share/pretrained_lm/meta-llama/TinyLlama_v1.1 has 1100.048384 Million params +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 640000 + validation: 35200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(32000, 2048) + (layers): ModuleList( + (0-21): 22 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=2048, out_features=2048, bias=False) + (k_proj): Linear(in_features=2048, out_features=256, bias=False) + (v_proj): Linear(in_features=2048, out_features=256, bias=False) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=2048, out_features=5632, bias=False) + (up_proj): Linear(in_features=2048, out_features=5632, bias=False) + (down_proj): Linear(in_features=5632, out_features=2048, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=32000, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5632, + "label_smoothing": 0.0, + "max_position_embeddings": 2048, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 22, + "num_key_value_heads": 4, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32000 +} +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 281, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 104, in train + batch = next(train_dataloader) + File "/project/src/llama_recipes/utils/train_utils.py", line 24, in cyclic_iter + for x in iter: + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 631, in __next__ + data = self._next_data() + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 1346, in _next_data + return self._process_data(data) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 1372, in _process_data + data.reraise() + File "/usr/local/lib/python3.10/dist-packages/torch/_utils.py", line 705, in reraise + raise exception +RuntimeError: Caught RuntimeError in DataLoader worker process 0. +Original Traceback (most recent call last): + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop + data = fetcher.fetch(index) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch + return self.collate_fn(data) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 277, in default_collate + return collate(batch, collate_fn_map=default_collate_fn_map) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 129, in collate + return elem_type({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 129, in + return elem_type({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 121, in collate + return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 174, in collate_tensor_fn + return torch.stack(batch, 0, out=out) +RuntimeError: stack expects each tensor to be equal size, but got [513] at entry 0 and [543] at entry 1 \ No newline at end of file diff --git a/wandb/run-20240804_211947-niq3ake5/files/requirements.txt b/wandb/run-20240804_211947-niq3ake5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json b/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e2c17885e01bd61999de97a48c75399f4b43203a --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-04T12:19:47.940599", + "startedAt": "2024-08-04T12:19:47.157671", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "2000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "2000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "--save", + "/work/llm_recipes/models/tiny-llama", + "--load", + "/work/llm_recipes/models/tiny-llama", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-llama", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-llama_train_2024-08-04-21:19:16" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.044999999999, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json b/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..619771471916cfbd0a1671799f389b4386995814 --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 67}} \ No newline at end of file diff --git a/wandb/run-20240804_211947-niq3ake5/logs/debug-internal.log b/wandb/run-20240804_211947-niq3ake5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c63e2a6a4e4077420d9a22dbb21abef5efa7250a --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/logs/debug-internal.log @@ -0,0 +1,213 @@ +2024-08-04 21:19:47,171 INFO StreamThr :10096 [internal.py:wandb_internal():86] W&B internal server running at pid: 10096, started at: 2024-08-04 21:19:47.170590 +2024-08-04 21:19:47,173 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status +2024-08-04 21:19:47,176 INFO WriterThread:10096 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb +2024-08-04 21:19:47,177 DEBUG SenderThread:10096 [sender.py:send():382] send: header +2024-08-04 21:19:47,316 DEBUG SenderThread:10096 [sender.py:send():382] send: run +2024-08-04 21:19:47,822 INFO SenderThread:10096 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_211947-niq3ake5/files +2024-08-04 21:19:47,822 INFO SenderThread:10096 [sender.py:_start_run_threads():1136] run started: niq3ake5 with start time 1722773987.17106 +2024-08-04 21:19:47,827 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 21:19:47,827 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: check_version +2024-08-04 21:19:47,917 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 21:19:47,923 DEBUG HandlerThread:10096 [system_info.py:__init__():27] System info init +2024-08-04 21:19:47,923 DEBUG HandlerThread:10096 [system_info.py:__init__():42] System info init done +2024-08-04 21:19:47,923 INFO HandlerThread:10096 [system_monitor.py:start():194] Starting system monitor +2024-08-04 21:19:47,923 INFO SystemMonitor:10096 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 21:19:47,924 INFO HandlerThread:10096 [system_monitor.py:probe():214] Collecting system info +2024-08-04 21:19:47,924 INFO SystemMonitor:10096 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 21:19:47,924 INFO SystemMonitor:10096 [interfaces.py:start():190] Started disk monitoring +2024-08-04 21:19:47,925 INFO SystemMonitor:10096 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 21:19:47,926 INFO SystemMonitor:10096 [interfaces.py:start():190] Started memory monitoring +2024-08-04 21:19:47,927 INFO SystemMonitor:10096 [interfaces.py:start():190] Started network monitoring +2024-08-04 21:19:47,940 DEBUG HandlerThread:10096 [system_info.py:probe():151] Probing system +2024-08-04 21:19:47,942 DEBUG HandlerThread:10096 [system_info.py:_probe_git():136] Probing git +2024-08-04 21:19:47,954 DEBUG HandlerThread:10096 [system_info.py:_probe_git():144] Probing git done +2024-08-04 21:19:47,954 DEBUG HandlerThread:10096 [system_info.py:probe():199] Probing system done +2024-08-04 21:19:47,954 DEBUG HandlerThread:10096 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-04T12:19:47.940599', 'startedAt': '2024-08-04T12:19:47.157671', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '2000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '2000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', '--save', '/work/llm_recipes/models/tiny-llama', '--load', '/work/llm_recipes/models/tiny-llama', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-llama', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-llama_train_2024-08-04-21:19:16'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.044999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-04 21:19:47,954 INFO HandlerThread:10096 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 21:19:47,954 INFO HandlerThread:10096 [system_monitor.py:probe():227] Publishing system info +2024-08-04 21:19:47,955 INFO HandlerThread:10096 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 21:19:47,982 DEBUG SenderThread:10096 [sender.py:send():382] send: files +2024-08-04 21:19:47,982 INFO SenderThread:10096 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 21:19:47,991 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 21:19:47,992 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: python_packages +2024-08-04 21:19:47,992 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 21:19:47,992 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 21:19:47,993 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: stop_status +2024-08-04 21:19:48,264 DEBUG SenderThread:10096 [sender.py:send():382] send: telemetry +2024-08-04 21:19:48,653 INFO wandb-upload_0:10096 [upload_job.py:push():131] Uploaded file /tmp/tmpc_z53slvwandb/somhprnl-wandb-metadata.json +2024-08-04 21:19:48,823 INFO Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/requirements.txt +2024-08-04 21:19:48,824 INFO Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json +2024-08-04 21:19:52,265 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:19:53,826 INFO Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/output.log +2024-08-04 21:19:55,827 INFO Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log +2024-08-04 21:19:57,441 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:19:59,829 INFO Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log +2024-08-04 21:20:02,739 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:02,991 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 21:20:02,991 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 21:20:02,992 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: stop_status +2024-08-04 21:20:08,241 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:13,242 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:17,991 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 21:20:17,991 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: stop_status +2024-08-04 21:20:18,032 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 21:20:18,266 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:18,841 INFO Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml +2024-08-04 21:20:23,460 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:28,461 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:32,991 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 21:20:32,991 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: stop_status +2024-08-04 21:20:33,032 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 21:20:34,171 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:39,171 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:44,172 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:47,927 DEBUG SystemMonitor:10096 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 21:20:47,929 DEBUG SenderThread:10096 [sender.py:send():382] send: stats +2024-08-04 21:20:47,991 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 21:20:47,991 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: stop_status +2024-08-04 21:20:48,032 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 21:20:49,259 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:53,862 INFO Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log +2024-08-04 21:20:54,334 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:54,437 DEBUG SenderThread:10096 [sender.py:send():382] send: config +2024-08-04 21:20:54,437 DEBUG SenderThread:10096 [sender.py:send():382] send: config +2024-08-04 21:20:55,009 DEBUG SenderThread:10096 [sender.py:send():382] send: exit +2024-08-04 21:20:55,009 INFO SenderThread:10096 [sender.py:send_exit():589] handling exit code: 1 +2024-08-04 21:20:55,009 INFO SenderThread:10096 [sender.py:send_exit():591] handling runtime: 67 +2024-08-04 21:20:55,010 INFO SenderThread:10096 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 21:20:55,011 INFO SenderThread:10096 [sender.py:send_exit():597] send defer +2024-08-04 21:20:55,011 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,011 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-04 21:20:55,011 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,011 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-04 21:20:55,011 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 1 +2024-08-04 21:20:55,011 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,011 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-04 21:20:55,011 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,011 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-04 21:20:55,012 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 2 +2024-08-04 21:20:55,012 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,012 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-04 21:20:55,012 INFO HandlerThread:10096 [system_monitor.py:finish():203] Stopping system monitor +2024-08-04 21:20:55,012 DEBUG SystemMonitor:10096 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-04 21:20:55,012 INFO HandlerThread:10096 [interfaces.py:finish():202] Joined cpu monitor +2024-08-04 21:20:55,012 DEBUG SystemMonitor:10096 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-04 21:20:55,012 INFO HandlerThread:10096 [interfaces.py:finish():202] Joined disk monitor +2024-08-04 21:20:55,046 INFO HandlerThread:10096 [interfaces.py:finish():202] Joined gpu monitor +2024-08-04 21:20:55,047 INFO HandlerThread:10096 [interfaces.py:finish():202] Joined memory monitor +2024-08-04 21:20:55,047 INFO HandlerThread:10096 [interfaces.py:finish():202] Joined network monitor +2024-08-04 21:20:55,047 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,047 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-04 21:20:55,047 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 3 +2024-08-04 21:20:55,047 DEBUG SenderThread:10096 [sender.py:send():382] send: stats +2024-08-04 21:20:55,047 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,048 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-04 21:20:55,048 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,048 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-04 21:20:55,048 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 4 +2024-08-04 21:20:55,048 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,048 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-04 21:20:55,048 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,048 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-04 21:20:55,048 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 5 +2024-08-04 21:20:55,048 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,048 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-04 21:20:55,049 DEBUG SenderThread:10096 [sender.py:send():382] send: summary +2024-08-04 21:20:55,050 INFO SenderThread:10096 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 21:20:55,050 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,050 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-04 21:20:55,050 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 6 +2024-08-04 21:20:55,050 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,050 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-04 21:20:55,050 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,050 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-04 21:20:55,053 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 21:20:55,265 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 7 +2024-08-04 21:20:55,265 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,265 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-04 21:20:55,265 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,265 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-04 21:20:55,512 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 8 +2024-08-04 21:20:55,513 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,513 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-04 21:20:55,513 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:55,513 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-04 21:20:55,513 INFO SenderThread:10096 [job_builder.py:build():296] Attempting to build job artifact +2024-08-04 21:20:55,514 INFO SenderThread:10096 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-04 21:20:55,528 INFO SenderThread:10096 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-04 21:20:55,537 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 9 +2024-08-04 21:20:55,537 DEBUG SenderThread:10096 [sender.py:send():382] send: artifact +2024-08-04 21:20:55,537 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:55,539 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-04 21:20:55,864 INFO Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml +2024-08-04 21:20:55,864 INFO Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log +2024-08-04 21:20:55,864 INFO Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json +2024-08-04 21:20:56,009 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 21:20:57,540 INFO SenderThread:10096 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}} +2024-08-04 21:20:57,540 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:57,540 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-04 21:20:57,540 INFO SenderThread:10096 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-04 21:20:57,865 INFO SenderThread:10096 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_211947-niq3ake5/files +2024-08-04 21:20:57,865 INFO SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/requirements.txt requirements.txt +2024-08-04 21:20:57,865 INFO SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml config.yaml +2024-08-04 21:20:57,867 INFO SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json wandb-metadata.json +2024-08-04 21:20:57,867 INFO SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json wandb-summary.json +2024-08-04 21:20:57,869 INFO SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/output.log output.log +2024-08-04 21:20:57,869 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 10 +2024-08-04 21:20:57,870 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 21:20:57,872 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:57,872 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-04 21:20:57,872 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:57,872 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-04 21:20:57,872 INFO SenderThread:10096 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 21:20:58,009 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 21:20:58,010 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 21:20:58,272 INFO wandb-upload_1:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml +2024-08-04 21:20:58,376 INFO wandb-upload_0:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/requirements.txt +2024-08-04 21:20:58,453 INFO wandb-upload_2:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json +2024-08-04 21:20:58,476 INFO wandb-upload_3:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/output.log +2024-08-04 21:20:58,677 INFO Thread-11 (_thread_body):10096 [sender.py:transition_state():617] send defer: 11 +2024-08-04 21:20:58,677 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:58,677 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-04 21:20:58,677 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:58,677 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-04 21:20:58,677 INFO SenderThread:10096 [file_pusher.py:join():178] waiting for file pusher +2024-08-04 21:20:58,677 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 12 +2024-08-04 21:20:58,677 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:58,677 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-04 21:20:58,678 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:58,678 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-04 21:20:58,678 INFO SenderThread:10096 [file_stream.py:finish():595] file stream finish called +2024-08-04 21:20:58,860 INFO SenderThread:10096 [file_stream.py:finish():599] file stream finish is done +2024-08-04 21:20:58,860 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 13 +2024-08-04 21:20:58,860 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:58,860 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-04 21:20:58,860 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:58,860 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-04 21:20:58,860 INFO SenderThread:10096 [sender.py:transition_state():617] send defer: 14 +2024-08-04 21:20:58,861 DEBUG SenderThread:10096 [sender.py:send():382] send: final +2024-08-04 21:20:58,861 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer +2024-08-04 21:20:58,861 DEBUG SenderThread:10096 [sender.py:send():382] send: footer +2024-08-04 21:20:58,861 INFO HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-04 21:20:58,861 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: defer +2024-08-04 21:20:58,861 INFO SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-04 21:20:58,861 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 21:20:58,862 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 21:20:58,862 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 21:20:58,862 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 21:20:58,862 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: server_info +2024-08-04 21:20:58,863 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: server_info +2024-08-04 21:20:58,864 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: get_summary +2024-08-04 21:20:58,864 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-04 21:20:58,864 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 21:20:58,865 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: job_info +2024-08-04 21:20:59,033 DEBUG SenderThread:10096 [sender.py:send_request():409] send_request: job_info +2024-08-04 21:20:59,033 INFO MainThread:10096 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-04 21:20:59,033 INFO MainThread:10096 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-04 21:20:59,033 INFO MainThread:10096 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-04 21:20:59,033 DEBUG HandlerThread:10096 [handler.py:handle_request():146] handle_request: shutdown +2024-08-04 21:20:59,033 INFO HandlerThread:10096 [handler.py:finish():869] shutting down handler +2024-08-04 21:20:59,865 INFO WriterThread:10096 [datastore.py:close():296] close: /project/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb +2024-08-04 21:21:00,033 INFO SenderThread:10096 [sender.py:finish():1572] shutting down sender +2024-08-04 21:21:00,033 INFO SenderThread:10096 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 21:21:00,033 INFO SenderThread:10096 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_211947-niq3ake5/logs/debug.log b/wandb/run-20240804_211947-niq3ake5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..158475f1f7f46ef0d2f96f78a620371bda4d3798 --- /dev/null +++ b/wandb/run-20240804_211947-niq3ake5/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-04 21:19:47,163 INFO MainThread:10025 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 21:19:47,163 INFO MainThread:10025 [wandb_setup.py:_flush():76] Configure stats pid to 10025 +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tiny llama sample'} +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_211947-niq3ake5/logs/debug.log +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_211947-niq3ake5/logs/debug-internal.log +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_init.py:init():566] calling init triggers +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-llama_train_2024-08-04-21:19:16', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-llama', 'save': '/work/llm_recipes/models/tiny-llama', 'base_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 2000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 2000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-llama', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32000, 'gradient_accumulation_steps': 40} +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_init.py:init():616] starting backend +2024-08-04 21:19:47,164 INFO MainThread:10025 [wandb_init.py:init():620] setting up manager +2024-08-04 21:19:47,169 INFO MainThread:10025 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 21:19:47,170 INFO MainThread:10025 [wandb_init.py:init():628] backend started and connected +2024-08-04 21:19:47,175 INFO MainThread:10025 [wandb_init.py:init():720] updated telemetry +2024-08-04 21:19:47,312 INFO MainThread:10025 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 21:19:47,827 INFO MainThread:10025 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 21:19:47,910 INFO MainThread:10025 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 21:19:47,910 INFO MainThread:10025 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 21:19:47,990 INFO MainThread:10025 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 21:19:47,991 INFO MainThread:10025 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 21:19:47,991 INFO MainThread:10025 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 21:19:47,991 INFO MainThread:10025 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 21:19:47,992 INFO MainThread:10025 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 21:20:54,436 INFO MainThread:10025 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 2048, 'model_type': 'llama', 'max_position_embeddings': 2048, 'num_attention_heads': 32, 'num_hidden_layers': 22, 'model_architecture': 'LlamaForCausalLM'} +2024-08-04 21:20:54,436 INFO MainThread:10025 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-04 21:21:00,034 WARNING MsgRouterThr:10025 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb b/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2dcf1b6fe1551b89720fb87c51a4436863245241 Binary files /dev/null and b/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb differ diff --git a/wandb/run-20240812_055620-qpw0uqx2/files/config.yaml b/wandb/run-20240812_055620-qpw0uqx2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d381f0696205958ffb1ce51ed1608618d9aad9 --- /dev/null +++ b/wandb/run-20240812_055620-qpw0uqx2/files/config.yaml @@ -0,0 +1,314 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Phi/Phi-2 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-sample-Phi-2_train_2024-08-12-05:56:09 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-sample-Phi-2 +save: + desc: null + value: /work/llm_recipes/models/yans-sample-Phi-2 +base_model: + desc: null + value: /share/pretrained_lm/Phi/Phi-2 +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-sample-Phi-2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 50304 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723409780.063771 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20240812_055620-qpw0uqx2/files/output.log b/wandb/run-20240812_055620-qpw0uqx2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9ae8c25630c2e411e9ee65a156e49bcfbc8c9b35 --- /dev/null +++ b/wandb/run-20240812_055620-qpw0uqx2/files/output.log @@ -0,0 +1,9 @@ +Created Hugging Face repository with ID koichi12/yans-sample-Phi-2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loading checkpoint shards: 0%| | 0/2 [00:00 Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/model.pt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Loaded model state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 6403200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/optimizer.pt +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Loaded optimizer state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-12 07:40:01,742] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 70.15750813918478, Tokens per sec: 17447.819809718534, Loss: 4.19143009185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 70.63742600582209, Tokens per sec: 17567.172972090553, Loss: 4.1978888511657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 70.59873626338937, Tokens per sec: 17557.551027521036, Loss: 4.1839118003845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 70.55718504215159, Tokens per sec: 17547.217447548494, Loss: 4.198607921600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 70.35793405694358, Tokens per sec: 17497.66472854483, Loss: 4.179503917694092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 70.57665510851896, Tokens per sec: 17552.05955523834, Loss: 4.1912522315979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 70.3601017703376, Tokens per sec: 17498.203827975543, Loss: 4.189367771148682 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 70.21123247125006, Tokens per sec: 17461.18078119735, Loss: 4.181336879730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 70.72049027318616, Tokens per sec: 17587.830638077194, Loss: 4.160770416259766 +------------------------------------------------------------------ + eval ppl=59.40974807739258, eval loss=4.084458351135254 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/optimizer.pt +[rank0]:[2024-08-12 07:51:18,781] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006773615001293365, 'preprocessing_with_comm': 0.0007632670003658859, 'state_converting': 0.8512952870005392, : 0.8602470229998289}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010, took 4.22s +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 70.33749038984072, Tokens per sec: 17492.580491243345, Loss: 4.118553161621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 70.40664183212826, Tokens per sec: 17509.778107530132, Loss: 4.171367168426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 70.31569128404021, Tokens per sec: 17487.159163147342, Loss: 4.141748428344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 70.1924173230747, Tokens per sec: 17456.501548371612, Loss: 4.183396339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 70.4392579787202, Tokens per sec: 17517.889579327206, Loss: 4.155776500701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 70.72069994073844, Tokens per sec: 17587.882781344095, Loss: 4.1592698097229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 71.15905280756027, Tokens per sec: 17696.89893708052, Loss: 4.171995162963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 71.11103054875616, Tokens per sec: 17684.956042575097, Loss: 4.1017913818359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 71.37717100981507, Tokens per sec: 17751.14383817666, Loss: 4.135533809661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 71.18641901270585, Tokens per sec: 17703.704774815073, Loss: 4.140715599060059 +------------------------------------------------------------------ + eval ppl=60.139408111572266, eval loss=4.096665382385254 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/model.pt +[rank0]:[2024-08-12 08:03:48,178] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063333339985547354, 'preprocessing_with_comm': 0.0006259910005610436, 'state_converting': 0.4321527940010128, : 0.440454599000077}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020, took 3.37s +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 71.27017989682467, Tokens per sec: 17724.53568588889, Loss: 4.155793190002441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 71.38280589518098, Tokens per sec: 17752.545205858045, Loss: 4.111002445220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 71.25229867575862, Tokens per sec: 17720.088716043283, Loss: 4.106908798217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 71.15559719518868, Tokens per sec: 17696.03954364437, Loss: 4.155755043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 71.24632612662501, Tokens per sec: 17718.603373079255, Loss: 4.120436191558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 71.19747838986139, Tokens per sec: 17706.45518635242, Loss: 4.1520562171936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 71.47796715695598, Tokens per sec: 17776.211333580566, Loss: 4.111474990844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 71.39237650508296, Tokens per sec: 17754.925368458855, Loss: 4.099495887756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 71.34803057727774, Tokens per sec: 17743.896759003324, Loss: 4.103178977966309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 71.36224996368146, Tokens per sec: 17747.43304896512, Loss: 4.109351634979248 +------------------------------------------------------------------ + eval ppl=58.537010192871094, eval loss=4.069659233093262 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/optimizer.pt +[rank0]:[2024-08-12 08:16:11,422] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006343725999613525, 'preprocessing_with_comm': 0.0006265169995458564, 'state_converting': 0.4283952869991481, : 0.436715710999124}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030, took 3.45s +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 71.22722350854951, Tokens per sec: 17713.852647933574, Loss: 4.1204447746276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 71.51046736843011, Tokens per sec: 17784.293972896277, Loss: 4.077567100524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 71.45437400486368, Tokens per sec: 17770.34383518491, Loss: 4.0575127601623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 71.45263627481563, Tokens per sec: 17769.911670452148, Loss: 4.075697898864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 71.22883535112506, Tokens per sec: 17714.253505084373, Loss: 4.072705268859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 71.29410213416948, Tokens per sec: 17730.485026133552, Loss: 4.064393520355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 71.30034475638077, Tokens per sec: 17732.03753491517, Loss: 4.042979717254639 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 71.36663665397894, Tokens per sec: 17748.523996803793, Loss: 4.041989326477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 71.41529894277743, Tokens per sec: 17760.62606355337, Loss: 4.023367404937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 71.34130114175095, Tokens per sec: 17742.22318219584, Loss: 4.050424575805664 +------------------------------------------------------------------ + eval ppl=44.75333023071289, eval loss=3.801165819168091 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/optimizer.pt +[rank0]:[2024-08-12 08:28:34,237] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006275761999859242, 'preprocessing_with_comm': 0.0006330559990601614, 'state_converting': 0.4367634319987701, : 0.44501904799835756}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040, took 3.36s +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 71.23060696982249, Tokens per sec: 17714.694097753985, Loss: 4.033792495727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 71.31470436482036, Tokens per sec: 17735.60869739282, Loss: 4.001635551452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 71.3887759065324, Tokens per sec: 17754.02991768839, Loss: 4.0076422691345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 71.28134634767, Tokens per sec: 17727.31272611496, Loss: 3.9992411136627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 71.46195852391331, Tokens per sec: 17772.230066968667, Loss: 4.008784294128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 71.34432872262153, Tokens per sec: 17742.97612634808, Loss: 3.9904208183288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 71.24083757830398, Tokens per sec: 17717.23839868575, Loss: 3.988025188446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 71.55476055802363, Tokens per sec: 17795.309466623512, Loss: 4.0077104568481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 71.35691342039011, Tokens per sec: 17746.10587745883, Loss: 3.9548730850219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 71.31570182202621, Tokens per sec: 17735.85675998879, Loss: 3.9721531867980957 +------------------------------------------------------------------ + eval ppl=60.937713623046875, eval loss=4.109852313995361 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/optimizer.pt +[rank0]:[2024-08-12 08:40:57,176] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006304694001300959, 'preprocessing_with_comm': 0.0006382149986166041, 'state_converting': 0.43427701800101204, : 0.4425508409985923}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050, took 3.45s +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 71.37350751474379, Tokens per sec: 17750.232745357476, Loss: 3.967709541320801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 71.14263995218035, Tokens per sec: 17692.817142404125, Loss: 3.963244676589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 71.30928812625147, Tokens per sec: 17734.261706071396, Loss: 3.9646313190460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 71.15196778439103, Tokens per sec: 17695.1369274128, Loss: 3.958186149597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 71.15607717813529, Tokens per sec: 17696.158912991803, Loss: 3.943728446960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 71.29376966801462, Tokens per sec: 17730.402343471116, Loss: 3.9011611938476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 70.97753280062385, Tokens per sec: 17651.755823294294, Loss: 3.9731457233428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 71.37159549500495, Tokens per sec: 17749.757235654226, Loss: 3.962306261062622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 71.53958743663881, Tokens per sec: 17791.53598756339, Loss: 3.9507999420166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 71.28782184500326, Tokens per sec: 17728.923149769475, Loss: 3.9491472244262695 +------------------------------------------------------------------ + eval ppl=47.686729431152344, eval loss=3.8646531105041504 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/model.pt +[rank0]:[2024-08-12 08:53:21,333] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062968999991426244, 'preprocessing_with_comm': 0.0006548479996126844, 'state_converting': 0.43674980200012214, : 0.4450435329999891}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060, took 3.93s +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 71.40775949372195, Tokens per sec: 17758.75104045073, Loss: 3.92801570892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 71.38289933083125, Tokens per sec: 17752.568442834854, Loss: 3.955374240875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 71.44090394652339, Tokens per sec: 17766.99389934794, Loss: 3.933164119720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 71.42531248816306, Tokens per sec: 17763.1163819837, Loss: 3.934380292892456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 71.3586754627974, Tokens per sec: 17746.544088553193, Loss: 3.9216089248657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 71.15746098299522, Tokens per sec: 17696.503058308368, Loss: 3.908468723297119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 71.33222784673501, Tokens per sec: 17739.966699308727, Loss: 3.8929126262664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 71.38954660220074, Tokens per sec: 17754.221585829204, Loss: 3.9339599609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 71.19306477588056, Tokens per sec: 17705.357542728958, Loss: 3.9401590824127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 71.00667392220164, Tokens per sec: 17659.00307382853, Loss: 3.881852626800537 +------------------------------------------------------------------ + eval ppl=48.8403205871582, eval loss=3.8885562419891357 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/model.pt +[rank0]:[2024-08-12 09:05:44,824] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006354943001497304, 'preprocessing_with_comm': 0.0006292520010902081, 'state_converting': 0.43317293799918843, : 0.4414920090002852}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070, took 3.58s +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 71.26497417886742, Tokens per sec: 17723.241049985943, Loss: 3.8524279594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 71.29788633987809, Tokens per sec: 17731.4261390818, Loss: 3.8764100074768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 71.26911429008719, Tokens per sec: 17724.270674846783, Loss: 3.871471881866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 71.38361475294558, Tokens per sec: 17752.746364720475, Loss: 3.821540355682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 71.25592703095538, Tokens per sec: 17720.99106975221, Loss: 3.891488552093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 71.28480957832222, Tokens per sec: 17728.174014179232, Loss: 3.8562443256378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 71.37878909715518, Tokens per sec: 17751.54624836903, Loss: 3.8804948329925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 71.39719039108451, Tokens per sec: 17756.12255772294, Loss: 3.8488972187042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 71.46791164459815, Tokens per sec: 17773.710578175087, Loss: 3.8672919273376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 71.28736912059362, Tokens per sec: 17728.810559483245, Loss: 3.863307237625122 +------------------------------------------------------------------ + eval ppl=43.56454086303711, eval loss=3.7742435932159424 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/optimizer.pt +[rank0]:[2024-08-12 09:18:07,940] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063301859991042875, 'preprocessing_with_comm': 0.0006200170009833528, 'state_converting': 0.429232709999269, : 0.4375340930000675}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080, took 3.58s +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 71.01955444362599, Tokens per sec: 17662.206394796278, Loss: 3.8227689266204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 71.40514582398568, Tokens per sec: 17758.101033918145, Loss: 3.845733165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 71.36408597524532, Tokens per sec: 17747.889655817104, Loss: 3.8542237281799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 71.3457307432552, Tokens per sec: 17743.32480183606, Loss: 3.8174970149993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 71.55438463922103, Tokens per sec: 17795.215977505883, Loss: 3.806553363800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 71.18267076004283, Tokens per sec: 17702.772603208636, Loss: 3.817875385284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 71.38224068459145, Tokens per sec: 17752.404640823992, Loss: 3.7627954483032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 71.34383716702422, Tokens per sec: 17742.853878940557, Loss: 3.8104918003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 71.33998124686028, Tokens per sec: 17741.89493096755, Loss: 3.791606903076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 71.32027329547194, Tokens per sec: 17736.993662468103, Loss: 3.7980990409851074 +------------------------------------------------------------------ + eval ppl=43.11927795410156, eval loss=3.763970136642456 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/optimizer.pt +[rank0]:[2024-08-12 09:30:31,238] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006287928999881842, 'preprocessing_with_comm': 0.0006322830013232306, 'state_converting': 0.43446040300113964, : 0.4427187019991834}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090, took 3.38s +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 71.49614137515584, Tokens per sec: 17780.731170341365, Loss: 3.7420685291290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 71.36506727957713, Tokens per sec: 17748.133701274448, Loss: 3.786738157272339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 71.3725433466146, Tokens per sec: 17749.99296158763, Loss: 3.7936668395996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 71.40583828379113, Tokens per sec: 17758.273245192908, Loss: 3.7525177001953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 71.41024973535588, Tokens per sec: 17759.37035103436, Loss: 3.7418646812438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 71.21086796041827, Tokens per sec: 17709.785105282015, Loss: 3.740734100341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 71.42734748552805, Tokens per sec: 17763.62247560474, Loss: 3.7625956535339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 71.41040057832089, Tokens per sec: 17759.407864921937, Loss: 3.747201442718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 71.27488577711941, Tokens per sec: 17725.706014676307, Loss: 3.731395721435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 71.51421819365333, Tokens per sec: 17785.226784285514, Loss: 3.741957426071167 +------------------------------------------------------------------ + eval ppl=35.38701629638672, eval loss=3.566344976425171 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/model.pt +[rank0]:[2024-08-12 09:42:53,692] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006295597999269376, 'preprocessing_with_comm': 0.0006358050013659522, 'state_converting': 0.4317934979990241, : 0.44011598200086155}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100, took 3.47s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001 +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 71.16838422293144, Tokens per sec: 17699.219613203208, Loss: 3.697054147720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 71.5223218781371, Tokens per sec: 17787.242129904505, Loss: 3.709463596343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 71.3414671766776, Tokens per sec: 17742.264474247953, Loss: 3.690587043762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 71.36246700203175, Tokens per sec: 17747.48702531245, Loss: 3.7309420108795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 71.40115392809703, Tokens per sec: 17757.108269465534, Loss: 3.709296226501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 71.29268022791301, Tokens per sec: 17730.131405191085, Loss: 3.696531295776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 71.28450698397172, Tokens per sec: 17728.09876048447, Loss: 3.6493892669677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 71.54226845088455, Tokens per sec: 17792.20274233715, Loss: 3.6633403301239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 71.36042261961256, Tokens per sec: 17746.978597675577, Loss: 3.6733145713806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 71.34027472950476, Tokens per sec: 17741.967918626808, Loss: 3.701913595199585 +------------------------------------------------------------------ + eval ppl=43.97718048095703, eval loss=3.7836709022521973 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/optimizer.pt +[rank0]:[2024-08-12 09:55:16,503] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006302640002104454, 'preprocessing_with_comm': 0.0006228269994608127, 'state_converting': 0.42162483999709366, : 0.4298874300002353}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010 +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 71.45918020116558, Tokens per sec: 17771.53911205933, Loss: 3.6734166145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 71.53983774315653, Tokens per sec: 17791.598237536833, Loss: 3.657925605773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 71.50571794662496, Tokens per sec: 17783.112815553974, Loss: 3.6600286960601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 71.52207366011257, Tokens per sec: 17787.180399328823, Loss: 3.654576063156128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 71.39161642006407, Tokens per sec: 17754.736339133247, Loss: 3.637361764907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 71.60045598096382, Tokens per sec: 17806.673688739374, Loss: 3.643190383911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 71.49158595127103, Tokens per sec: 17779.59825930722, Loss: 3.6731386184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 71.1844716763009, Tokens per sec: 17703.220482034394, Loss: 3.5908491611480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 71.52849150640552, Tokens per sec: 17788.776485459246, Loss: 3.6359899044036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 71.30678138763591, Tokens per sec: 17733.638292771313, Loss: 3.621721029281616 +------------------------------------------------------------------ + eval ppl=41.1710205078125, eval loss=3.7177345752716064 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/model.pt +[rank0]:[2024-08-12 10:07:38,421] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006313498000963591, 'preprocessing_with_comm': 0.0006810109989601187, 'state_converting': 0.4493604600029357, : 0.4577443270027288}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120, took 3.59s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000020 +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 71.32716941626356, Tokens per sec: 17738.70869306354, Loss: 3.619030237197876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 71.41513473208659, Tokens per sec: 17760.585225179537, Loss: 3.6020267009735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 71.28863637955105, Tokens per sec: 17729.125720419368, Loss: 3.600602626800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 71.3744171929914, Tokens per sec: 17750.45897776752, Loss: 3.5987024307250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 71.34551239502912, Tokens per sec: 17743.270499729195, Loss: 3.572201728820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 71.33002910207054, Tokens per sec: 17739.419882556955, Loss: 3.578944444656372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 71.41591184926143, Tokens per sec: 17760.77849031676, Loss: 3.6057186126708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 71.31557373913327, Tokens per sec: 17735.824906416794, Loss: 3.575101375579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 71.26415474768352, Tokens per sec: 17723.03726156725, Loss: 3.5497827529907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 71.33844612907806, Tokens per sec: 17741.513154887398, Loss: 3.528101921081543 +------------------------------------------------------------------ + eval ppl=28.944433212280273, eval loss=3.365377902984619 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/model.pt +[rank0]:[2024-08-12 10:20:01,500] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063676649988337886, 'preprocessing_with_comm': 0.0006632449985772837, 'state_converting': 0.4440008149977075, : 0.4523857409985794}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130, took 3.55s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000030 +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 71.37382208434326, Tokens per sec: 17750.310977236495, Loss: 3.5375630855560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 71.213344260144, Tokens per sec: 17710.400948583116, Loss: 3.563973903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 71.30746186919535, Tokens per sec: 17733.807525116477, Loss: 3.5539896488189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 71.43612872532756, Tokens per sec: 17765.806325826787, Loss: 3.550021171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 71.15437102452655, Tokens per sec: 17695.734601160857, Loss: 3.532137155532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 71.48127197651807, Tokens per sec: 17777.033225602565, Loss: 3.5362603664398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 71.41523423999584, Tokens per sec: 17760.609972296654, Loss: 3.5196797847747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 71.4266165751872, Tokens per sec: 17763.440701874995, Loss: 3.524487257003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 71.47807419404708, Tokens per sec: 17776.237953167376, Loss: 3.5057082176208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 71.46249293425667, Tokens per sec: 17772.362972136238, Loss: 3.532160520553589 +------------------------------------------------------------------ + eval ppl=33.220821380615234, eval loss=3.5031769275665283 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/model.pt +[rank0]:[2024-08-12 10:32:24,230] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006335614998533856, 'preprocessing_with_comm': 0.0006517119982163422, 'state_converting': 0.42516524999882677, : 0.4334798119998595}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000040 +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 71.4515274735552, Tokens per sec: 17769.635917149186, Loss: 3.52252459526062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 71.22736124642348, Tokens per sec: 17713.88690265085, Loss: 3.501617431640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 71.47655296348755, Tokens per sec: 17775.859630769733, Loss: 3.5251970291137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 71.36349131998888, Tokens per sec: 17747.741768042317, Loss: 3.525052070617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 71.53114384049893, Tokens per sec: 17789.43610762321, Loss: 3.5120511054992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 71.36147456549779, Tokens per sec: 17747.24021133236, Loss: 3.514223337173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 71.19503511527601, Tokens per sec: 17705.847556237815, Loss: 3.510416269302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 71.46779120339151, Tokens per sec: 17773.680625052082, Loss: 3.5200393199920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 71.5273175369999, Tokens per sec: 17788.48452516598, Loss: 3.4799935817718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 71.41546795188785, Tokens per sec: 17760.668095270044, Loss: 3.476386070251465 +------------------------------------------------------------------ + eval ppl=32.805946350097656, eval loss=3.490609884262085 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/model.pt +[rank0]:[2024-08-12 10:44:46,913] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006387574001564644, 'preprocessing_with_comm': 0.0006399539997801185, 'state_converting': 0.4428889490009169, : 0.4513291109979036}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150, took 3.79s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000050 +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 71.29993715144631, Tokens per sec: 17731.93616561574, Loss: 3.5156750679016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 71.29632709605053, Tokens per sec: 17731.038362975374, Loss: 3.4623684883117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 71.14363776254974, Tokens per sec: 17693.065292830088, Loss: 3.496480941772461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 71.27608736079631, Tokens per sec: 17726.004842500115, Loss: 3.473090410232544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 71.32858595150698, Tokens per sec: 17739.060978262045, Loss: 3.466932773590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 71.42702526302138, Tokens per sec: 17763.542340486147, Loss: 3.4773786067962646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 71.45787042462139, Tokens per sec: 17771.21337721298, Loss: 3.497468948364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 71.214054539623, Tokens per sec: 17710.57759152117, Loss: 3.4262540340423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 71.27063097881083, Tokens per sec: 17724.647867712658, Loss: 3.4860448837280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 71.37587195059815, Tokens per sec: 17750.820768675756, Loss: 3.4299774169921875 +------------------------------------------------------------------ + eval ppl=33.61028289794922, eval loss=3.514832019805908 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/model.pt +[rank0]:[2024-08-12 10:57:10,245] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006288763997872593, 'preprocessing_with_comm': 0.0006427329972211737, 'state_converting': 0.43425100700187613, : 0.4425296369990974}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160, took 3.43s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000060 +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 71.50721202931149, Tokens per sec: 17783.484386411983, Loss: 3.4685349464416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 71.51391831005206, Tokens per sec: 17785.15220474046, Loss: 3.408677101135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 71.56277027033296, Tokens per sec: 17797.30144183483, Loss: 3.451547861099243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 71.22613111248026, Tokens per sec: 17713.580974519242, Loss: 3.3957221508026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 71.3442410063742, Tokens per sec: 17742.95431175805, Loss: 3.445971727371216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 71.36766846449294, Tokens per sec: 17748.78060289491, Loss: 3.426664352416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 71.2937809295398, Tokens per sec: 17730.405144155855, Loss: 3.4311137199401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 71.46350414494141, Tokens per sec: 17772.614455153307, Loss: 3.4251503944396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 71.35235417173524, Tokens per sec: 17744.972015223586, Loss: 3.4292571544647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 71.37747312244187, Tokens per sec: 17751.218972069157, Loss: 3.4002389907836914 +------------------------------------------------------------------ + eval ppl=32.70000076293945, eval loss=3.487375020980835 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/model.pt +[rank0]:[2024-08-12 11:09:32,638] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062912149987823796, 'preprocessing_with_comm': 0.0006369539987645112, 'state_converting': 0.4315558559974306, : 0.4398208019993035}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000070 +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 71.40288282701442, Tokens per sec: 17757.53823794047, Loss: 3.414121389389038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 71.26636508971909, Tokens per sec: 17723.58696252695, Loss: 3.4145240783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 71.36741236048165, Tokens per sec: 17748.71691111394, Loss: 3.406275749206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 71.46696730590247, Tokens per sec: 17773.475725885735, Loss: 3.3955025672912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 71.5447855179102, Tokens per sec: 17792.828724260933, Loss: 3.4123451709747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 71.36792803698987, Tokens per sec: 17748.845157270825, Loss: 3.3915462493896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 71.33162973699253, Tokens per sec: 17739.817952420617, Loss: 3.3935883045196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 71.403472001463, Tokens per sec: 17757.684762665875, Loss: 3.391031503677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 71.28576892617086, Tokens per sec: 17728.41259916952, Loss: 3.3704984188079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 71.52066257144905, Tokens per sec: 17786.829468667383, Loss: 3.3653695583343506 +------------------------------------------------------------------ + eval ppl=37.590213775634766, eval loss=3.626743793487549 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/model.pt +[rank0]:[2024-08-12 11:21:55,177] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006337372000416508, 'preprocessing_with_comm': 0.0006353630014928058, 'state_converting': 0.43409819199951016, : 0.44243083100082004}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180, took 3.59s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000080 +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 71.38096902175906, Tokens per sec: 17752.088384666284, Loss: 3.3919055461883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 71.5057435629951, Tokens per sec: 17783.11918621653, Loss: 3.364794969558716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 71.40375534874504, Tokens per sec: 17757.75522971141, Loss: 3.363936185836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 71.48048955546506, Tokens per sec: 17776.83864141749, Loss: 3.373929262161255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 71.66165678955541, Tokens per sec: 17821.894022369197, Loss: 3.3947222232818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 71.50631375145906, Tokens per sec: 17783.260989222945, Loss: 3.3672213554382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 71.38688884549227, Tokens per sec: 17753.560615088118, Loss: 3.3092775344848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 71.50282619245691, Tokens per sec: 17782.393650820268, Loss: 3.3439831733703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 71.36557067476623, Tokens per sec: 17748.25889312897, Loss: 3.3398244380950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 71.4828003005766, Tokens per sec: 17777.41331211777, Loss: 3.365333080291748 +------------------------------------------------------------------ + eval ppl=29.560213088989258, eval loss=3.3864293098449707 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/optimizer.pt +[rank0]:[2024-08-12 11:34:16,909] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006370680002873996, 'preprocessing_with_comm': 0.0006304650014499202, 'state_converting': 0.4337493769999128, : 0.4421176700016076}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190, took 3.53s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000090 +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 71.41171372099296, Tokens per sec: 17759.73443690738, Loss: 3.3088104724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 71.27333377470951, Tokens per sec: 17725.320039474078, Loss: 3.3106508255004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 71.416167076641, Tokens per sec: 17760.84196408382, Loss: 3.317422866821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 71.53096187240347, Tokens per sec: 17789.39085307206, Loss: 3.3046820163726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 71.39351863150749, Tokens per sec: 17755.2094095627, Loss: 3.331263303756714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 71.33746137383514, Tokens per sec: 17741.268251205798, Loss: 3.3423666954040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 71.48311910665812, Tokens per sec: 17777.492597588593, Loss: 3.294832468032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 71.43694771447508, Tokens per sec: 17766.010004313255, Loss: 3.3418898582458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 71.3147216336307, Tokens per sec: 17735.612992059192, Loss: 3.2919421195983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 71.55579597105115, Tokens per sec: 17795.566968641644, Loss: 3.3087046146392822 +------------------------------------------------------------------ + eval ppl=30.52009391784668, eval loss=3.4183852672576904 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/optimizer.pt +[rank0]:[2024-08-12 11:46:39,188] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006319565000012517, 'preprocessing_with_comm': 0.0006297519976214971, 'state_converting': 0.4267941010002687, : 0.4350674520028406}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200, took 3.43s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000100 +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 71.3386326641758, Tokens per sec: 17741.559545229073, Loss: 3.2816944122314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 71.57565014257655, Tokens per sec: 17800.50460135458, Loss: 3.3253989219665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 71.42144815100744, Tokens per sec: 17762.155340747147, Loss: 3.270653247833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 71.5592255663681, Tokens per sec: 17796.41989176701, Loss: 3.2683653831481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 71.47595116439993, Tokens per sec: 17775.709966360024, Loss: 3.240926504135132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 71.34630838910803, Tokens per sec: 17743.468459457625, Loss: 3.2857863903045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 71.26313097603303, Tokens per sec: 17722.7826547011, Loss: 3.245161771774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 71.52930527445393, Tokens per sec: 17788.978865484594, Loss: 3.284116268157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 71.35905959500144, Tokens per sec: 17746.639620302496, Loss: 3.27899169921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 71.49957224137265, Tokens per sec: 17781.584409533145, Loss: 3.2606873512268066 +------------------------------------------------------------------ + eval ppl=26.113039016723633, eval loss=3.262434720993042 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/model.pt +[rank0]:[2024-08-12 11:59:01,149] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006335731999570271, 'preprocessing_with_comm': 0.0006361420018947683, 'state_converting': 0.4242741699999897, : 0.4325900690018898}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000110 +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 71.51813170076979, Tokens per sec: 17786.2000538443, Loss: 3.234670639038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 71.23736199198179, Tokens per sec: 17716.374037828427, Loss: 3.277296304702759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 71.42501960509499, Tokens per sec: 17763.043543436084, Loss: 3.202859401702881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 71.47322401239687, Tokens per sec: 17775.03173735781, Loss: 3.2451024055480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 71.48940162974395, Tokens per sec: 17779.055029517254, Loss: 3.240194320678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 71.39350273807814, Tokens per sec: 17755.20545694667, Loss: 3.2103183269500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 71.43568908657191, Tokens per sec: 17765.69698987696, Loss: 3.1813576221466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 71.56058529982087, Tokens per sec: 17796.75805064555, Loss: 3.1950645446777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 71.39825089036816, Tokens per sec: 17756.386298566944, Loss: 3.234914541244507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 71.51325348847395, Tokens per sec: 17784.986866953983, Loss: 3.223423957824707 +------------------------------------------------------------------ + eval ppl=25.846744537353516, eval loss=3.2521846294403076 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/optimizer.pt +[rank0]:[2024-08-12 12:11:23,053] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006351658001221949, 'preprocessing_with_comm': 0.0006635559984715655, 'state_converting': 0.4322330830000283, : 0.4405764059993089}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000120 +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 71.27210596159544, Tokens per sec: 17725.01468852102, Loss: 3.1767425537109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 71.3484096864877, Tokens per sec: 17743.99104155923, Loss: 3.199399709701538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 71.47761977780083, Tokens per sec: 17776.124942129874, Loss: 3.2125253677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 71.50772131654462, Tokens per sec: 17783.611043588255, Loss: 3.1973462104797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 71.30741055271903, Tokens per sec: 17733.794762966627, Loss: 3.201457977294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 71.23300156857849, Tokens per sec: 17715.28962243437, Loss: 3.164062738418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 71.27216046010663, Tokens per sec: 17725.028242026954, Loss: 3.1623127460479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 71.46863399806008, Tokens per sec: 17773.89022385204, Loss: 3.207576036453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 71.47703909757165, Tokens per sec: 17775.980529874163, Loss: 3.1986446380615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 71.25335720187302, Tokens per sec: 17720.351966169994, Loss: 3.1415014266967773 +------------------------------------------------------------------ + eval ppl=20.29952049255371, eval loss=3.0105972290039062 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/optimizer.pt +[rank0]:[2024-08-12 12:23:45,915] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006412277001800248, 'preprocessing_with_comm': 0.0006377660029102117, 'state_converting': 0.4262171660011518, : 0.4346382330004417}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230, took 3.45s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000130 +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 71.42351074230045, Tokens per sec: 17762.668296839463, Loss: 3.1896684169769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 71.33893764514197, Tokens per sec: 17741.635392463166, Loss: 3.1621556282043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 71.27798538476517, Tokens per sec: 17726.47687152568, Loss: 3.1599769592285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 71.52802005003797, Tokens per sec: 17788.659236629184, Loss: 3.1660284996032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 71.58613647814562, Tokens per sec: 17803.11250032828, Loss: 3.1631691455841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 71.3498869903818, Tokens per sec: 17744.358439615866, Loss: 3.1469080448150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 71.42729528503136, Tokens per sec: 17763.609493603468, Loss: 3.1494030952453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 71.55875539881268, Tokens per sec: 17796.302963458056, Loss: 3.1508395671844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 71.42495447244472, Tokens per sec: 17763.027345273185, Loss: 3.117873191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 71.20308365995615, Tokens per sec: 17707.849188864657, Loss: 3.1231136322021484 +------------------------------------------------------------------ + eval ppl=25.343486785888672, eval loss=3.2325217723846436 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/optimizer.pt +[rank0]:[2024-08-12 12:36:08,352] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006238500998733798, 'preprocessing_with_comm': 0.0006290259989327751, 'state_converting': 0.43641388999822084, : 0.4446600800001761}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240, took 3.68s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000140 +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 71.43425235745003, Tokens per sec: 17765.339682562335, Loss: 3.1643662452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 71.5565158080905, Tokens per sec: 17795.745988496936, Loss: 3.1512746810913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 71.37634784256669, Tokens per sec: 17750.9391206175, Loss: 3.1474790573120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 71.559152531471, Tokens per sec: 17796.401728355024, Loss: 3.0810937881469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 71.51352693156457, Tokens per sec: 17785.05487087687, Loss: 3.0920653343200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 71.39391599942552, Tokens per sec: 17755.308232967578, Loss: 3.0577306747436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 71.49192888802858, Tokens per sec: 17779.683545955944, Loss: 3.108473300933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 71.33819470442887, Tokens per sec: 17741.450626840277, Loss: 3.144646167755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 71.37357661121996, Tokens per sec: 17750.249929303962, Loss: 3.0891730785369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 71.40539618004262, Tokens per sec: 17758.16329621174, Loss: 3.1118881702423096 +------------------------------------------------------------------ + eval ppl=17.121583938598633, eval loss=2.8403398990631104 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/model.pt +[rank0]:[2024-08-12 12:48:30,356] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006393652998667676, 'preprocessing_with_comm': 0.000621442999545252, 'state_converting': 0.4290278100015712, : 0.4373637669996242}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000150 +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 71.25445413928524, Tokens per sec: 17720.62476899351, Loss: 3.0740408897399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 71.46168281009365, Tokens per sec: 17772.161498326877, Loss: 3.062232494354248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 71.40775373660625, Tokens per sec: 17758.749608684982, Loss: 3.0798683166503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 71.28824619048757, Tokens per sec: 17729.028682359552, Loss: 3.0973424911499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 71.37690941500628, Tokens per sec: 17751.078780861943, Loss: 3.109220504760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 71.39812182615671, Tokens per sec: 17756.354200945963, Loss: 3.0730795860290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 71.34026349962839, Tokens per sec: 17741.96512581297, Loss: 3.0864992141723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 71.59291667250137, Tokens per sec: 17804.798700601517, Loss: 3.0595834255218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 71.2634594967159, Tokens per sec: 17722.864356144473, Loss: 3.0585439205169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 71.23091724156123, Tokens per sec: 17714.771260776633, Loss: 3.0599024295806885 +------------------------------------------------------------------ + eval ppl=21.251737594604492, eval loss=3.056438684463501 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/model.pt +[rank0]:[2024-08-12 13:00:53,290] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063202339988492895, 'preprocessing_with_comm': 0.0006362480016832706, 'state_converting': 0.43524141699890606, : 0.4436021690016787}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260, took 3.59s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000160 +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 71.35915929830881, Tokens per sec: 17746.664416014137, Loss: 3.0979971885681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 71.25360779002725, Tokens per sec: 17720.414286185023, Loss: 3.0485777854919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 71.4542850134598, Tokens per sec: 17770.32170346985, Loss: 3.0791895389556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 71.44529485315512, Tokens per sec: 17768.085895767766, Loss: 3.070594310760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 71.37282277070581, Tokens per sec: 17750.062452955313, Loss: 3.0798332691192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 71.46363434563058, Tokens per sec: 17772.64683541059, Loss: 3.0304036140441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 71.41981075703443, Tokens per sec: 17761.748129092168, Loss: 3.045954704284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 71.37039547192711, Tokens per sec: 17749.458795943567, Loss: 3.0669164657592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 71.2754368671109, Tokens per sec: 17725.843067988568, Loss: 3.0707294940948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 71.35549047182153, Tokens per sec: 17745.751997298576, Loss: 3.0363452434539795 +------------------------------------------------------------------ + eval ppl=22.825763702392578, eval loss=3.12788987159729 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/model.pt +[rank0]:[2024-08-12 13:13:15,955] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006290364999586018, 'preprocessing_with_comm': 0.0006296579995250795, 'state_converting': 0.4315383169996494, : 0.43984982400070294}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270, took 3.54s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000170 +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 71.2982324486606, Tokens per sec: 17731.51221459719, Loss: 3.061197280883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 71.28714596919268, Tokens per sec: 17728.755062850916, Loss: 3.0184879302978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 71.46673084242751, Tokens per sec: 17773.416918607483, Loss: 3.040440797805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 71.37464865290227, Tokens per sec: 17750.51654068452, Loss: 3.035027027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 71.53711506528037, Tokens per sec: 17790.921121227504, Loss: 2.9963977336883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 71.49615226048077, Tokens per sec: 17780.733877466988, Loss: 3.011275291442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 71.3327212925124, Tokens per sec: 17740.08941679454, Loss: 3.054415464401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 71.40736089610373, Tokens per sec: 17758.651911225592, Loss: 2.9690439701080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 71.22773368210329, Tokens per sec: 17713.979525533337, Loss: 3.0029799938201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 71.20618388801014, Tokens per sec: 17708.620200006444, Loss: 3.042771339416504 +------------------------------------------------------------------ + eval ppl=21.929391860961914, eval loss=3.0878279209136963 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/model.pt +[rank0]:[2024-08-12 13:25:38,778] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006286948999331798, 'preprocessing_with_comm': 0.0006339969986584038, 'state_converting': 0.42057467700215057, : 0.4288464800010843}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280, took 3.48s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000180 +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 71.32567483308725, Tokens per sec: 17738.336997736234, Loss: 3.0310821533203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 71.4206423147635, Tokens per sec: 17761.954933321722, Loss: 3.0274338722229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 71.52417048123294, Tokens per sec: 17787.70186820724, Loss: 3.0005078315734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 71.48978574456197, Tokens per sec: 17779.15055694274, Loss: 2.98514986038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 71.43084976363374, Tokens per sec: 17764.493474574523, Loss: 2.9824182987213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 71.42585295155111, Tokens per sec: 17763.25079251307, Loss: 2.976505756378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 71.43467009331393, Tokens per sec: 17765.443571373024, Loss: 3.016970157623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 71.36660381671614, Tokens per sec: 17748.515830341505, Loss: 3.0389747619628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 71.30744432225592, Tokens per sec: 17733.803161280794, Loss: 2.971992254257202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 71.5065382935848, Tokens per sec: 17783.31683172154, Loss: 2.980898380279541 +------------------------------------------------------------------ + eval ppl=15.854172706604004, eval loss=2.763432741165161 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/optimizer.pt +[rank0]:[2024-08-12 13:38:01,040] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063441780002904125, 'preprocessing_with_comm': 0.0006324440000753384, 'state_converting': 0.4217493600008311, : 0.43009708799945656}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290, took 3.59s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000190 +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 71.35298035931886, Tokens per sec: 17745.127744929727, Loss: 3.0088274478912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 71.39965302065086, Tokens per sec: 17756.735001324083, Loss: 2.9887821674346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 71.33945881370599, Tokens per sec: 17741.765004466735, Loss: 2.986785411834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 71.26462279468726, Tokens per sec: 17723.153662505658, Loss: 2.97701096534729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 71.42542636434395, Tokens per sec: 17763.144702417783, Loss: 2.9654152393341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 71.290434281921, Tokens per sec: 17729.57284970629, Loss: 2.963472366333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 71.05802680260255, Tokens per sec: 17671.77427719236, Loss: 2.979712963104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 71.18157350386764, Tokens per sec: 17702.499721110376, Loss: 3.006873607635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 71.0761289365242, Tokens per sec: 17676.276186955314, Loss: 3.001466989517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 71.43594800409797, Tokens per sec: 17765.76138136491, Loss: 3.023026704788208 +------------------------------------------------------------------ + eval ppl=15.913963317871094, eval loss=2.7671968936920166 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/model.pt +[rank0]:[2024-08-12 13:50:24,803] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006394543997885194, 'preprocessing_with_comm': 0.0006675059994449839, 'state_converting': 0.434766465001303, : 0.4432218119982281}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300, took 3.64s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000200 +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 71.01783487914906, Tokens per sec: 17661.77874774979, Loss: 2.9557931423187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 71.38061552239947, Tokens per sec: 17752.0004711515, Loss: 2.9856410026550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 71.34529265096745, Tokens per sec: 17743.215850485027, Loss: 2.982142448425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 71.20917475892804, Tokens per sec: 17709.364014577877, Loss: 2.927558422088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 71.41958254642405, Tokens per sec: 17761.69137425988, Loss: 2.9518802165985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 71.13380371280869, Tokens per sec: 17690.619614064795, Loss: 2.9662413597106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 71.35034574234477, Tokens per sec: 17744.472528924332, Loss: 2.968040704727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 71.21294493058679, Tokens per sec: 17710.301637328434, Loss: 2.9542624950408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 71.31131590015546, Tokens per sec: 17734.76600325398, Loss: 2.956205129623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 71.38543159526738, Tokens per sec: 17753.19820427827, Loss: 2.9945549964904785 +------------------------------------------------------------------ + eval ppl=19.209747314453125, eval loss=2.9554178714752197 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/optimizer.pt +[rank0]:[2024-08-12 14:02:48,556] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006322492998151574, 'preprocessing_with_comm': 0.0006584670009033289, 'state_converting': 0.44362447099774727, : 0.45202806900124415}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310, took 3.63s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000210 +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 71.31019414832416, Tokens per sec: 17734.487029209104, Loss: 2.985847234725952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 71.3163675824342, Tokens per sec: 17736.022331257544, Loss: 2.952986717224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 71.53444585453597, Tokens per sec: 17790.257301925227, Loss: 2.910379409790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 71.46541752124783, Tokens per sec: 17773.090302228095, Loss: 2.933797597885132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 71.3837033557538, Tokens per sec: 17752.768399793746, Loss: 2.956239700317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 71.4139856808258, Tokens per sec: 17760.2994619037, Loss: 2.9545300006866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 71.43745711556708, Tokens per sec: 17766.136689805655, Loss: 2.930745840072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 71.26064699354079, Tokens per sec: 17722.16490073457, Loss: 2.923931837081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 71.23413413217347, Tokens per sec: 17715.57128531055, Loss: 2.9279088973999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 71.41551446960914, Tokens per sec: 17760.67966399363, Loss: 2.9169435501098633 +------------------------------------------------------------------ + eval ppl=18.098722457885742, eval loss=2.895841360092163 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/optimizer.pt +[rank0]:[2024-08-12 14:15:11,358] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00629366299835965, 'preprocessing_with_comm': 0.0006361409978126176, 'state_converting': 0.4433248850036762, : 0.4516217060008785}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320, took 3.43s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000220 +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 71.47443991834831, Tokens per sec: 17775.334127059337, Loss: 2.900538206100464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 71.60357087704567, Tokens per sec: 17807.448347745998, Loss: 2.915191888809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 71.39700651917603, Tokens per sec: 17756.07682970306, Loss: 2.9425909519195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 71.60478478144412, Tokens per sec: 17807.750239671353, Loss: 2.9192731380462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 71.48657671826986, Tokens per sec: 17778.35248822573, Loss: 2.9186041355133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 71.56685989716476, Tokens per sec: 17798.318511482008, Loss: 2.92071533203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 71.311020431269, Tokens per sec: 17734.692521626323, Loss: 2.940591812133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 71.44478323140918, Tokens per sec: 17767.958658009884, Loss: 2.9271440505981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 71.39684330541255, Tokens per sec: 17756.03623926005, Loss: 2.9390602111816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 71.46492580796502, Tokens per sec: 17772.96801560498, Loss: 2.9320552349090576 +------------------------------------------------------------------ + eval ppl=17.46542739868164, eval loss=2.8602232933044434 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/model.pt +[rank0]:[2024-08-12 14:27:33,123] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006354673001624178, 'preprocessing_with_comm': 0.0006362990025081672, 'state_converting': 0.43597314300131984, : 0.44429837599454913}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330, took 3.67s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000230 +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 71.47292419149049, Tokens per sec: 17774.957173404666, Loss: 2.901871919631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 71.40815924103369, Tokens per sec: 17758.850455598873, Loss: 2.9227163791656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 71.49476106800725, Tokens per sec: 17780.387894888037, Loss: 2.8898911476135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 71.3655757130522, Tokens per sec: 17748.260146125376, Loss: 2.884542465209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 71.32157001418217, Tokens per sec: 17737.31614989675, Loss: 2.85263729095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 71.63894103261673, Tokens per sec: 17816.244727740297, Loss: 2.8923392295837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 71.25983051984568, Tokens per sec: 17721.96184782848, Loss: 2.886599540710449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 71.39078667892105, Tokens per sec: 17754.529986679518, Loss: 2.9395768642425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 71.4053896253749, Tokens per sec: 17758.161666098807, Loss: 2.8970248699188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 71.2134900876783, Tokens per sec: 17710.437215158272, Loss: 2.898270606994629 +------------------------------------------------------------------ + eval ppl=23.15216827392578, eval loss=3.1420884132385254 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/model.pt +[rank0]:[2024-08-12 14:39:55,560] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006322821005596779, 'preprocessing_with_comm': 0.000650487003440503, 'state_converting': 0.43658958499872824, : 0.4449079449987039}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340, took 3.45s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000240 +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 71.37689915064672, Tokens per sec: 17751.076228167283, Loss: 2.8734471797943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 71.46094425519507, Tokens per sec: 17771.97782343403, Loss: 2.8889808654785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 71.39333898535197, Tokens per sec: 17755.16473246635, Loss: 2.8778295516967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 71.5715300316787, Tokens per sec: 17799.479950473324, Loss: 2.906144380569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 71.46886295051829, Tokens per sec: 17773.947163178156, Loss: 2.923163652420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 71.50295498376401, Tokens per sec: 17782.425680571312, Loss: 2.8760480880737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 71.6326605189235, Tokens per sec: 17814.682795537483, Loss: 2.8934314250946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 71.53621575447208, Tokens per sec: 17790.69746714753, Loss: 2.8861119747161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 71.36951062757564, Tokens per sec: 17749.238739598728, Loss: 2.9065306186676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 71.51999282632787, Tokens per sec: 17786.662906420475, Loss: 2.9185264110565186 +------------------------------------------------------------------ + eval ppl=18.83494758605957, eval loss=2.93571400642395 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/model.pt +[rank0]:[2024-08-12 14:52:17,147] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006329178002488334, 'preprocessing_with_comm': 0.0006597170067834668, 'state_converting': 0.4325906700032647, : 0.4409352759976173}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000250 +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 71.47451889136761, Tokens per sec: 17775.353767252487, Loss: 2.8725426197052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 71.37106653058038, Tokens per sec: 17749.6256848593, Loss: 2.8695647716522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 71.33115878445615, Tokens per sec: 17739.70082889084, Loss: 2.8767354488372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 71.20691849536755, Tokens per sec: 17708.80289316563, Loss: 2.8822784423828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 71.33721102186924, Tokens per sec: 17741.205989929622, Loss: 2.840752363204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 71.18580756472086, Tokens per sec: 17703.55271077306, Loss: 2.9097602367401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 71.21781982188729, Tokens per sec: 17711.513998303886, Loss: 2.8679380416870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 71.50427177310523, Tokens per sec: 17782.75315946505, Loss: 2.830575704574585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 70.91975798115101, Tokens per sec: 17637.38751594647, Loss: 2.8780882358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 71.11412926797203, Tokens per sec: 17685.726678476418, Loss: 2.853468656539917 +------------------------------------------------------------------ + eval ppl=20.795949935913086, eval loss=3.0347583293914795 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/model.pt +[rank0]:[2024-08-12 15:04:41,170] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006320107000647113, 'preprocessing_with_comm': 0.0006328919989755377, 'state_converting': 0.44044044699694496, : 0.44878217100631446}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360, took 3.72s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000260 +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 71.12345365218322, Tokens per sec: 17688.04560598494, Loss: 2.863391160964966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 71.44311154494935, Tokens per sec: 17767.54291798572, Loss: 2.8463735580444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 71.00671855077479, Tokens per sec: 17659.01417273046, Loss: 2.8854238986968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 71.36901398663483, Tokens per sec: 17749.11522749182, Loss: 2.8444724082946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 71.30270761850308, Tokens per sec: 17732.625166854203, Loss: 2.8645358085632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 71.20106445612183, Tokens per sec: 17707.347023015343, Loss: 2.8320021629333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 71.15130178829332, Tokens per sec: 17694.971297529202, Loss: 2.837764024734497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 71.2587337068195, Tokens per sec: 17721.68907593915, Loss: 2.8612513542175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 71.19483662417699, Tokens per sec: 17705.798192498685, Loss: 2.869105815887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 71.5070397592574, Tokens per sec: 17783.441543715, Loss: 2.874285936355591 +------------------------------------------------------------------ + eval ppl=16.56218719482422, eval loss=2.807122230529785 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/model.pt +[rank0]:[2024-08-12 15:17:05,125] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062655939982505515, 'preprocessing_with_comm': 0.0006220459981705062, 'state_converting': 0.4233646360007697, : 0.43156674400233896}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000270 +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 71.39588976956325, Tokens per sec: 17755.799099684802, Loss: 2.802107334136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 71.21774440826164, Tokens per sec: 17711.495243314057, Loss: 2.8049352169036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 71.3436624468252, Tokens per sec: 17742.810426904838, Loss: 2.8503150939941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 71.23155877059519, Tokens per sec: 17714.9308058244, Loss: 2.861161470413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 71.12686445729658, Tokens per sec: 17688.893856081082, Loss: 2.8411827087402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 71.53285982634013, Tokens per sec: 17789.862864680974, Loss: 2.828770875930786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 71.26065573719949, Tokens per sec: 17722.167075238558, Loss: 2.8110740184783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 71.19467996104281, Tokens per sec: 17705.759231164273, Loss: 2.839715003967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 71.3301578903053, Tokens per sec: 17739.451911543918, Loss: 2.8373219966888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 71.00940498125725, Tokens per sec: 17659.68227449508, Loss: 2.857083797454834 +------------------------------------------------------------------ + eval ppl=17.703458786010742, eval loss=2.8737599849700928 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/model.pt +[rank0]:[2024-08-12 15:29:29,056] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006328980001853779, 'preprocessing_with_comm': 0.0006294330014497973, 'state_converting': 0.4305196019995492, : 0.43882325499725994}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380, took 3.71s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000280 +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 71.24680929389378, Tokens per sec: 17718.72353435161, Loss: 2.84857177734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 71.52927370918223, Tokens per sec: 17788.971015360105, Loss: 2.8476223945617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 71.39797525984655, Tokens per sec: 17756.317750640956, Loss: 2.8035929203033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 71.27156070261208, Tokens per sec: 17724.879085351185, Loss: 2.869858741760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 71.29117362606729, Tokens per sec: 17729.756720881076, Loss: 2.8369603157043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 71.3261227779112, Tokens per sec: 17738.44839936352, Loss: 2.8439130783081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 71.28933707612069, Tokens per sec: 17729.299980136006, Loss: 2.839444875717163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 71.45601008223082, Tokens per sec: 17770.75071940662, Loss: 2.8406827449798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 71.38781847858087, Tokens per sec: 17753.791810166866, Loss: 2.8539016246795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 71.19350263898625, Tokens per sec: 17705.46643708356, Loss: 2.793947458267212 +------------------------------------------------------------------ + eval ppl=16.983978271484375, eval loss=2.832270383834839 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/model.pt +[rank0]:[2024-08-12 15:41:52,354] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006360054998367559, 'preprocessing_with_comm': 0.0006571160047315061, 'state_converting': 0.4325812750030309, : 0.44093896399863297}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390, took 3.46s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000290 +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 71.20316907330387, Tokens per sec: 17707.870430735125, Loss: 2.8286139965057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 71.32564558966246, Tokens per sec: 17738.3297250434, Loss: 2.807579517364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 71.2375841011581, Tokens per sec: 17716.429275264723, Loss: 2.8533337116241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 71.12616636362546, Tokens per sec: 17688.72024369223, Loss: 2.8403143882751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 71.35795115572714, Tokens per sec: 17746.363957023666, Loss: 2.826833724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 71.39233842319899, Tokens per sec: 17754.915897685634, Loss: 2.828218698501587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 71.37385024343425, Tokens per sec: 17750.317980260956, Loss: 2.8419291973114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 71.31254906319394, Tokens per sec: 17735.07268470636, Loss: 2.824533700942993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 71.3473071873252, Tokens per sec: 17743.71685555637, Loss: 2.8498551845550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 71.34739277998398, Tokens per sec: 17743.738142020593, Loss: 2.8143956661224365 +------------------------------------------------------------------ + eval ppl=17.0340518951416, eval loss=2.835214376449585 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/model.pt +[rank0]:[2024-08-12 15:54:15,795] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006307097995886579, 'preprocessing_with_comm': 0.000619343998550903, 'state_converting': 0.4243439150013728, : 0.43262095899990527}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000300 +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 71.32200269909785, Tokens per sec: 17737.423756461514, Loss: 2.8065075874328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 71.3866422452314, Tokens per sec: 17753.499286842132, Loss: 2.7675604820251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 71.32098880210809, Tokens per sec: 17737.17160537388, Loss: 2.814645528793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 71.4152097855331, Tokens per sec: 17760.603890594622, Loss: 2.8047914505004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 71.23578763711473, Tokens per sec: 17715.982503682393, Loss: 2.794010877609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 71.24932515189495, Tokens per sec: 17719.349215597078, Loss: 2.802422285079956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 71.27526543542065, Tokens per sec: 17725.80043378846, Loss: 2.798942804336548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 71.25594057956354, Tokens per sec: 17720.994439222995, Loss: 2.7944555282592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 71.37321548252221, Tokens per sec: 17750.16011841111, Loss: 2.790208578109741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 71.58457255652628, Tokens per sec: 17802.723560878538, Loss: 2.797511339187622 +------------------------------------------------------------------ + eval ppl=19.53003692626953, eval loss=2.9719536304473877 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/model.pt +[rank0]:[2024-08-12 16:06:38,823] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006326778995571658, 'preprocessing_with_comm': 0.0006618929983233102, 'state_converting': 0.44234845000028145, : 0.45066935899376404}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000310 +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 71.12254194840833, Tokens per sec: 17687.81886983644, Loss: 2.8055005073547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 71.0909337579534, Tokens per sec: 17679.95807166677, Loss: 2.793623447418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 71.39678501606313, Tokens per sec: 17756.021742991692, Loss: 2.803358554840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 71.31161552528985, Tokens per sec: 17734.840518519613, Loss: 2.799896240234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 71.29892473319818, Tokens per sec: 17731.684382283718, Loss: 2.781027317047119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 71.46055073333078, Tokens per sec: 17771.879956523393, Loss: 2.7848587036132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 71.19446831878099, Tokens per sec: 17705.70659679703, Loss: 2.7892608642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 71.26885346360011, Tokens per sec: 17724.20580860981, Loss: 2.7824811935424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 70.89087200620905, Tokens per sec: 17630.203719098732, Loss: 2.8029961585998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 71.09091144917993, Tokens per sec: 17679.95252358689, Loss: 2.7954261302948 +------------------------------------------------------------------ + eval ppl=18.060728073120117, eval loss=2.893739938735962 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/model.pt +[rank0]:[2024-08-12 16:19:03,376] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063209540021489374, 'preprocessing_with_comm': 0.0006383889995049685, 'state_converting': 0.44310279300407274, : 0.45139676799590234}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420, took 3.68s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000320 +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 71.20949488902525, Tokens per sec: 17709.443629324753, Loss: 2.799957752227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 71.28527273476679, Tokens per sec: 17728.289198860148, Loss: 2.779658794403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 71.37380910546293, Tokens per sec: 17750.30774945416, Loss: 2.7858381271362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 71.43763047938629, Tokens per sec: 17766.179804516527, Loss: 2.786844253540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 71.40244448228664, Tokens per sec: 17757.42922380887, Loss: 2.795949935913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 71.36738746823207, Tokens per sec: 17748.710720536525, Loss: 2.7775838375091553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 71.36969661305821, Tokens per sec: 17749.28499325387, Loss: 2.8214287757873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 71.15918199249504, Tokens per sec: 17696.931064724784, Loss: 2.8058016300201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 71.2687517579109, Tokens per sec: 17724.180514915868, Loss: 2.7877886295318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 71.42174296231525, Tokens per sec: 17762.2286588383, Loss: 2.7756757736206055 +------------------------------------------------------------------ + eval ppl=17.71763801574707, eval loss=2.874560594558716 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430, took 3.41s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000330 +[rank0]:[2024-08-12 16:31:26,527] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006251183003769256, 'preprocessing_with_comm': 0.0006316329963738099, 'state_converting': 0.43930369000008795, : 0.44753279400174506}) +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 71.31762348917879, Tokens per sec: 17736.334668955365, Loss: 2.769212007522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 71.19176239433972, Tokens per sec: 17705.03364698279, Loss: 2.7977280616760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 71.282662572058, Tokens per sec: 17727.640064507676, Loss: 2.789738416671753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 71.31599297761572, Tokens per sec: 17735.929168921164, Loss: 2.80627703666687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 71.30118030918372, Tokens per sec: 17732.245332699527, Loss: 2.7569663524627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 71.40651960590802, Tokens per sec: 17758.44268658048, Loss: 2.7792696952819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 71.46224660961879, Tokens per sec: 17772.301712436303, Loss: 2.788717031478882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 71.45108938939066, Tokens per sec: 17769.526967818358, Loss: 2.7840495109558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 71.26156121360476, Tokens per sec: 17722.39226267152, Loss: 2.750985622406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 71.28248923763597, Tokens per sec: 17727.59695710773, Loss: 2.7709739208221436 +------------------------------------------------------------------ + eval ppl=16.580562591552734, eval loss=2.8082311153411865 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/optimizer.pt +[rank0]:[2024-08-12 16:43:49,659] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006317738996585831, 'preprocessing_with_comm': 0.0006146760060801171, 'state_converting': 0.436813759006327, : 0.445116554001288}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000340 +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 71.30239923138163, Tokens per sec: 17732.548472526403, Loss: 2.7841391563415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 71.34345473385827, Tokens per sec: 17742.75876973349, Loss: 2.757150411605835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 71.33472102493002, Tokens per sec: 17740.586740200342, Loss: 2.754398822784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 71.39598983689613, Tokens per sec: 17755.823985927764, Loss: 2.727827787399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 71.455162350309, Tokens per sec: 17770.53989273659, Loss: 2.7377073764801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 71.25280236890777, Tokens per sec: 17720.213981998964, Loss: 2.7632646560668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 71.31384249499224, Tokens per sec: 17735.394354696517, Loss: 2.7628839015960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 71.55734677287755, Tokens per sec: 17795.952645264788, Loss: 2.768831729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 71.29207824400436, Tokens per sec: 17729.981694817314, Loss: 2.749878406524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 71.20825403564847, Tokens per sec: 17709.13503532387, Loss: 2.7419042587280273 +------------------------------------------------------------------ + eval ppl=16.607484817504883, eval loss=2.8098535537719727 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/optimizer.pt +[rank0]:[2024-08-12 16:56:12,695] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062873269998817705, 'preprocessing_with_comm': 0.0006209850034792908, 'state_converting': 0.42129584700160194, : 0.42954683500283863}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450, took 3.47s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000350 +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 71.36898489413413, Tokens per sec: 17749.107992333054, Loss: 2.7636568546295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 71.33527449818891, Tokens per sec: 17740.72438621923, Loss: 2.7610669136047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 71.24431354807122, Tokens per sec: 17718.102854903926, Loss: 2.7439451217651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 71.22695483255056, Tokens per sec: 17713.78582956244, Loss: 2.6883370876312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 71.24439280192217, Tokens per sec: 17718.122564938494, Loss: 2.759510040283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 71.53401776148253, Tokens per sec: 17790.150837333535, Loss: 2.7845752239227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 71.41221548758944, Tokens per sec: 17759.8592237391, Loss: 2.739708185195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 71.24417636757329, Tokens per sec: 17718.068738803275, Loss: 2.75081729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 71.32267854607726, Tokens per sec: 17737.591836209904, Loss: 2.7446675300598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 71.07242667951647, Tokens per sec: 17675.35545423727, Loss: 2.7445075511932373 +------------------------------------------------------------------ + eval ppl=17.526418685913086, eval loss=2.8637094497680664 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/optimizer.pt +[rank0]:[2024-08-12 17:08:36,219] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006308957003057003, 'preprocessing_with_comm': 0.000649535002594348, 'state_converting': 0.44313740100187715, : 0.45144735900248634}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000360 +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 71.30396539550723, Tokens per sec: 17732.937969676183, Loss: 2.734239339828491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 71.50343172713076, Tokens per sec: 17782.544244251512, Loss: 2.720053195953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 71.23818080423895, Tokens per sec: 17716.57767232315, Loss: 2.7366397380828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 71.35666156517117, Tokens per sec: 17746.043242331172, Loss: 2.75830078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 71.43877182468214, Tokens per sec: 17766.463651357593, Loss: 2.717864751815796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 71.12808141235135, Tokens per sec: 17689.196506689335, Loss: 2.7447001934051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 71.53109209965946, Tokens per sec: 17789.423239936372, Loss: 2.7466063499450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 71.5114180149229, Tokens per sec: 17784.53039390308, Loss: 2.7413744926452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 71.38908738274341, Tokens per sec: 17754.10738025724, Loss: 2.742523670196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 71.37992186321655, Tokens per sec: 17751.827961597588, Loss: 2.698054075241089 +------------------------------------------------------------------ + eval ppl=12.811196327209473, eval loss=2.5503194332122803 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/optimizer.pt +[rank0]:[2024-08-12 17:20:58,841] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006247498000448104, 'preprocessing_with_comm': 0.0006198960036272183, 'state_converting': 0.4352126500016311, : 0.4434730830034823}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470, took 3.38s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000370 +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 71.45127397575637, Tokens per sec: 17769.572873520163, Loss: 2.7473018169403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 71.36804525585586, Tokens per sec: 17748.874309013903, Loss: 2.7357137203216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 71.19147897439558, Tokens per sec: 17704.96316186656, Loss: 2.7331082820892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 71.38504125453426, Tokens per sec: 17753.101128498973, Loss: 2.685518264770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 71.36863234052522, Tokens per sec: 17749.020314021716, Loss: 2.6849067211151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 71.4163338510918, Tokens per sec: 17760.883440051875, Loss: 2.7769601345062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 71.42994938859823, Tokens per sec: 17764.26955582659, Loss: 2.7634339332580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 71.33716709185956, Tokens per sec: 17741.195064756914, Loss: 2.7304208278656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 71.33135652396476, Tokens per sec: 17739.750005713206, Loss: 2.7160305976867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 71.2945153804078, Tokens per sec: 17730.587798396915, Loss: 2.728977918624878 +------------------------------------------------------------------ + eval ppl=17.977853775024414, eval loss=2.8891406059265137 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/optimizer.pt +[rank0]:[2024-08-12 17:33:21,744] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063107769965427, 'preprocessing_with_comm': 0.0006441200021072291, 'state_converting': 0.4336591999963275, : 0.4419551379978657}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000380 +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 71.33547561878663, Tokens per sec: 17740.774403901472, Loss: 2.7079522609710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 71.42484490121683, Tokens per sec: 17763.000095459283, Loss: 2.7196710109710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 71.41872807555676, Tokens per sec: 17761.47887164796, Loss: 2.731116771697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 71.21773464374586, Tokens per sec: 17711.492814928042, Loss: 2.7009599208831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 71.43119734650315, Tokens per sec: 17764.579916687922, Loss: 2.748000144958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 71.43251114283184, Tokens per sec: 17764.906651234483, Loss: 2.7189767360687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 71.45555156475406, Tokens per sec: 17770.6366884137, Loss: 2.6864852905273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 71.63382472826358, Tokens per sec: 17814.972328551485, Loss: 2.699620008468628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 71.32412344452342, Tokens per sec: 17737.951175194434, Loss: 2.6900835037231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 71.33370083773289, Tokens per sec: 17740.333024769738, Loss: 2.701692581176758 +------------------------------------------------------------------ + eval ppl=12.66712760925293, eval loss=2.5390102863311768 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/optimizer.pt +[rank0]:[2024-08-12 17:45:44,180] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006371210998622701, 'preprocessing_with_comm': 0.0006507479993160814, 'state_converting': 0.4404348489988479, : 0.4488238070043735}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000390 +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 71.48124051177452, Tokens per sec: 17777.02540047892, Loss: 2.6722497940063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 71.30334059739532, Tokens per sec: 17732.782585524677, Loss: 2.6567323207855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 71.41007750364722, Tokens per sec: 17759.327517873688, Loss: 2.7200515270233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 71.344114010494, Tokens per sec: 17742.922728520654, Loss: 2.676629066467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 71.43246640814752, Tokens per sec: 17764.895525943237, Loss: 2.7151832580566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 71.42557352046182, Tokens per sec: 17763.181299405, Loss: 2.6910808086395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 71.25171425462925, Tokens per sec: 17719.94337344448, Loss: 2.6762008666992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 71.36903239965744, Tokens per sec: 17749.11980671804, Loss: 2.7345473766326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 71.42966190295674, Tokens per sec: 17764.198059591847, Loss: 2.691314458847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 71.3477554261073, Tokens per sec: 17743.82833028957, Loss: 2.685879707336426 +------------------------------------------------------------------ + eval ppl=12.106124877929688, eval loss=2.493711471557617 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/optimizer.pt +[rank0]:[2024-08-12 17:58:06,824] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063113889991655014, 'preprocessing_with_comm': 0.0006313880003290251, 'state_converting': 0.43062668399943504, : 0.4389281390031101}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000400 +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 71.49585291623428, Tokens per sec: 17780.659432056764, Loss: 2.7004311084747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 71.34802113622206, Tokens per sec: 17743.894411060206, Loss: 2.7072315216064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 71.4910686355156, Tokens per sec: 17779.469605477785, Loss: 2.7046525478363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 71.16073425489444, Tokens per sec: 17697.3171045851, Loss: 2.703293800354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 71.56182080223444, Tokens per sec: 17797.065313888757, Loss: 2.667715311050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 71.4512660955971, Tokens per sec: 17769.570913764146, Loss: 2.7108442783355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 71.16781304013601, Tokens per sec: 17699.077562911552, Loss: 2.6552345752716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 71.49797225364163, Tokens per sec: 17781.18650062262, Loss: 2.7356133460998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 71.36311982814904, Tokens per sec: 17747.649379888084, Loss: 2.658285617828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 71.45899853992965, Tokens per sec: 17771.4939338226, Loss: 2.7086610794067383 +------------------------------------------------------------------ + eval ppl=13.58387565612793, eval loss=2.6088833808898926 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/model.pt +[rank0]:[2024-08-12 18:10:29,280] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006339090999972541, 'preprocessing_with_comm': 0.0006341330008581281, 'state_converting': 0.43845089599926723, : 0.4467501179969986}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510, took 3.50s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000410 +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 71.35283289179803, Tokens per sec: 17745.091070498158, Loss: 2.7209694385528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 71.50375570040927, Tokens per sec: 17782.624814778264, Loss: 2.689950942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 71.31192184769266, Tokens per sec: 17734.916699362257, Loss: 2.7041847705841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 71.41391080505272, Tokens per sec: 17760.280840675106, Loss: 2.6702167987823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 71.4175918367122, Tokens per sec: 17761.196294755675, Loss: 2.6443042755126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 71.52609543607375, Tokens per sec: 17788.180594805355, Loss: 2.686383008956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 71.18762263722779, Tokens per sec: 17704.00411018679, Loss: 2.6859962940216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 71.46404651078944, Tokens per sec: 17772.74933881492, Loss: 2.6802923679351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 71.35725038564516, Tokens per sec: 17746.18967902488, Loss: 2.7338337898254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 71.27388029717389, Tokens per sec: 17725.45595686529, Loss: 2.6909403800964355 +------------------------------------------------------------------ + eval ppl=14.011518478393555, eval loss=2.6398797035217285 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/model.pt +[rank0]:[2024-08-12 18:22:51,906] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006330696000077296, 'preprocessing_with_comm': 0.0006451410008594394, 'state_converting': 0.42921708700305317, : 0.437568918998295}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000420 +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 71.00885725708486, Tokens per sec: 17659.54605824512, Loss: 2.7097086906433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 71.4572954709794, Tokens per sec: 17771.070389130728, Loss: 2.6923580169677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 71.35757228124078, Tokens per sec: 17746.269732842335, Loss: 2.6708343029022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 71.29854907151086, Tokens per sec: 17731.590957109376, Loss: 2.6646013259887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 71.30779069263563, Tokens per sec: 17733.8893018541, Loss: 2.683196783065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 71.01333652351853, Tokens per sec: 17660.660029303777, Loss: 2.6897125244140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 70.99172928011977, Tokens per sec: 17655.286416423194, Loss: 2.6994898319244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 71.52344633998017, Tokens per sec: 17787.521777915685, Loss: 2.677521228790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 71.30219687765424, Tokens per sec: 17732.49814817101, Loss: 2.6926581859588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 71.33693502452256, Tokens per sec: 17741.137350776087, Loss: 2.7270374298095703 +------------------------------------------------------------------ + eval ppl=12.035457611083984, eval loss=2.4878571033477783 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/model.pt +[rank0]:[2024-08-12 18:35:15,876] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006338309001876041, 'preprocessing_with_comm': 0.0006478209979832172, 'state_converting': 0.4372278599985293, : 0.4455666130015743}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000430 +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 71.5061434643715, Tokens per sec: 17783.21863967978, Loss: 2.6804616451263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 71.48054819188738, Tokens per sec: 17776.853224001137, Loss: 2.6721439361572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 71.2664081030492, Tokens per sec: 17723.597659726056, Loss: 2.63501238822937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 71.2061040670375, Tokens per sec: 17708.600348931555, Loss: 2.6624550819396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 70.95128044405162, Tokens per sec: 17645.226994104065, Loss: 2.6979074478149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 71.33115877191373, Tokens per sec: 17739.7008257716, Loss: 2.6408212184906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 71.23192561763138, Tokens per sec: 17715.0220388393, Loss: 2.667269229888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 71.40794074556557, Tokens per sec: 17758.796116873647, Loss: 2.6800148487091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 71.52924357938525, Tokens per sec: 17788.963522230963, Loss: 2.670928716659546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 71.22665533619299, Tokens per sec: 17713.71134632295, Loss: 2.694857358932495 +------------------------------------------------------------------ + eval ppl=16.034610748291016, eval loss=2.774749517440796 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/model.pt +[rank0]:[2024-08-12 18:47:39,070] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006338416998914909, 'preprocessing_with_comm': 0.0006396450044121593, 'state_converting': 0.429151185999217, : 0.4375133920038934}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540, took 3.41s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000440 +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 71.04935385340404, Tokens per sec: 17669.617358298747, Loss: 2.637488603591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 71.2831842160886, Tokens per sec: 17727.769794757394, Loss: 2.599623441696167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 71.13530113735105, Tokens per sec: 17690.992016025568, Loss: 2.6746304035186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 71.36609482756207, Tokens per sec: 17748.389247296, Loss: 2.6397786140441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 71.32160977194371, Tokens per sec: 17737.326037452305, Loss: 2.6720895767211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 71.28902590480538, Tokens per sec: 17729.222593393188, Loss: 2.6700663566589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 71.33636645871256, Tokens per sec: 17740.995951315563, Loss: 2.6700663566589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 71.32143331373624, Tokens per sec: 17737.282153182616, Loss: 2.6648812294006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 71.30091697397299, Tokens per sec: 17732.17984255559, Loss: 2.6763954162597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 71.36164595038056, Tokens per sec: 17747.28283389168, Loss: 2.65657639503479 +------------------------------------------------------------------ + eval ppl=13.024474143981934, eval loss=2.5668301582336426 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/optimizer.pt +[rank0]:[2024-08-12 19:00:02,937] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062814259945298545, 'preprocessing_with_comm': 0.0006470160005846992, 'state_converting': 0.440695520999725, : 0.44894890000432497}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550, took 3.65s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000450 +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 71.51577731088234, Tokens per sec: 17785.61452890751, Loss: 2.6504523754119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 71.36107716256434, Tokens per sec: 17747.141379219327, Loss: 2.631622552871704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 71.27755239145252, Tokens per sec: 17726.36918826414, Loss: 2.6189422607421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 71.22979777225535, Tokens per sec: 17714.492854384367, Loss: 2.6427462100982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 71.47568426120591, Tokens per sec: 17775.64358887657, Loss: 2.6368637084960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 71.52425389708851, Tokens per sec: 17787.722613311445, Loss: 2.652611255645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 71.42896842160751, Tokens per sec: 17764.02559426428, Loss: 2.5924720764160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 71.3710118439969, Tokens per sec: 17749.612084580727, Loss: 2.6253368854522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 71.58056875810679, Tokens per sec: 17801.727836326307, Loss: 2.6484415531158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 71.35783811470797, Tokens per sec: 17746.335844290104, Loss: 2.6488840579986572 +------------------------------------------------------------------ + eval ppl=13.801796913146973, eval loss=2.6247987747192383 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/model.pt +[rank0]:[2024-08-12 19:12:25,221] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062803909968351945, 'preprocessing_with_comm': 0.0006357319944072515, 'state_converting': 0.4298501520024729, : 0.43810601199947996}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560, took 3.48s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000460 +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 70.83001026384602, Tokens per sec: 17615.067709536524, Loss: 2.6226279735565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 71.50812056590881, Tokens per sec: 17783.71033489934, Loss: 2.587860584259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 71.15443612176982, Tokens per sec: 17695.750790518217, Loss: 2.633995294570923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 71.32383711212654, Tokens per sec: 17737.87996576584, Loss: 2.6245288848876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 71.40822108141738, Tokens per sec: 17758.865834991615, Loss: 2.6585490703582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 71.26870413528638, Tokens per sec: 17724.1686714084, Loss: 2.6438260078430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 71.18629290969422, Tokens per sec: 17703.67341362961, Loss: 2.645984172821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 71.31342056229114, Tokens per sec: 17735.289422153506, Loss: 2.6452620029449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 71.36429068212196, Tokens per sec: 17747.940565388893, Loss: 2.6681880950927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 71.50747885471642, Tokens per sec: 17783.550744549684, Loss: 2.6584486961364746 +------------------------------------------------------------------ +[rank0]:[2024-08-12 19:24:48,774] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006274485000176355, 'preprocessing_with_comm': 0.0006557449960382655, 'state_converting': 0.43321124000067357, : 0.4414836420037318}) + eval ppl=14.231071472167969, eval loss=2.6554276943206787 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000470 +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 71.36606555387978, Tokens per sec: 17748.38196707828, Loss: 2.6042890548706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 71.40092000815598, Tokens per sec: 17757.050094751386, Loss: 2.657681703567505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 71.36052851575302, Tokens per sec: 17747.004933513694, Loss: 2.6241819858551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 71.10141697867999, Tokens per sec: 17682.565195994794, Loss: 2.6607813835144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 70.9794684489358, Tokens per sec: 17652.237209304967, Loss: 2.6269454956054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 71.34942207267322, Tokens per sec: 17744.24281691744, Loss: 2.664999008178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 71.11339303310655, Tokens per sec: 17685.543580564146, Loss: 2.644778251647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 71.39686112941295, Tokens per sec: 17756.040671999406, Loss: 2.6233057975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 71.33682180102356, Tokens per sec: 17741.109192660708, Loss: 2.6304049491882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 71.25083747190337, Tokens per sec: 17719.725321985425, Loss: 2.627342462539673 +------------------------------------------------------------------ + eval ppl=11.543144226074219, eval loss=2.446091651916504 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/model.pt +[rank0]:[2024-08-12 19:37:12,644] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006327047995000612, 'preprocessing_with_comm': 0.0006427970001823269, 'state_converting': 0.4275820350012509, : 0.43592022100347094}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580, took 3.71s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000480 +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 70.80010968288968, Tokens per sec: 17607.631585270294, Loss: 2.6506459712982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 71.34198334048062, Tokens per sec: 17742.39284159261, Loss: 2.6596314907073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 71.42581217361382, Tokens per sec: 17763.24065124493, Loss: 2.634934186935425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 71.06003874644486, Tokens per sec: 17672.27463751813, Loss: 2.6522092819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 71.3591418968909, Tokens per sec: 17746.660088368928, Loss: 2.662107467651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 71.11456836135864, Tokens per sec: 17685.835878795704, Loss: 2.5989673137664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 71.14683675093669, Tokens per sec: 17693.860865170518, Loss: 2.6257832050323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 70.71818517996414, Tokens per sec: 17587.257372973443, Loss: 2.6589393615722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 71.22878744979911, Tokens per sec: 17714.24159226526, Loss: 2.650970220565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 71.26222650011191, Tokens per sec: 17722.55771608352, Loss: 2.60752272605896 +------------------------------------------------------------------ + eval ppl=11.430678367614746, eval loss=2.436300754547119 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/model.pt +[rank0]:[2024-08-12 19:49:37,981] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006378848003805615, 'preprocessing_with_comm': 0.000630259994068183, 'state_converting': 0.4259007080036099, : 0.43426011900010053}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590, took 3.50s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000490 +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 71.4409324102922, Tokens per sec: 17767.00097814425, Loss: 2.6066040992736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 71.26573268140045, Tokens per sec: 17723.429685755265, Loss: 2.6202123165130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 71.12279935512014, Tokens per sec: 17687.88288559248, Loss: 2.6158788204193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 71.29261569795204, Tokens per sec: 17730.115356913986, Loss: 2.6118133068084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 70.84651645759395, Tokens per sec: 17619.17271699612, Loss: 2.616863965988159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 71.36078469013994, Tokens per sec: 17747.06864279673, Loss: 2.58803391456604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 71.47082436732693, Tokens per sec: 17774.434957684498, Loss: 2.6488990783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 71.11509007114579, Tokens per sec: 17685.96562539875, Loss: 2.6277363300323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 71.43458603348753, Tokens per sec: 17765.422666116505, Loss: 2.626641035079956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 71.27709118566993, Tokens per sec: 17726.254488703064, Loss: 2.613434314727783 +------------------------------------------------------------------ + eval ppl=13.468440055847168, eval loss=2.600349187850952 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/model.pt +[rank0]:[2024-08-12 20:02:01,891] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063202300007105805, 'preprocessing_with_comm': 0.0006578980028280057, 'state_converting': 0.4374033120038803, : 0.44570843700057594}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600, took 3.50s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000500 +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 71.1356350654134, Tokens per sec: 17691.075062257056, Loss: 2.621626615524292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 70.93336354115141, Tokens per sec: 17640.771150365563, Loss: 2.605522394180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 71.18069114011273, Tokens per sec: 17702.280281677442, Loss: 2.632402181625366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 71.31142353165957, Tokens per sec: 17734.79277066832, Loss: 2.636129379272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 71.32488183829679, Tokens per sec: 17738.139783915773, Loss: 2.6068406105041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 71.48789790572985, Tokens per sec: 17778.68106090951, Loss: 2.57273530960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 71.25364586177713, Tokens per sec: 17720.423754437954, Loss: 2.6424052715301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 70.82573269439688, Tokens per sec: 17614.00389950461, Loss: 2.61812424659729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 71.3303849468258, Tokens per sec: 17739.508379359857, Loss: 2.5809335708618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 71.17653326007898, Tokens per sec: 17701.24623780183, Loss: 2.616703510284424 +------------------------------------------------------------------ + eval ppl=16.000377655029297, eval loss=2.7726123332977295 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/model.pt +[rank0]:[2024-08-12 20:14:26,314] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006287023003096692, 'preprocessing_with_comm': 0.0006141159974504262, 'state_converting': 0.42495324699848425, : 0.433214898002916}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610, took 3.36s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000510 +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 71.32210090068168, Tokens per sec: 17737.448178702056, Loss: 2.620007038116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 71.35609112598097, Tokens per sec: 17745.901376970403, Loss: 2.614288330078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 71.35087515582275, Tokens per sec: 17744.60419139658, Loss: 2.6673507690429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 71.35618086200442, Tokens per sec: 17745.92369386861, Loss: 2.629185438156128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 70.97511371811926, Tokens per sec: 17651.154209628636, Loss: 2.597139835357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 71.33739539391654, Tokens per sec: 17741.251842331516, Loss: 2.603801965713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 71.44332167145636, Tokens per sec: 17767.59517539236, Loss: 2.643608331680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 71.29710018235664, Tokens per sec: 17731.230625655793, Loss: 2.618769884109497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 71.24126001484214, Tokens per sec: 17717.34345653051, Loss: 2.6243410110473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 71.54442440914059, Tokens per sec: 17792.738918324107, Loss: 2.6085336208343506 +------------------------------------------------------------------ + eval ppl=12.083281517028809, eval loss=2.4918227195739746 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/optimizer.pt +[rank0]:[2024-08-12 20:26:49,427] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006255723004869651, 'preprocessing_with_comm': 0.0006345360015984625, 'state_converting': 0.42676708700309973, : 0.4350143589981599}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620, took 3.38s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000520 +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 71.12135255494233, Tokens per sec: 17687.52307365684, Loss: 2.6421337127685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 71.02133688241656, Tokens per sec: 17662.649678368693, Loss: 2.6211330890655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 71.43513910892894, Tokens per sec: 17765.560213200184, Loss: 2.597341775894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 71.35894000433464, Tokens per sec: 17746.609878704472, Loss: 2.5995335578918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 71.42834940571105, Tokens per sec: 17763.8716481207, Loss: 2.5979650020599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 71.58747455854642, Tokens per sec: 17803.44527420157, Loss: 2.6123340129852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 71.26116974878258, Tokens per sec: 17722.29490733693, Loss: 2.609933614730835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 70.96871734050924, Tokens per sec: 17649.563462651804, Loss: 2.5999748706817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 71.17233491552967, Tokens per sec: 17700.202130604495, Loss: 2.6181793212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 71.39109798029041, Tokens per sec: 17754.607405766128, Loss: 2.57389235496521 +------------------------------------------------------------------ + eval ppl=12.938837051391602, eval loss=2.5602333545684814 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/optimizer.pt +[rank0]:[2024-08-12 20:39:13,162] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006289069002377801, 'preprocessing_with_comm': 0.0006412090006051585, 'state_converting': 0.4318644820014015, : 0.44013802699919324}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630, took 3.55s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000530 +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 71.29551825109031, Tokens per sec: 17730.83720729733, Loss: 2.6343817710876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 71.33889003679599, Tokens per sec: 17741.623552506695, Loss: 2.629565715789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 71.19813880710335, Tokens per sec: 17706.619428802624, Loss: 2.635324001312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 71.20358919650283, Tokens per sec: 17707.97491326405, Loss: 2.5843393802642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 70.8475029298827, Tokens per sec: 17619.41804769839, Loss: 2.601383924484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 71.30739554640506, Tokens per sec: 17733.79103097173, Loss: 2.5948500633239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 71.31265902716974, Tokens per sec: 17735.100032194692, Loss: 2.6270480155944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 71.50375709312502, Tokens per sec: 17782.625161139673, Loss: 2.6108357906341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 71.30735813288166, Tokens per sec: 17733.781726416422, Loss: 2.6226799488067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 71.49379518994438, Tokens per sec: 17780.14768586628, Loss: 2.6193089485168457 +------------------------------------------------------------------ + eval ppl=13.19344425201416, eval loss=2.5797200202941895 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/optimizer.pt +[rank0]:[2024-08-12 20:51:36,930] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006276637999690138, 'preprocessing_with_comm': 0.0006412500006263144, 'state_converting': 0.43868140000267886, : 0.4469413490005536}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640, took 3.71s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000540 +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 70.76803563498271, Tokens per sec: 17599.654930692726, Loss: 2.606494903564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 70.92866954921877, Tokens per sec: 17639.603778154073, Loss: 2.565028429031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 71.3883015415685, Tokens per sec: 17753.911945505028, Loss: 2.612762451171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 71.45152400846652, Tokens per sec: 17769.63505539904, Loss: 2.565844774246216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 71.26560481742628, Tokens per sec: 17723.397886627266, Loss: 2.624528169631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 71.42777531898453, Tokens per sec: 17763.728875635978, Loss: 2.6076228618621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 71.41115905998826, Tokens per sec: 17759.596495502075, Loss: 2.5873470306396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 71.09410280104935, Tokens per sec: 17680.746196763772, Loss: 2.618896007537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 71.12287452846026, Tokens per sec: 17687.901580824502, Loss: 2.611088991165161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 71.25523394767592, Tokens per sec: 17720.818703422592, Loss: 2.588839054107666 +------------------------------------------------------------------ + eval ppl=12.178522109985352, eval loss=2.499673843383789 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/model.pt +[rank0]:[2024-08-12 21:04:01,568] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062954099994385615, 'preprocessing_with_comm': 0.0006420279969461262, 'state_converting': 0.43637275499349926, : 0.44468258900451474}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650, took 3.68s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000550 +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 71.42670059206074, Tokens per sec: 17763.46159644935, Loss: 2.601329803466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 71.34729736600292, Tokens per sec: 17743.714413042857, Loss: 2.558255434036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 71.36656801729094, Tokens per sec: 17748.506927204307, Loss: 2.6186420917510986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 71.09734759510918, Tokens per sec: 17681.553160744923, Loss: 2.5706381797790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 70.6835074031032, Tokens per sec: 17578.633184085906, Loss: 2.6005733013153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 71.12792423095678, Tokens per sec: 17689.157416466172, Loss: 2.6383578777313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 71.33933441633556, Tokens per sec: 17741.73406746567, Loss: 2.5920746326446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 71.1874997099253, Tokens per sec: 17703.973538784237, Loss: 2.6279075145721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 71.13672318698647, Tokens per sec: 17691.345672625663, Loss: 2.6031033992767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 71.08326327950316, Tokens per sec: 17678.050462211933, Loss: 2.5821990966796875 +------------------------------------------------------------------ + eval ppl=11.79783821105957, eval loss=2.467916250228882 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/model.pt +[rank0]:[2024-08-12 21:16:26,410] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006284832001256291, 'preprocessing_with_comm': 0.0006323900015559047, 'state_converting': 0.4416960569942603, : 0.44996102100412827}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000560 +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 71.0942131048304, Tokens per sec: 17680.773628759962, Loss: 2.5959324836730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 70.6563674671985, Tokens per sec: 17571.883618374806, Loss: 2.602684736251831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 71.57321327029814, Tokens per sec: 17799.898563461553, Loss: 2.5757980346679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 71.11378461524879, Tokens per sec: 17685.64096507565, Loss: 2.602213144302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 71.02359686549036, Tokens per sec: 17663.211724805216, Loss: 2.6478471755981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 71.32398370037184, Tokens per sec: 17737.916421526, Loss: 2.573497772216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 71.29030133826294, Tokens per sec: 17729.53978728642, Loss: 2.5749292373657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 70.96134614858792, Tokens per sec: 17647.730284253124, Loss: 2.560171604156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 70.91460529261222, Tokens per sec: 17636.106068193483, Loss: 2.6154417991638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 71.24311395202707, Tokens per sec: 17717.804521394402, Loss: 2.5759501457214355 +------------------------------------------------------------------ + eval ppl=10.824126243591309, eval loss=2.38177752494812 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/model.pt +[rank0]:[2024-08-12 21:28:51,716] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006284208997385576, 'preprocessing_with_comm': 0.0006354910001391545, 'state_converting': 0.42412450299889315, : 0.4323929940001108}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670, took 3.47s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000570 +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 71.34417121839557, Tokens per sec: 17742.936955838373, Loss: 2.5892159938812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 71.2249557495409, Tokens per sec: 17713.288667661134, Loss: 2.6050572395324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 71.20065510920053, Tokens per sec: 17707.245220492536, Loss: 2.606461524963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 71.20556370308276, Tokens per sec: 17708.465963130748, Loss: 2.6009769439697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 71.13101037735649, Tokens per sec: 17689.924925571282, Loss: 2.608743906021118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 71.07780445086293, Tokens per sec: 17676.69287895373, Loss: 2.5961365699768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 71.24200246773421, Tokens per sec: 17717.528100834756, Loss: 2.586277723312378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 71.11815371538805, Tokens per sec: 17686.72753833035, Loss: 2.6181719303131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 71.19288555831089, Tokens per sec: 17705.31297221975, Loss: 2.59688401222229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 71.50500719783672, Tokens per sec: 17782.936055901067, Loss: 2.583843231201172 +------------------------------------------------------------------ + eval ppl=14.570671081542969, eval loss=2.6790106296539307 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/model.pt +[rank0]:[2024-08-12 21:41:15,918] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063253429980250075, 'preprocessing_with_comm': 0.0006557419983437285, 'state_converting': 0.43596650499966927, : 0.44429953799408395}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680, took 3.38s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000580 +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 71.26215009844907, Tokens per sec: 17722.538715373805, Loss: 2.593885898590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 70.68401164949313, Tokens per sec: 17578.758587629796, Loss: 2.597064256668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 71.36953186995665, Tokens per sec: 17749.244022472165, Loss: 2.571523904800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 71.2789513589451, Tokens per sec: 17726.71710445127, Loss: 2.581200361251831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 71.1748407578723, Tokens per sec: 17700.825321006, Loss: 2.609408140182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 71.2928143277673, Tokens per sec: 17730.16475515114, Loss: 2.576585054397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 71.336993568536, Tokens per sec: 17741.151910378107, Loss: 2.5837936401367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 71.29468629533828, Tokens per sec: 17730.630304081464, Loss: 2.5391180515289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 70.91559015901001, Tokens per sec: 17636.350999518734, Loss: 2.5756478309631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 71.17136907122675, Tokens per sec: 17699.96192997866, Loss: 2.5816328525543213 +------------------------------------------------------------------ + eval ppl=12.810911178588867, eval loss=2.550297260284424 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/optimizer.pt +[rank0]:[2024-08-12 21:53:40,553] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006282383001234848, 'preprocessing_with_comm': 0.0006334589998004958, 'state_converting': 0.42278370699932566, : 0.4310380739989341}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000590 +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 71.41392349282417, Tokens per sec: 17760.28399606012, Loss: 2.5697414875030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 71.28392142518437, Tokens per sec: 17727.95313495585, Loss: 2.5843818187713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 71.36442706676287, Tokens per sec: 17747.974483563907, Loss: 2.605919122695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 71.4587167901251, Tokens per sec: 17771.42386406169, Loss: 2.608048439025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 71.11488499649198, Tokens per sec: 17685.914624362624, Loss: 2.535573959350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 70.87518385940677, Tokens per sec: 17626.30215580473, Loss: 2.559352397918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 71.29451466594375, Tokens per sec: 17730.587620713293, Loss: 2.540969133377075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 71.43881356712927, Tokens per sec: 17766.474032494487, Loss: 2.5812196731567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 71.3977840618814, Tokens per sec: 17756.270200667586, Loss: 2.5416018962860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 71.48153181485407, Tokens per sec: 17777.09784609132, Loss: 2.569896697998047 +------------------------------------------------------------------ + eval ppl=15.659719467163086, eval loss=2.751091718673706 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/model.pt +[rank0]:[2024-08-12 22:06:04,034] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006313623001915403, 'preprocessing_with_comm': 0.0006334999998216517, 'state_converting': 0.4302136129990686, : 0.43852521600638283}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700, took 3.68s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000600 +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 71.29701553573098, Tokens per sec: 17731.20957446525, Loss: 2.6193084716796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 71.21054023225349, Tokens per sec: 17709.703600933914, Loss: 2.5875046253204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 71.11428805569993, Tokens per sec: 17685.766168186612, Loss: 2.544961452484131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 71.0909133656026, Tokens per sec: 17679.953000191577, Loss: 2.554158926010132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 71.47304012657648, Tokens per sec: 17774.9860058781, Loss: 2.56420636177063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 71.50902653083824, Tokens per sec: 17783.93564382581, Loss: 2.5759711265563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 71.11350097835872, Tokens per sec: 17685.570426006056, Loss: 2.577972173690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 71.51805355114512, Tokens per sec: 17786.18061842526, Loss: 2.5750341415405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 71.0386109836073, Tokens per sec: 17666.945660551424, Loss: 2.5854315757751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 71.28861819018428, Tokens per sec: 17729.121196815235, Loss: 2.5675086975097656 +------------------------------------------------------------------ + eval ppl=12.357229232788086, eval loss=2.5142412185668945 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/optimizer.pt +[rank0]:[2024-08-12 22:18:27,921] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006322193999949377, 'preprocessing_with_comm': 0.0006529430029331706, 'state_converting': 0.4425056420004694, : 0.45082700399507303}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710, took 3.61s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000610 +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 71.23845716814657, Tokens per sec: 17716.64640263855, Loss: 2.601999282836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 71.42399927111448, Tokens per sec: 17762.789791501193, Loss: 2.589268207550049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 71.4150868410683, Tokens per sec: 17760.57331492389, Loss: 2.5273613929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 71.38169223038034, Tokens per sec: 17752.268243017068, Loss: 2.5452420711517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 71.18845170583919, Tokens per sec: 17704.21029538528, Loss: 2.5864081382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 70.86774927807554, Tokens per sec: 17624.453212778262, Loss: 2.5755064487457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 71.39766048504947, Tokens per sec: 17756.239467730324, Loss: 2.556262254714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 71.36460615686266, Tokens per sec: 17748.01902237199, Loss: 2.539137601852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 71.22120261303716, Tokens per sec: 17712.355281468048, Loss: 2.5474915504455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 71.41152094131907, Tokens per sec: 17759.686493570985, Loss: 2.5224242210388184 +------------------------------------------------------------------ + eval ppl=12.595060348510742, eval loss=2.5333046913146973 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/model.pt +[rank0]:[2024-08-12 22:30:51,561] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063079120009206235, 'preprocessing_with_comm': 0.0006327459996100515, 'state_converting': 0.4322910490009235, : 0.4406165609980235}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720, took 3.53s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000620 +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 71.26443437189126, Tokens per sec: 17723.106802702907, Loss: 2.6006627082824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 71.4045617866037, Tokens per sec: 17757.955786755392, Loss: 2.5850253105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 70.88495200120178, Tokens per sec: 17628.731443594956, Loss: 2.5653982162475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 71.5212175579208, Tokens per sec: 17786.967491014708, Loss: 2.56524920463562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 71.31497261114532, Tokens per sec: 17735.675408906198, Loss: 2.5658392906188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 71.2977086625537, Tokens per sec: 17731.38195162386, Loss: 2.559966564178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 71.26606180315814, Tokens per sec: 17723.511536682927, Loss: 2.5923962593078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 71.34390576192835, Tokens per sec: 17742.8709381486, Loss: 2.6183242797851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 71.18010782000815, Tokens per sec: 17702.135212897974, Loss: 2.5987911224365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 70.92917979422941, Tokens per sec: 17639.730673524795, Loss: 2.550365924835205 +------------------------------------------------------------------ + eval ppl=16.477243423461914, eval loss=2.8019802570343018 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/optimizer.pt +[rank0]:[2024-08-12 22:43:15,669] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0068694959991262294, 'preprocessing_with_comm': 0.000630709997494705, 'state_converting': 0.42986105599993607, : 0.43871431699517416}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730, took 3.58s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000630 +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 71.3672531330998, Tokens per sec: 17748.677312064017, Loss: 2.5500237941741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 71.47101299066873, Tokens per sec: 17774.481867361977, Loss: 2.556959629058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 71.34066222899696, Tokens per sec: 17742.064287803758, Loss: 2.581493616104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 71.3859612540406, Tokens per sec: 17753.329927754214, Loss: 2.5906383991241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 71.50145210542358, Tokens per sec: 17782.05192227836, Loss: 2.5624539852142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 70.77921543746233, Tokens per sec: 17602.435291403166, Loss: 2.555405616760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 71.29207269161193, Tokens per sec: 17729.980313965214, Loss: 2.5522050857543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 71.45554321679917, Tokens per sec: 17770.634612319256, Loss: 2.5628206729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 71.29475454313572, Tokens per sec: 17730.647276965818, Loss: 2.5663154125213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 71.2744266719025, Tokens per sec: 17725.591837515363, Loss: 2.5622189044952393 +------------------------------------------------------------------ + eval ppl=12.292380332946777, eval loss=2.508979558944702 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/model.pt +[rank0]:[2024-08-12 22:55:39,003] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006310019998636562, 'preprocessing_with_comm': 0.000629059002676513, 'state_converting': 0.42135418000543723, : 0.42964880100043956}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000640 +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 71.486611037727, Tokens per sec: 17778.3610233023, Loss: 2.5783023834228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 71.36651629784716, Tokens per sec: 17748.494064838476, Loss: 2.574528217315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 70.7652063564751, Tokens per sec: 17598.951303341903, Loss: 2.5615923404693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 71.48239430856664, Tokens per sec: 17777.312343944555, Loss: 2.560299873352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 71.2978175673026, Tokens per sec: 17731.409035687782, Loss: 2.578855037689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 71.43139528775393, Tokens per sec: 17764.629143682556, Loss: 2.509596109390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 71.43699353221425, Tokens per sec: 17766.0213989548, Loss: 2.533257007598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 71.29648921321389, Tokens per sec: 17731.07868069944, Loss: 2.5529074668884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 71.25680889542275, Tokens per sec: 17721.21038501484, Loss: 2.5612378120422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 70.92259015120574, Tokens per sec: 17638.091862410536, Loss: 2.516007900238037 +------------------------------------------------------------------ + eval ppl=13.330920219421387, eval loss=2.5900862216949463 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/optimizer.pt +[rank0]:[2024-08-12 23:08:02,734] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006293662001553457, 'preprocessing_with_comm': 0.0006375729935825802, 'state_converting': 0.42740269399655517, : 0.43569621800270397}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000650 +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 71.21912692576119, Tokens per sec: 17711.839068470563, Loss: 2.5709147453308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 71.33220896237451, Tokens per sec: 17739.962002863143, Loss: 2.5525460243225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 71.31004176735249, Tokens per sec: 17734.449132826998, Loss: 2.5394599437713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 71.38983442437478, Tokens per sec: 17754.293165757896, Loss: 2.5681066513061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 71.29434254660042, Tokens per sec: 17730.544815497316, Loss: 2.5521726608276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 70.84635206344903, Tokens per sec: 17619.131832998177, Loss: 2.548510789871216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 70.83935836638759, Tokens per sec: 17617.39253567465, Loss: 2.559166431427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 71.4266404200336, Tokens per sec: 17763.446631968505, Loss: 2.5640153884887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 71.32425050531747, Tokens per sec: 17737.98277457557, Loss: 2.5477168560028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 71.32920335237829, Tokens per sec: 17739.214522756974, Loss: 2.5505850315093994 +------------------------------------------------------------------ + eval ppl=11.949628829956055, eval loss=2.4807002544403076 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/model.pt +[rank0]:[2024-08-12 23:20:26,968] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006319103995338082, 'preprocessing_with_comm': 0.0006267810094868764, 'state_converting': 0.4217691560043022, : 0.43008219100011047}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000660 +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 71.37425475602191, Tokens per sec: 17750.418580509275, Loss: 2.539921283721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 71.30916986949909, Tokens per sec: 17734.232296211187, Loss: 2.5966529846191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 70.63995875186299, Tokens per sec: 17567.802853306654, Loss: 2.5705740451812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 71.23253352583906, Tokens per sec: 17715.173222556477, Loss: 2.5620296001434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 71.40272723919836, Tokens per sec: 17757.49954403227, Loss: 2.5511021614074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 71.3639473940986, Tokens per sec: 17747.855191382132, Loss: 2.537559986114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 71.42197050260836, Tokens per sec: 17762.285246966054, Loss: 2.53531551361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 71.04616067604182, Tokens per sec: 17668.82323113097, Loss: 2.5583157539367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 71.17150931337872, Tokens per sec: 17699.996807497297, Loss: 2.498750686645508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 70.72349581193103, Tokens per sec: 17588.57810046351, Loss: 2.5390498638153076 +------------------------------------------------------------------ + eval ppl=12.898937225341797, eval loss=2.5571448802948 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/model.pt +[rank0]:[2024-08-12 23:32:51,906] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006300162000115961, 'preprocessing_with_comm': 0.0006627219991059974, 'state_converting': 0.42987301699758973, : 0.4382267439941643}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770, took 3.47s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000670 +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 71.253653995036, Tokens per sec: 17720.42577713857, Loss: 2.5637881755828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 71.56848717446985, Tokens per sec: 17798.7232071725, Loss: 2.548152446746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 71.35239145575221, Tokens per sec: 17744.981287571296, Loss: 2.579843044281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 71.36770576644578, Tokens per sec: 17748.789879703178, Loss: 2.531553268432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 71.37056779923608, Tokens per sec: 17749.501652879553, Loss: 2.5924811363220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 70.9763708188127, Tokens per sec: 17651.466844255545, Loss: 2.5674595832824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 71.21773653525885, Tokens per sec: 17711.493285337816, Loss: 2.534860372543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 71.35286253707181, Tokens per sec: 17745.098443128813, Loss: 2.5285539627075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 71.27038232034562, Tokens per sec: 17724.586027601592, Loss: 2.5760111808776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 71.02498100827559, Tokens per sec: 17663.555954162097, Loss: 2.514777660369873 +------------------------------------------------------------------ + eval ppl=12.109440803527832, eval loss=2.493985414505005 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/optimizer.pt +[rank0]:[2024-08-12 23:45:15,628] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006306216004304588, 'preprocessing_with_comm': 0.0006505959900096059, 'state_converting': 0.4319867290032562, : 0.44035735199577175}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000680 +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 71.39967437380994, Tokens per sec: 17756.74031174747, Loss: 2.532226324081421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 71.30272164300673, Tokens per sec: 17732.628654677803, Loss: 2.5886192321777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 70.99048159097225, Tokens per sec: 17654.976122400476, Loss: 2.526139974594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 71.14736652096181, Tokens per sec: 17693.99261631425, Loss: 2.540769577026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 71.30286108976323, Tokens per sec: 17732.663334385576, Loss: 2.5502614974975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 71.48015380470754, Tokens per sec: 17776.755141890873, Loss: 2.55222487449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 71.4394343279167, Tokens per sec: 17766.62841258368, Loss: 2.567939281463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 71.3574982317309, Tokens per sec: 17746.25131710125, Loss: 2.5567739009857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 71.2755824421611, Tokens per sec: 17725.8792717722, Loss: 2.539708375930786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 70.76480304126375, Tokens per sec: 17598.851000875053, Loss: 2.5517373085021973 +------------------------------------------------------------------ + eval ppl=13.880173683166504, eval loss=2.6304614543914795 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/model.pt +[rank0]:[2024-08-12 23:57:39,610] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0062757129926467314, 'preprocessing_with_comm': 0.0006179199990583584, 'state_converting': 0.431222687009722, : 0.4394792690000031}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000690 +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 71.2465942391313, Tokens per sec: 17718.6700513126, Loss: 2.544667959213257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 71.45077975615503, Tokens per sec: 17769.449963588224, Loss: 2.5483715534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 71.23985579708692, Tokens per sec: 17716.994234629437, Loss: 2.530604362487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 71.23567506560471, Tokens per sec: 17715.9545077134, Loss: 2.586543083190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 71.29897061417144, Tokens per sec: 17731.695792651266, Loss: 2.5545523166656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 71.05959114904273, Tokens per sec: 17672.163322292905, Loss: 2.5750772953033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 70.91327567832448, Tokens per sec: 17635.775399799975, Loss: 2.57017183303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 71.2357233708532, Tokens per sec: 17715.966520986018, Loss: 2.528266191482544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 71.413523527142, Tokens per sec: 17760.184526604357, Loss: 2.509748697280884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 71.51701545398718, Tokens per sec: 17785.92244887738, Loss: 2.5652565956115723 +------------------------------------------------------------------ + eval ppl=15.893733024597168, eval loss=2.7659249305725098 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/model.pt +[rank0]:[2024-08-13 00:10:03,619] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006288963995757513, 'preprocessing_with_comm': 0.0006425869942177087, 'state_converting': 0.4228805239981739, : 0.4312063270044746}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800, took 3.43s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000700 +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 71.39185854928152, Tokens per sec: 17754.796555453195, Loss: 2.4899988174438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 71.10462146927702, Tokens per sec: 17683.36213670723, Loss: 2.5292105674743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 70.98082167482417, Tokens per sec: 17652.573749784926, Loss: 2.530738115310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 71.11569912582334, Tokens per sec: 17686.117094237226, Loss: 2.5199437141418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 71.43124664417519, Tokens per sec: 17764.59217677129, Loss: 2.503594398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 71.49753860239527, Tokens per sec: 17781.078653736276, Loss: 2.540034532546997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 71.32271528573554, Tokens per sec: 17737.60097317834, Loss: 2.5495355129241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 71.39592142028111, Tokens per sec: 17755.806971059323, Loss: 2.528298854827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 71.28262435170203, Tokens per sec: 17727.63055929716, Loss: 2.511029005050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 70.81286471372862, Tokens per sec: 17610.8036973599, Loss: 2.562926769256592 +------------------------------------------------------------------ + eval ppl=13.134077072143555, eval loss=2.5752100944519043 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/optimizer.pt +[rank0]:[2024-08-13 00:22:27,780] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006327884999336675, 'preprocessing_with_comm': 0.0007151040044846013, 'state_converting': 0.43312792900542263, : 0.44154818999231793}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810, took 3.57s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000710 +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 71.18519138545032, Tokens per sec: 17703.399470084103, Loss: 2.556276798248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 71.41866232643817, Tokens per sec: 17761.46252017248, Loss: 2.565661668777466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 71.31026798876502, Tokens per sec: 17734.50539295578, Loss: 2.5178496837615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 71.41591097466717, Tokens per sec: 17760.778272809563, Loss: 2.5511627197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 71.55274559719763, Tokens per sec: 17794.80835598905, Loss: 2.571293354034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 71.36189628937434, Tokens per sec: 17747.34509194176, Loss: 2.556765556335449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 70.72428134479951, Tokens per sec: 17588.77345854146, Loss: 2.571324110031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 71.38229616936742, Tokens per sec: 17752.418439609024, Loss: 2.5627784729003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 71.32930143771449, Tokens per sec: 17739.23891608732, Loss: 2.5488951206207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 71.22328527361178, Tokens per sec: 17712.87322868987, Loss: 2.5324196815490723 +------------------------------------------------------------------ + eval ppl=11.948407173156738, eval loss=2.480597972869873 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/optimizer.pt +[rank0]:[2024-08-13 00:34:51,340] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006296039005974308, 'preprocessing_with_comm': 0.0006525939970742911, 'state_converting': 0.43325114699837286, : 0.44160585099598393}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820, took 3.46s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000720 +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 71.41688612341342, Tokens per sec: 17761.0207874037, Loss: 2.5179367065429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 71.42394227573794, Tokens per sec: 17762.775617037387, Loss: 2.529581308364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 71.25528332254282, Tokens per sec: 17720.830982703934, Loss: 2.518451452255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 70.82912644990036, Tokens per sec: 17614.84790944843, Loss: 2.5343775749206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 71.47741145368273, Tokens per sec: 17776.073132968304, Loss: 2.537330389022827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 71.22962039851215, Tokens per sec: 17714.448742425542, Loss: 2.531458854675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 71.52200159957316, Tokens per sec: 17787.1624782347, Loss: 2.5204966068267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 71.55143923029124, Tokens per sec: 17794.48346910249, Loss: 2.5364837646484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 71.40582986814061, Tokens per sec: 17758.271152262907, Loss: 2.5504727363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 71.4070290906152, Tokens per sec: 17758.569392867557, Loss: 2.546621799468994 +------------------------------------------------------------------ + eval ppl=12.703929901123047, eval loss=2.5419113636016846 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/optimizer.pt +[rank0]:[2024-08-13 00:47:14,277] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006335243990179151, 'preprocessing_with_comm': 0.0006474960100604221, 'state_converting': 0.43559274599829223, : 0.4439440580026712}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830, took 3.60s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000730 +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 70.96656693815126, Tokens per sec: 17649.028668388783, Loss: 2.5556836128234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 71.22938408442806, Tokens per sec: 17714.389972300047, Loss: 2.5438129901885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 71.48004745826614, Tokens per sec: 17776.72869406517, Loss: 2.5367300510406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 71.1981721559115, Tokens per sec: 17706.62772248367, Loss: 2.513378620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 71.14912465374648, Tokens per sec: 17694.42985510511, Loss: 2.5856621265411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 71.23344443642998, Tokens per sec: 17715.39976144411, Loss: 2.5408856868743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 70.56333581126826, Tokens per sec: 17548.747112926943, Loss: 2.52040696144104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 70.97602564240135, Tokens per sec: 17651.381000616177, Loss: 2.5687499046325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 71.24244559193204, Tokens per sec: 17717.638303596545, Loss: 2.550708532333374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 71.21345408575958, Tokens per sec: 17710.428261661957, Loss: 2.548085927963257 +------------------------------------------------------------------ + eval ppl=13.301420211791992, eval loss=2.5878708362579346 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/optimizer.pt +[rank0]:[2024-08-13 00:59:39,749] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006341403000988066, 'preprocessing_with_comm': 0.0006522329931613058, 'state_converting': 0.44875571799639147, : 0.4571254150068853}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840, took 3.65s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000740 +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 70.96778026462567, Tokens per sec: 17649.33041658734, Loss: 2.5107765197753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 71.26142851441412, Tokens per sec: 17722.35926104948, Loss: 2.5259227752685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 71.41454816105687, Tokens per sec: 17760.43934791132, Loss: 2.5677003860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 70.80934107989067, Tokens per sec: 17609.9273873268, Loss: 2.5433871746063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 71.18105036066956, Tokens per sec: 17702.369618025234, Loss: 2.4985268115997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 71.29893255665094, Tokens per sec: 17731.686327937114, Loss: 2.533405303955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 71.3152467319255, Tokens per sec: 17735.74358136707, Loss: 2.5673153400421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 71.11479099290885, Tokens per sec: 17685.891246143765, Loss: 2.495504379272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 71.10850149668171, Tokens per sec: 17684.327080029892, Loss: 2.509228229522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 71.07536262132578, Tokens per sec: 17676.085608215362, Loss: 2.532881259918213 +------------------------------------------------------------------ + eval ppl=13.128334999084473, eval loss=2.574772834777832 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/optimizer.pt +[rank0]:[2024-08-13 01:12:04,978] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006396972996299155, 'preprocessing_with_comm': 0.0007417239976348355, 'state_converting': 0.4412809890054632, : 0.4498009009985253}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850, took 3.76s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000750 +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 70.69560072782666, Tokens per sec: 17581.640733188942, Loss: 2.489776134490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 71.06490725109481, Tokens per sec: 17673.485410165682, Loss: 2.511200428009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 71.24163249767477, Tokens per sec: 17717.436091139672, Loss: 2.512148857116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 71.04351838667259, Tokens per sec: 17668.166107039448, Loss: 2.502609968185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 71.37691069745816, Tokens per sec: 17751.079099801278, Loss: 2.4990100860595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 71.25403320855864, Tokens per sec: 17720.52008563652, Loss: 2.4964377880096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 70.67705617199988, Tokens per sec: 17577.028795320864, Loss: 2.536435842514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 70.7452875698623, Tokens per sec: 17593.99760118134, Loss: 2.533745050430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 71.27301445603662, Tokens per sec: 17725.24062652435, Loss: 2.4872474670410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 71.21003915932886, Tokens per sec: 17709.578986614895, Loss: 2.5753331184387207 +------------------------------------------------------------------ + eval ppl=13.076066017150879, eval loss=2.5707836151123047 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/optimizer.pt +[rank0]:[2024-08-13 01:24:31,027] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006318119005300105, 'preprocessing_with_comm': 0.0008225040073739365, 'state_converting': 0.43788660000427626, : 0.4464324040018255}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860, took 3.47s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000760 +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 71.18298644865848, Tokens per sec: 17702.851113381363, Loss: 2.4970245361328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 71.29710405964488, Tokens per sec: 17731.2315899179, Loss: 2.5450568199157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 71.13285314049234, Tokens per sec: 17690.383211505305, Loss: 2.5422110557556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 70.7972652831097, Tokens per sec: 17606.924197334996, Loss: 2.5030806064605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 71.05413162472352, Tokens per sec: 17670.805566022842, Loss: 2.547537326812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 71.21965859746679, Tokens per sec: 17711.971292552684, Loss: 2.5330681800842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 71.16478185358234, Tokens per sec: 17698.32372204404, Loss: 2.5082592964172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 71.20320989556177, Tokens per sec: 17707.88058302558, Loss: 2.504725933074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 71.24186289403139, Tokens per sec: 17717.493389556068, Loss: 2.544630527496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 71.42812801969646, Tokens per sec: 17763.8165905311, Loss: 2.531947374343872 +------------------------------------------------------------------ + eval ppl=14.447243690490723, eval loss=2.670503616333008 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/optimizer.pt +[rank0]:[2024-08-13 01:36:55,850] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006287922995397821, 'preprocessing_with_comm': 0.0006587810057681054, 'state_converting': 0.43766839799354784, : 0.4459974830097053}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870, took 3.58s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000770 +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 70.51223059570195, Tokens per sec: 17536.037502563235, Loss: 2.5414512157440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 71.2992555155151, Tokens per sec: 17731.766646184118, Loss: 2.5442917346954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 71.27067899423804, Tokens per sec: 17724.659808908178, Loss: 2.529419422149658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 71.08333500595863, Tokens per sec: 17678.068300221064, Loss: 2.5235300064086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 71.25573202097843, Tokens per sec: 17720.942571750653, Loss: 2.5046329498291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 71.18097418774549, Tokens per sec: 17702.350674201705, Loss: 2.5218610763549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 70.8124079038923, Tokens per sec: 17610.69009104858, Loss: 2.510213613510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 70.89699942469868, Tokens per sec: 17631.72757729355, Loss: 2.5217440128326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 71.30394861798173, Tokens per sec: 17732.933797189882, Loss: 2.543921709060669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 71.19972388981408, Tokens per sec: 17707.013630909485, Loss: 2.5410475730895996 +------------------------------------------------------------------ + eval ppl=13.366466522216797, eval loss=2.5927491188049316 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/model.pt +[rank0]:[2024-08-13 01:49:21,682] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006330317002721131, 'preprocessing_with_comm': 0.00067044299794361, 'state_converting': 0.4347392969939392, : 0.4430797060049372}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880, took 3.52s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000780 +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 71.34027855454326, Tokens per sec: 17741.968869894663, Loss: 2.5116302967071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 71.34802273919274, Tokens per sec: 17743.89480971096, Loss: 2.540105104446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 71.21177596883733, Tokens per sec: 17710.01092241409, Loss: 2.5079736709594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 70.70786439047998, Tokens per sec: 17584.690644479375, Loss: 2.491309881210327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 71.37286676202434, Tokens per sec: 17750.073393375227, Loss: 2.5351369380950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 71.23254522331189, Tokens per sec: 17715.176131659202, Loss: 2.537794828414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 71.15273984637395, Tokens per sec: 17695.328935349185, Loss: 2.5475101470947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 71.26484969912373, Tokens per sec: 17723.21009249906, Loss: 2.5102698802948 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 71.23693928881316, Tokens per sec: 17716.268913674012, Loss: 2.5180745124816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 71.07096103541801, Tokens per sec: 17674.990955913236, Loss: 2.502842903137207 +------------------------------------------------------------------ + eval ppl=12.541333198547363, eval loss=2.5290298461914062 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/optimizer.pt +[rank0]:[2024-08-13 02:01:46,301] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006387999004800804, 'preprocessing_with_comm': 0.0006559080065926537, 'state_converting': 0.44438474100024905, : 0.45283564399869647}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000790 +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 70.81133712598732, Tokens per sec: 17610.423793963088, Loss: 2.524041175842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 71.20863786495138, Tokens per sec: 17709.23049174319, Loss: 2.538909435272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 71.46688971566243, Tokens per sec: 17773.456429582846, Loss: 2.500671148300171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 71.19381151936268, Tokens per sec: 17705.543254081396, Loss: 2.520402431488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 71.26365684756264, Tokens per sec: 17722.91343630859, Loss: 2.512890577316284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 71.40789117342239, Tokens per sec: 17758.783788530684, Loss: 2.512657880783081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 70.6844094827425, Tokens per sec: 17578.857526760235, Loss: 2.4861927032470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 71.01855578441459, Tokens per sec: 17661.95803326757, Loss: 2.5435361862182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 71.37944208371609, Tokens per sec: 17751.708642846188, Loss: 2.51224684715271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 71.30616470955455, Tokens per sec: 17733.484928030615, Loss: 2.4880661964416504 +------------------------------------------------------------------ + eval ppl=14.256708145141602, eval loss=2.6572275161743164 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/optimizer.pt +[rank0]:[2024-08-13 02:14:11,109] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063246850040741265, 'preprocessing_with_comm': 0.0006354270008159801, 'state_converting': 0.4379355709970696, : 0.4462177579989657}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000800 +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 71.22609743338379, Tokens per sec: 17713.572598697152, Loss: 2.5348942279815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 71.15166552642354, Tokens per sec: 17695.0617573748, Loss: 2.505711317062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 71.2539447896336, Tokens per sec: 17720.498096294068, Loss: 2.524866819381714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 70.7115415256965, Tokens per sec: 17585.60512953415, Loss: 2.491720199584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 71.30303531506897, Tokens per sec: 17732.706663343823, Loss: 2.5505073070526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 71.01891238395935, Tokens per sec: 17662.04671778283, Loss: 2.542440414428711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 71.1576706783184, Tokens per sec: 17696.55520848175, Loss: 2.507657289505005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 71.30697348936802, Tokens per sec: 17733.68606750699, Loss: 2.4948439598083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 71.1435590140132, Tokens per sec: 17693.045708464677, Loss: 2.53745174407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 71.12072715914124, Tokens per sec: 17687.36754086304, Loss: 2.55765438079834 +------------------------------------------------------------------ + eval ppl=12.623397827148438, eval loss=2.5355520248413086 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/model.pt +[rank0]:[2024-08-13 02:26:36,295] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063716220029164106, 'preprocessing_with_comm': 0.0006601890054298565, 'state_converting': 0.43966635000833776, : 0.4480654249928193}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000810 +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 70.99649876312333, Tokens per sec: 17656.472562884763, Loss: 2.5050275325775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 71.30242793209185, Tokens per sec: 17732.555610248844, Loss: 2.5046825408935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 71.3518160359452, Tokens per sec: 17744.838183556145, Loss: 2.493887186050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 71.38962359244184, Tokens per sec: 17754.240732915372, Loss: 2.499063491821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 70.99779291143777, Tokens per sec: 17656.794411068888, Loss: 2.5444416999816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 71.28963975004623, Tokens per sec: 17729.375253620685, Loss: 2.4885313510894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 71.06404295632595, Tokens per sec: 17673.270464398796, Loss: 2.5244643688201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 71.14578849163914, Tokens per sec: 17693.60016834957, Loss: 2.547147274017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 71.32706451180903, Tokens per sec: 17738.682603852725, Loss: 2.505039930343628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 71.42351691031106, Tokens per sec: 17762.669830792714, Loss: 2.5121564865112305 +------------------------------------------------------------------ + eval ppl=9.461701393127441, eval loss=2.2472522258758545 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/optimizer.pt +[rank0]:[2024-08-13 02:39:00,478] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00626992600155063, 'preprocessing_with_comm': 0.0006392849900294095, 'state_converting': 0.42624189499474596, : 0.43451103499683086}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000820 +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 71.00927385746661, Tokens per sec: 17659.649664667104, Loss: 2.5136194229125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 71.17550459815817, Tokens per sec: 17700.990414750024, Loss: 2.492363214492798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 71.06812648064438, Tokens per sec: 17674.286016381564, Loss: 2.4959216117858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 70.71203462793389, Tokens per sec: 17585.727761583294, Loss: 2.535923957824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 71.2392528353813, Tokens per sec: 17716.84428108247, Loss: 2.5286223888397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 71.40609349236914, Tokens per sec: 17758.336714284116, Loss: 2.531712293624878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 71.34194461718494, Tokens per sec: 17742.383211303513, Loss: 2.4899520874023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 71.32980596437355, Tokens per sec: 17739.36438933273, Loss: 2.4794762134552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 71.13588804767501, Tokens per sec: 17691.137977674556, Loss: 2.491630792617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 71.21943525784495, Tokens per sec: 17711.915749110765, Loss: 2.516887664794922 +------------------------------------------------------------------ + eval ppl=10.657057762145996, eval loss=2.366222381591797 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/optimizer.pt +[rank0]:[2024-08-13 02:51:25,392] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006310404001851566, 'preprocessing_with_comm': 0.0006323690031422302, 'state_converting': 0.42676694999681786, : 0.43504637600563}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930, took 3.45s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000830 +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 70.93580988952328, Tokens per sec: 17641.379544915428, Loss: 2.5301408767700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 71.40098829026024, Tokens per sec: 17757.06707616767, Loss: 2.5090272426605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 71.20178577700749, Tokens per sec: 17707.52641189576, Loss: 2.541907787322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 71.26306656121714, Tokens per sec: 17722.766635060005, Loss: 2.53654146194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 71.03702714092844, Tokens per sec: 17666.55176683427, Loss: 2.5023763179779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 71.38660024279659, Tokens per sec: 17753.48884104761, Loss: 2.543023109436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 71.18586850054946, Tokens per sec: 17703.5678652075, Loss: 2.5256128311157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 70.7534738573851, Tokens per sec: 17596.03348976115, Loss: 2.4666543006896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 71.25290140279849, Tokens per sec: 17720.238611230052, Loss: 2.516977310180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 71.19134102531449, Tokens per sec: 17704.92885462313, Loss: 2.5178515911102295 +------------------------------------------------------------------ + eval ppl=12.54210090637207, eval loss=2.5290911197662354 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/optimizer.pt +[rank0]:[2024-08-13 03:03:50,312] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006397781005944125, 'preprocessing_with_comm': 0.0006435660034185275, 'state_converting': 0.43271251099940855, : 0.4411444370052777}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940, took 3.55s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000840 +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 71.28974324681519, Tokens per sec: 17729.400992747163, Loss: 2.4804768562316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 71.39618313415984, Tokens per sec: 17755.872057986136, Loss: 2.5182316303253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 71.2365533436505, Tokens per sec: 17716.172931051005, Loss: 2.5144541263580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 70.68903789366176, Tokens per sec: 17580.008589303172, Loss: 2.519331693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 71.13793386336259, Tokens per sec: 17691.646761758082, Loss: 2.511082410812378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 71.0948704207243, Tokens per sec: 17680.937099920484, Loss: 2.505112886428833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 71.17594767100898, Tokens per sec: 17701.10060474207, Loss: 2.5289571285247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 71.25053417978484, Tokens per sec: 17719.649894759255, Loss: 2.49685001373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 71.26066411288659, Tokens per sec: 17722.169158229863, Loss: 2.477969169616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 71.2047334039208, Tokens per sec: 17708.259471900514, Loss: 2.486011266708374 +------------------------------------------------------------------ + eval ppl=12.226787567138672, eval loss=2.503629207611084 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/optimizer.pt +[rank0]:[2024-08-13 03:16:15,210] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006312968005659059, 'preprocessing_with_comm': 0.0006407940090866759, 'state_converting': 0.45213882099778857, : 0.46044197799346875}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950, took 3.67s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000850 +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 70.69949043974567, Tokens per sec: 17582.608085001666, Loss: 2.534461498260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 71.11829519951617, Tokens per sec: 17686.762724722208, Loss: 2.5125925540924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 71.08222005657485, Tokens per sec: 17677.791017910717, Loss: 2.4797821044921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 71.25599284196592, Tokens per sec: 17721.007436619904, Loss: 2.5373220443725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 71.28810092135153, Tokens per sec: 17728.992554655237, Loss: 2.4877707958221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 71.17399168728616, Tokens per sec: 17700.61416141687, Loss: 2.5367488861083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 71.06828085222685, Tokens per sec: 17674.324407818574, Loss: 2.4753530025482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 70.61620741205245, Tokens per sec: 17561.896014420192, Loss: 2.504455804824829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 71.12749027390039, Tokens per sec: 17689.04949352641, Loss: 2.5204896926879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 71.23560582772451, Tokens per sec: 17715.937288600442, Loss: 2.479182481765747 +------------------------------------------------------------------ + eval ppl=11.389263153076172, eval loss=2.432671070098877 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/model.pt +[rank0]:[2024-08-13 03:28:41,126] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063025819981703535, 'preprocessing_with_comm': 0.0006330739997792989, 'state_converting': 0.42324775499582756, : 0.43153162000817247}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000860 +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 71.13252247696262, Tokens per sec: 17690.300977146686, Loss: 2.528198719024658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 71.0600746719502, Tokens per sec: 17672.283572010823, Loss: 2.499155282974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 71.01440070470015, Tokens per sec: 17660.924685817306, Loss: 2.5154786109924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 70.71273037630563, Tokens per sec: 17585.900790707972, Loss: 2.511220693588257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 70.96528485380021, Tokens per sec: 17648.709820451746, Loss: 2.46844220161438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 71.2458848577539, Tokens per sec: 17718.493631727903, Loss: 2.4896771907806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 71.02276617338829, Tokens per sec: 17663.005135852745, Loss: 2.491431713104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 71.07644034210585, Tokens per sec: 17676.353631959, Loss: 2.512799024581909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 71.10519369444687, Tokens per sec: 17683.504446232164, Loss: 2.4945008754730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 71.25631069053826, Tokens per sec: 17721.086483963, Loss: 2.5388479232788086 +------------------------------------------------------------------ + eval ppl=13.662372589111328, eval loss=2.614645481109619 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/model.pt +[rank0]:[2024-08-13 03:41:07,106] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00637941800232511, 'preprocessing_with_comm': 0.0006496500136563554, 'state_converting': 0.42794302900438197, : 0.4363534229923971}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970, took 3.56s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000870 +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 70.58155478551618, Tokens per sec: 17553.278080292046, Loss: 2.4945571422576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 70.81343596426781, Tokens per sec: 17610.94576449909, Loss: 2.5200822353363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 70.82471257043561, Tokens per sec: 17613.750199800448, Loss: 2.5021581649780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 71.10654368979122, Tokens per sec: 17683.840183292057, Loss: 2.503295421600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 71.2060283406376, Tokens per sec: 17708.581516156344, Loss: 2.4915595054626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 71.32900484499123, Tokens per sec: 17739.165154967093, Loss: 2.5083062648773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 71.37741595293097, Tokens per sec: 17751.20475429901, Loss: 2.5014803409576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 70.6796286416799, Tokens per sec: 17577.668555606448, Loss: 2.4663503170013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 71.00402510215268, Tokens per sec: 17658.344325589766, Loss: 2.4983835220336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 71.23682112270838, Tokens per sec: 17716.239526357404, Loss: 2.517317056655884 +------------------------------------------------------------------ + eval ppl=11.422377586364746, eval loss=2.4355742931365967 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/optimizer.pt +[rank0]:[2024-08-13 03:53:33,606] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006285940995439887, 'preprocessing_with_comm': 0.000642782004433684, 'state_converting': 0.433410907004145, : 0.4416760790045373}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980, took 3.53s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000880 +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 71.20868053161733, Tokens per sec: 17709.241102728658, Loss: 2.5023601055145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 71.18191455416948, Tokens per sec: 17702.584538607043, Loss: 2.49569034576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 71.09832770667204, Tokens per sec: 17681.79690956662, Loss: 2.4816408157348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 70.78198921175924, Tokens per sec: 17603.125115135634, Loss: 2.5083494186401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 70.80366427418816, Tokens per sec: 17608.515594268305, Loss: 2.4773027896881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 70.9414524817396, Tokens per sec: 17642.782829251795, Loss: 2.512493848800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 71.1823280425692, Tokens per sec: 17702.687371094722, Loss: 2.4555509090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 71.4223208623516, Tokens per sec: 17762.372379674023, Loss: 2.4558298587799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 71.02215169680059, Tokens per sec: 17662.85231861245, Loss: 2.490522623062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 70.9668498226092, Tokens per sec: 17649.09902033229, Loss: 2.4563827514648438 +------------------------------------------------------------------ + eval ppl=9.61293888092041, eval loss=2.2631099224090576 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/model.pt +[rank0]:[2024-08-13 04:05:59,497] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006248531994060613, 'preprocessing_with_comm': 0.0006368929898599163, 'state_converting': 0.42863789400144015, : 0.43686915800208226}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990, took 3.46s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000890 +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 70.72470696283912, Tokens per sec: 17588.879307609648, Loss: 2.513686418533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 70.856022503006, Tokens per sec: 17621.536822732647, Loss: 2.4964613914489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 71.14505863423037, Tokens per sec: 17693.41865647874, Loss: 2.490058422088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 71.00969846868777, Tokens per sec: 17659.755263344603, Loss: 2.504167079925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 71.23635741355554, Tokens per sec: 17716.12420422072, Loss: 2.4593703746795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 71.04909746640877, Tokens per sec: 17669.553596141082, Loss: 2.494349241256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 71.30938372218013, Tokens per sec: 17734.28548029859, Loss: 2.5052175521850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 70.61712691802413, Tokens per sec: 17562.124690935838, Loss: 2.4789533615112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 71.30037232712884, Tokens per sec: 17732.044391621705, Loss: 2.503946304321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 71.02445608787403, Tokens per sec: 17663.42540909538, Loss: 2.493548631668091 +------------------------------------------------------------------ + eval ppl=10.395578384399414, eval loss=2.3413805961608887 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/model.pt +[rank0]:[2024-08-13 04:18:25,742] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006290545003139414, 'preprocessing_with_comm': 0.0006086229986976832, 'state_converting': 0.42030858299403917, : 0.42853963500238024}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000, took 3.31s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000900 +------------------------------------------------------------------ +iteration: 1001 , TFLOPS: 71.23885576343189, Tokens per sec: 17716.745531283508, Loss: 2.470845937728882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1002 , TFLOPS: 70.87639586314761, Tokens per sec: 17626.603575046087, Loss: 2.47996187210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1003 , TFLOPS: 71.2478002258683, Tokens per sec: 17718.969974155334, Loss: 2.4653069972991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1004 , TFLOPS: 71.07212214095419, Tokens per sec: 17675.279717026802, Loss: 2.4367780685424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1005 , TFLOPS: 70.84394082053535, Tokens per sec: 17618.532168999398, Loss: 2.513883590698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1006 , TFLOPS: 71.17244500193581, Tokens per sec: 17700.22950854064, Loss: 2.5105104446411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1007 , TFLOPS: 70.87662476463144, Tokens per sec: 17626.660501695136, Loss: 2.529297113418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1008 , TFLOPS: 71.11220211341237, Tokens per sec: 17685.24740481927, Loss: 2.4388184547424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1009 , TFLOPS: 71.25444222275001, Tokens per sec: 17720.621805411065, Loss: 2.507357120513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1010 , TFLOPS: 71.27416426855584, Tokens per sec: 17725.526579121335, Loss: 2.551703691482544 +------------------------------------------------------------------ + eval ppl=12.213748931884766, eval loss=2.5025622844696045 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/model.pt +[rank0]:[2024-08-13 04:30:51,202] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006355218996759504, 'preprocessing_with_comm': 0.0006171489949338138, 'state_converting': 0.44057189699378796, : 0.4489548349956749}) +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000910 +------------------------------------------------------------------ +iteration: 1011 , TFLOPS: 70.8310795309007, Tokens per sec: 17615.333630881127, Loss: 2.4708638191223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1012 , TFLOPS: 71.0456538891774, Tokens per sec: 17668.697195783836, Loss: 2.474076509475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1013 , TFLOPS: 70.98738558429461, Tokens per sec: 17654.206161093785, Loss: 2.5167572498321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1014 , TFLOPS: 71.15658003043511, Tokens per sec: 17696.283969832486, Loss: 2.4539875984191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1015 , TFLOPS: 71.37049714714372, Tokens per sec: 17749.48408205913, Loss: 2.537517786026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1016 , TFLOPS: 71.0502327035434, Tokens per sec: 17669.835923913146, Loss: 2.4953529834747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1017 , TFLOPS: 71.05152423505407, Tokens per sec: 17670.15712131132, Loss: 2.4675488471984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1018 , TFLOPS: 70.68339402566686, Tokens per sec: 17578.604987687082, Loss: 2.4895453453063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1019 , TFLOPS: 71.2289399745448, Tokens per sec: 17714.279524403246, Loss: 2.474761486053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1020 , TFLOPS: 71.21821280026161, Tokens per sec: 17711.611730051296, Loss: 2.4946401119232178 +------------------------------------------------------------------ + eval ppl=12.942785263061523, eval loss=2.5605385303497314 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020, took 3.41s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000920 +[rank0]:[2024-08-13 04:43:17,083] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006444713988457806, 'preprocessing_with_comm': 0.0006401959981303662, 'state_converting': 0.4340807200060226, : 0.44249211699934676}) +------------------------------------------------------------------ +iteration: 1021 , TFLOPS: 71.45925463138092, Tokens per sec: 17771.55762247995, Loss: 2.5106360912323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1022 , TFLOPS: 71.33588233567808, Tokens per sec: 17740.87555234907, Loss: 2.484265089035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1023 , TFLOPS: 71.33715490534921, Tokens per sec: 17741.192034033014, Loss: 2.5106360912323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1024 , TFLOPS: 71.16986984342095, Tokens per sec: 17699.58907955534, Loss: 2.512360095977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1025 , TFLOPS: 70.82421816092021, Tokens per sec: 17613.627242637835, Loss: 2.4997243881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1026 , TFLOPS: 71.17786917924053, Tokens per sec: 17701.578474185782, Loss: 2.4928081035614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1027 , TFLOPS: 71.26377081481665, Tokens per sec: 17722.9417793921, Loss: 2.488577127456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1028 , TFLOPS: 71.37932606548766, Tokens per sec: 17751.67978969564, Loss: 2.474820613861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1029 , TFLOPS: 70.96884239296482, Tokens per sec: 17649.594562569255, Loss: 2.4937543869018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1030 , TFLOPS: 71.08887426823887, Tokens per sec: 17679.445886921443, Loss: 2.462290048599243 +------------------------------------------------------------------ + eval ppl=10.847935676574707, eval loss=2.38397479057312 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/optimizer.pt +[rank0]:[2024-08-13 04:55:41,556] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006353133998345584, 'preprocessing_with_comm': 0.0006317130028037354, 'state_converting': 0.422305347994552, : 0.4306488169968361}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030, took 3.44s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000930 +------------------------------------------------------------------ +iteration: 1031 , TFLOPS: 70.8129355312049, Tokens per sec: 17610.82130931047, Loss: 2.490074396133423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1032 , TFLOPS: 71.10743491062475, Tokens per sec: 17684.06182543595, Loss: 2.487534284591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1033 , TFLOPS: 71.2663346520716, Tokens per sec: 17723.57939283694, Loss: 2.5068917274475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1034 , TFLOPS: 71.14640579987082, Tokens per sec: 17693.753689805493, Loss: 2.5188376903533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1035 , TFLOPS: 71.01543952901481, Tokens per sec: 17661.18303620541, Loss: 2.4720284938812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1036 , TFLOPS: 71.34342599047189, Tokens per sec: 17742.751621397707, Loss: 2.47607159614563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1037 , TFLOPS: 70.89357616463674, Tokens per sec: 17630.87622971427, Loss: 2.5363612174987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1038 , TFLOPS: 60.43794031666894, Tokens per sec: 15030.612122422168, Loss: 2.4823286533355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1039 , TFLOPS: 48.36236938788574, Tokens per sec: 12027.47829892755, Loss: 2.4845030307769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1040 , TFLOPS: 50.13254674501675, Tokens per sec: 12467.712514447174, Loss: 2.4909701347351074 +------------------------------------------------------------------ + eval ppl=12.363626480102539, eval loss=2.514758825302124 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/optimizer.pt +[rank0]:[2024-08-13 05:09:26,261] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006305172006250359, 'preprocessing_with_comm': 0.0015547200018772855, 'state_converting': 0.42889984299836215, : 0.4381236609915504}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040, took 3.41s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000940 +------------------------------------------------------------------ +iteration: 1041 , TFLOPS: 63.19162135353528, Tokens per sec: 15715.438762065143, Loss: 2.530165195465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1042 , TFLOPS: 71.0593551399776, Tokens per sec: 17672.104628024095, Loss: 2.4655134677886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1043 , TFLOPS: 70.55897083007335, Tokens per sec: 17547.66156403301, Loss: 2.480891466140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1044 , TFLOPS: 71.45457353608478, Tokens per sec: 17770.393457597173, Loss: 2.46209716796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1045 , TFLOPS: 71.40768128237235, Tokens per sec: 17758.73158968101, Loss: 2.505699396133423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1046 , TFLOPS: 71.32508874638549, Tokens per sec: 17738.191240917946, Loss: 2.4712610244750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1047 , TFLOPS: 71.15069309644213, Tokens per sec: 17694.819918923815, Loss: 2.498621702194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1048 , TFLOPS: 71.34073323235545, Tokens per sec: 17742.0819459823, Loss: 2.4762377738952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1049 , TFLOPS: 57.190429249836406, Tokens per sec: 14222.972435280633, Loss: 2.48111629486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1050 , TFLOPS: 48.34816683212762, Tokens per sec: 12023.946194662762, Loss: 2.4685628414154053 +------------------------------------------------------------------ + eval ppl=10.099555015563965, eval loss=2.3124914169311523 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/model.pt +[rank0]:[2024-08-13 05:22:53,547] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006300262990407646, 'preprocessing_with_comm': 0.0016832609981065616, 'state_converting': 0.437411830003839, : 0.4467268580046948}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050, took 3.39s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000950 +------------------------------------------------------------------ +iteration: 1051 , TFLOPS: 50.95158659812813, Tokens per sec: 12671.40361113942, Loss: 2.478546380996704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1052 , TFLOPS: 66.61980032055978, Tokens per sec: 16568.009648326464, Loss: 2.469444990158081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1053 , TFLOPS: 71.19662138961728, Tokens per sec: 17706.242054697195, Loss: 2.4885268211364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1054 , TFLOPS: 71.27290958361475, Tokens per sec: 17725.214545279905, Loss: 2.486264228820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1055 , TFLOPS: 70.41713559770555, Tokens per sec: 17512.38785430955, Loss: 2.474672317504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1056 , TFLOPS: 70.80122257679987, Tokens per sec: 17607.908356394684, Loss: 2.4873194694519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1057 , TFLOPS: 49.73984037159128, Tokens per sec: 12370.04841229488, Loss: 2.4925413131713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1058 , TFLOPS: 48.741106570754326, Tokens per sec: 12121.668333568201, Loss: 2.5047314167022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1059 , TFLOPS: 50.93738297285122, Tokens per sec: 12667.871240890683, Loss: 2.4920740127563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1060 , TFLOPS: 52.284575451483995, Tokens per sec: 13002.911242162681, Loss: 2.4827539920806885 +------------------------------------------------------------------ + eval ppl=10.604275703430176, eval loss=2.361257314682007 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/optimizer.pt +[rank0]:[2024-08-13 05:37:56,110] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006316209997748956, 'preprocessing_with_comm': 0.0015641360078006983, 'state_converting': 0.44004986601066776, : 0.4493177989934338}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060, took 3.42s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000960 +------------------------------------------------------------------ +iteration: 1061 , TFLOPS: 59.46366339192145, Tokens per sec: 14788.314345909968, Loss: 2.500140905380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1062 , TFLOPS: 71.16745401850528, Tokens per sec: 17698.98827603568, Loss: 2.445186138153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1063 , TFLOPS: 71.28647074339936, Tokens per sec: 17728.58713758839, Loss: 2.4843077659606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1064 , TFLOPS: 71.44707894669153, Tokens per sec: 17768.529590867158, Loss: 2.486433506011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1065 , TFLOPS: 71.00318686621058, Tokens per sec: 17658.135860522183, Loss: 2.4911229610443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1066 , TFLOPS: 62.86542811571263, Tokens per sec: 15634.316142581749, Loss: 2.470139503479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1067 , TFLOPS: 48.18398885010688, Tokens per sec: 11983.115955348418, Loss: 2.512752056121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1068 , TFLOPS: 49.481225170928084, Tokens per sec: 12305.732111147556, Loss: 2.476386070251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1069 , TFLOPS: 51.84412271899614, Tokens per sec: 12893.372860384638, Loss: 2.4787886142730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1070 , TFLOPS: 53.25134073303551, Tokens per sec: 13243.340910749806, Loss: 2.495248556137085 +------------------------------------------------------------------ + eval ppl=10.7848482131958, eval loss=2.3781421184539795 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/model.pt +[rank0]:[2024-08-13 05:52:45,459] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006330256990622729, 'preprocessing_with_comm': 0.0011056270013796166, 'state_converting': 0.43454217600810807, : 0.4433594550064299}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001070, took 3.42s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000970 +------------------------------------------------------------------ +iteration: 1071 , TFLOPS: 65.88778599424676, Tokens per sec: 16385.96136294119, Loss: 2.4689412117004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1072 , TFLOPS: 70.96003238566007, Tokens per sec: 17647.403558013182, Loss: 2.4729397296905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1073 , TFLOPS: 71.2793007352159, Tokens per sec: 17726.803992574583, Loss: 2.491272449493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1074 , TFLOPS: 71.26754164923362, Tokens per sec: 17723.879566967295, Loss: 2.50597882270813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1075 , TFLOPS: 71.36019924149413, Tokens per sec: 17746.92304465975, Loss: 2.5100698471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1076 , TFLOPS: 71.12128382697209, Tokens per sec: 17687.505981355927, Loss: 2.500380277633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1077 , TFLOPS: 71.23874639373491, Tokens per sec: 17716.718331589327, Loss: 2.4941654205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1078 , TFLOPS: 71.15283055927439, Tokens per sec: 17695.351495191793, Loss: 2.4732985496520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1079 , TFLOPS: 71.23134351240438, Tokens per sec: 17714.877272193782, Loss: 2.4796059131622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1080 , TFLOPS: 71.35222947442885, Tokens per sec: 17744.941003629952, Loss: 2.440629720687866 +------------------------------------------------------------------ + eval ppl=10.419781684875488, eval loss=2.3437061309814453 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/optimizer.pt +[rank0]:[2024-08-13 06:05:15,744] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006376416000421159, 'preprocessing_with_comm': 0.0006492390093626454, 'state_converting': 0.45341919700149447, : 0.4618298030109145}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001080, took 3.47s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000980 +------------------------------------------------------------------ +iteration: 1081 , TFLOPS: 71.32680956381945, Tokens per sec: 17738.619199568555, Loss: 2.496708393096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1082 , TFLOPS: 71.18135207348166, Tokens per sec: 17702.44465248583, Loss: 2.464858293533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1083 , TFLOPS: 71.23130899487983, Tokens per sec: 17714.86868785884, Loss: 2.447918176651001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1084 , TFLOPS: 71.40540094733994, Tokens per sec: 17758.164481814634, Loss: 2.4847593307495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1085 , TFLOPS: 70.77846168524893, Tokens per sec: 17602.24783701447, Loss: 2.4876949787139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1086 , TFLOPS: 71.37771766945382, Tokens per sec: 17751.279789682445, Loss: 2.468573570251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1087 , TFLOPS: 71.32351903237121, Tokens per sec: 17737.800860928688, Loss: 2.472226619720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1088 , TFLOPS: 71.15619051340984, Tokens per sec: 17696.187098905197, Loss: 2.492342233657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1089 , TFLOPS: 71.29750292578257, Tokens per sec: 17731.33078592248, Loss: 2.4947876930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1090 , TFLOPS: 71.44005591852958, Tokens per sec: 17766.782999046292, Loss: 2.4685218334198 +------------------------------------------------------------------ + eval ppl=13.62159252166748, eval loss=2.6116561889648438 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/optimizer.pt +[rank0]:[2024-08-13 06:17:39,810] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063309800025308505, 'preprocessing_with_comm': 0.0006292559992289171, 'state_converting': 0.44467174900637474, : 0.4529652729979716}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001090, took 3.60s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000990 +------------------------------------------------------------------ +iteration: 1091 , TFLOPS: 71.01699633816513, Tokens per sec: 17661.570206819833, Loss: 2.49204158782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1092 , TFLOPS: 71.12955461220824, Tokens per sec: 17689.562884092535, Loss: 2.4447267055511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1093 , TFLOPS: 71.3230000536104, Tokens per sec: 17737.671793518173, Loss: 2.4516122341156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1094 , TFLOPS: 71.26805585456617, Tokens per sec: 17724.007447250184, Loss: 2.4753711223602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1095 , TFLOPS: 71.38595051727998, Tokens per sec: 17753.327257575787, Loss: 2.487525224685669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1096 , TFLOPS: 71.1765362353513, Tokens per sec: 17701.24697773711, Loss: 2.5031867027282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1097 , TFLOPS: 71.36946459393491, Tokens per sec: 17749.227291263545, Loss: 2.498368978500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1098 , TFLOPS: 71.09038076902937, Tokens per sec: 17679.820546099527, Loss: 2.509889841079712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1099 , TFLOPS: 71.06215047914901, Tokens per sec: 17672.799814832513, Loss: 2.4606661796569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1100 , TFLOPS: 71.22672403892037, Tokens per sec: 17713.72843234609, Loss: 2.4614665508270264 +------------------------------------------------------------------ + eval ppl=10.87258243560791, eval loss=2.386244297027588 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/optimizer.pt +[rank0]:[2024-08-13 06:30:04,175] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006297986998106353, 'preprocessing_with_comm': 0.000631586997769773, 'state_converting': 0.4221911770000588, : 0.43050746800145134}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001100, took 3.35s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001000 +------------------------------------------------------------------ +iteration: 1101 , TFLOPS: 71.3011946320414, Tokens per sec: 17732.24889472228, Loss: 2.4541900157928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1102 , TFLOPS: 71.3264647071525, Tokens per sec: 17738.533435448007, Loss: 2.490520477294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1103 , TFLOPS: 71.17345997554177, Tokens per sec: 17700.48192737731, Loss: 2.4726152420043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1104 , TFLOPS: 71.40270669562854, Tokens per sec: 17757.494434949665, Loss: 2.459009885787964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1105 , TFLOPS: 70.8435487352826, Tokens per sec: 17618.434659366834, Loss: 2.476517915725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1106 , TFLOPS: 71.40500035851437, Tokens per sec: 17758.06485738623, Loss: 2.4866888523101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1107 , TFLOPS: 71.40317240956298, Tokens per sec: 17757.610255665462, Loss: 2.479566812515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1108 , TFLOPS: 71.20225555264425, Tokens per sec: 17707.643242736518, Loss: 2.4845035076141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1109 , TFLOPS: 71.29673776171911, Tokens per sec: 17731.14049346401, Loss: 2.444629669189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1110 , TFLOPS: 71.4059631501597, Tokens per sec: 17758.304298831456, Loss: 2.4601945877075195 +------------------------------------------------------------------ + eval ppl=14.242786407470703, eval loss=2.656250476837158 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/optimizer.pt +[rank0]:[2024-08-13 06:42:27,802] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006299267988651991, 'preprocessing_with_comm': 0.0006260819936869666, 'state_converting': 0.43465598300099373, : 0.44291795800381806}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001110, took 3.43s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001010 +------------------------------------------------------------------ +iteration: 1111 , TFLOPS: 71.11900784225816, Tokens per sec: 17686.939955391896, Loss: 2.4738616943359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1112 , TFLOPS: 70.9857509365713, Tokens per sec: 17653.79963241731, Loss: 2.515223741531372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1113 , TFLOPS: 71.16760331672965, Tokens per sec: 17699.02540575402, Loss: 2.4570817947387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1114 , TFLOPS: 71.33617211637403, Tokens per sec: 17740.947619352326, Loss: 2.4658660888671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1115 , TFLOPS: 71.32022729637517, Tokens per sec: 17736.98222272383, Loss: 2.4610064029693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1116 , TFLOPS: 71.3360256440688, Tokens per sec: 17740.911192425876, Loss: 2.476661443710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1117 , TFLOPS: 71.16725348777103, Tokens per sec: 17698.93840504952, Loss: 2.4794304370880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1118 , TFLOPS: 70.79626041350143, Tokens per sec: 17606.674291311785, Loss: 2.442260503768921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1119 , TFLOPS: 71.1988098334854, Tokens per sec: 17706.78630969267, Loss: 2.4577503204345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1120 , TFLOPS: 71.0761302704688, Tokens per sec: 17676.276518700633, Loss: 2.5057506561279297 +------------------------------------------------------------------ + eval ppl=10.758519172668457, eval loss=2.3756978511810303 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/optimizer.pt +[rank0]:[2024-08-13 06:54:52,860] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006325291004031897, 'preprocessing_with_comm': 0.0006334399949992076, 'state_converting': 0.4480844200006686, : 0.4564305929961847}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001120, took 3.65s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001020 +------------------------------------------------------------------ +iteration: 1121 , TFLOPS: 71.18499940487366, Tokens per sec: 17703.35172547918, Loss: 2.443347930908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1122 , TFLOPS: 71.21029278073117, Tokens per sec: 17709.642060983482, Loss: 2.4564871788024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1123 , TFLOPS: 71.15908780164156, Tokens per sec: 17696.90763993273, Loss: 2.4694817066192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1124 , TFLOPS: 71.28083277941202, Tokens per sec: 17727.18500426932, Loss: 2.5032215118408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1125 , TFLOPS: 70.80020987853919, Tokens per sec: 17607.65650342495, Loss: 2.474421262741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1126 , TFLOPS: 71.2143631337113, Tokens per sec: 17710.654337320582, Loss: 2.494691848754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1127 , TFLOPS: 71.16544915787522, Tokens per sec: 17698.489677269205, Loss: 2.442568302154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1128 , TFLOPS: 71.16465834420093, Tokens per sec: 17698.29300588138, Loss: 2.4756505489349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1129 , TFLOPS: 70.99794276179504, Tokens per sec: 17656.83167809991, Loss: 2.4691827297210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1130 , TFLOPS: 71.35646218437563, Tokens per sec: 17745.993657328996, Loss: 2.494615316390991 +------------------------------------------------------------------ + eval ppl=11.094965934753418, eval loss=2.40649151802063 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/optimizer.pt +[rank0]:[2024-08-13 07:07:17,932] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006308366995654069, 'preprocessing_with_comm': 0.0006225719989743084, 'state_converting': 0.4482559050084092, : 0.45653351199871395}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001130, took 3.59s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001030 +------------------------------------------------------------------ +iteration: 1131 , TFLOPS: 70.72717373055072, Tokens per sec: 17589.492780346718, Loss: 2.476990222930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1132 , TFLOPS: 70.73184633707746, Tokens per sec: 17590.654834115197, Loss: 2.452390193939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1133 , TFLOPS: 71.24621604842525, Tokens per sec: 17718.575997183998, Loss: 2.4930529594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1134 , TFLOPS: 71.1844017228176, Tokens per sec: 17703.203084954534, Loss: 2.477077007293701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1135 , TFLOPS: 71.06990448576445, Tokens per sec: 17674.728197322398, Loss: 2.4361133575439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1136 , TFLOPS: 71.29131756376343, Tokens per sec: 17729.792517462978, Loss: 2.4678397178649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1137 , TFLOPS: 71.17848795249198, Tokens per sec: 17701.73235998476, Loss: 2.467200756072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1138 , TFLOPS: 70.4511804496475, Tokens per sec: 17520.854637949487, Loss: 2.462362051010132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1139 , TFLOPS: 71.22600192588835, Tokens per sec: 17713.54884646286, Loss: 2.454740047454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1140 , TFLOPS: 71.16294579238657, Tokens per sec: 17697.867102848842, Loss: 2.440718173980713 +------------------------------------------------------------------ + eval ppl=14.097245216369629, eval loss=2.645979404449463 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/optimizer.pt +[rank0]:[2024-08-13 07:19:44,317] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063564860029146075, 'preprocessing_with_comm': 0.0006386360037140548, 'state_converting': 0.4439645560050849, : 0.4523756420094287}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001140, took 3.43s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001040 +------------------------------------------------------------------ +iteration: 1141 , TFLOPS: 71.18507220671113, Tokens per sec: 17703.36983093041, Loss: 2.491312265396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1142 , TFLOPS: 71.32901231795331, Tokens per sec: 17739.167013455222, Loss: 2.464977502822876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1143 , TFLOPS: 71.35290036727379, Tokens per sec: 17745.10785130999, Loss: 2.4820053577423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1144 , TFLOPS: 71.30152236773343, Tokens per sec: 17732.330400942374, Loss: 2.469080686569214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1145 , TFLOPS: 70.41932074046466, Tokens per sec: 17512.931288335796, Loss: 2.4699528217315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1146 , TFLOPS: 71.20633031044157, Tokens per sec: 17708.656614529522, Loss: 2.503364086151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1147 , TFLOPS: 71.28212038430767, Tokens per sec: 17727.505225138066, Loss: 2.494542121887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1148 , TFLOPS: 71.22512619350219, Tokens per sec: 17713.331056218012, Loss: 2.4812211990356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1149 , TFLOPS: 71.16086951678625, Tokens per sec: 17697.350743538052, Loss: 2.4654581546783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1150 , TFLOPS: 71.36452761880689, Tokens per sec: 17747.99949035209, Loss: 2.493213415145874 +------------------------------------------------------------------ + eval ppl=12.525251388549805, eval loss=2.5277466773986816 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/optimizer.pt +[rank0]:[2024-08-13 07:32:09,168] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006257941990043037, 'preprocessing_with_comm': 0.0006588569958694279, 'state_converting': 0.44384913399699144, : 0.4521027590089943}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001150, took 3.67s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001050 +------------------------------------------------------------------ +iteration: 1151 , TFLOPS: 71.02556695507785, Tokens per sec: 17663.701676188084, Loss: 2.456284523010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1152 , TFLOPS: 70.75831581668436, Tokens per sec: 17597.237660713345, Loss: 2.4795167446136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1153 , TFLOPS: 71.25410690673672, Tokens per sec: 17720.53841400316, Loss: 2.4631547927856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1154 , TFLOPS: 71.11243090978375, Tokens per sec: 17685.304305327383, Loss: 2.474699020385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1155 , TFLOPS: 71.28635929371552, Tokens per sec: 17728.55942061193, Loss: 2.5045523643493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1156 , TFLOPS: 71.37118084261017, Tokens per sec: 17749.654113686818, Loss: 2.4660840034484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1157 , TFLOPS: 71.18054942971744, Tokens per sec: 17702.245039014073, Loss: 2.474604606628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1158 , TFLOPS: 70.87374561879912, Tokens per sec: 17625.944472591185, Loss: 2.477003812789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1159 , TFLOPS: 71.10223182386264, Tokens per sec: 17682.76784389802, Loss: 2.457900285720825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1160 , TFLOPS: 70.97458889051923, Tokens per sec: 17651.02368764119, Loss: 2.470151424407959 +------------------------------------------------------------------ + eval ppl=11.470099449157715, eval loss=2.4397435188293457 +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/model.pt +[rank0]:[2024-08-13 07:44:34,752] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006300738008576445, 'preprocessing_with_comm': 0.0006512639956781641, 'state_converting': 0.4367705839977134, : 0.44508136600779835}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160, took 3.49s +Deleting checkpoint /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001060 +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 71.26158123388396, Tokens per sec: 17722.39724161438, Loss: 2.442603349685669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1162 , TFLOPS: 71.25890950718718, Tokens per sec: 17721.73279660739, Loss: 2.467410087585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1163 , TFLOPS: 71.35609785993921, Tokens per sec: 17745.903051671983, Loss: 2.448531150817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1164 , TFLOPS: 71.35379257349722, Tokens per sec: 17745.3297385154, Loss: 2.4926505088806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1165 , TFLOPS: 70.46316554596581, Tokens per sec: 17523.835271191892, Loss: 2.44860577583313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1166 , TFLOPS: 71.18810660631905, Tokens per sec: 17704.12447086837, Loss: 2.474088191986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1167 , TFLOPS: 71.54763648032535, Tokens per sec: 17793.53774429062, Loss: 2.4635202884674072 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240812_073955-ikoro1zp/files/requirements.txt b/wandb/run-20240812_073955-ikoro1zp/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240812_073955-ikoro1zp/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240812_073955-ikoro1zp/files/wandb-metadata.json b/wandb/run-20240812_073955-ikoro1zp/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..be323f6633689813602463b0f0fddf6f15e60a7b --- /dev/null +++ b/wandb/run-20240812_073955-ikoro1zp/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-11T22:39:56.314869", + "startedAt": "2024-08-11T22:39:55.672249", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--test-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-qwen2-0.5B_train_2024-08-12-07:39:43" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0429999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487823486328125 + } +} diff --git a/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json b/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8b2bea7ea56368e5598feb96384be9046f652060 --- /dev/null +++ b/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.4635202884674072, "training/perplexity": 11.746088463770842, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 1167, "optimizer/lr": 1.9945203423500063e-05, "optimizer/variance_l2": 0.0048320659825907535, "optimizer/variance_sqrt_l2": 0.5318417899390797, "optimizer/momentum_l2": 0.127020084622386, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.2829437255859375, "optimizer/variance_sqrt_l1": 4615.0, "optimizer/momentum_l1": 979.125, "optimizer/weight_l1": 6918144.0, "optimizer/variance_abs_max": 0.0029296875, "optimizer/variance_sqrt_abs_max": 0.05419921875, "optimizer/momentum_abs_max": 0.01129150390625, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 73.68068221400608, "stats/tokens_per_sec": 17793.53774429062, "stats/tokens_per_sec_per_gpu": 17793.53774429062, "stats/tflops": 71.54763648032535, "_timestamp": 1723503194.8273196, "_runtime": 87199.14199066162, "_step": 1167, "evaluation/val_loss": 2.4397435188293457, "evaluation/val_ppl": 11.470099449157715, "_wandb": {"runtime": 87227}} \ No newline at end of file diff --git a/wandb/run-20240812_073955-ikoro1zp/logs/debug-internal.log b/wandb/run-20240812_073955-ikoro1zp/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..847277fa17f0e59f456686f3ef47de0fdfd807ba --- /dev/null +++ b/wandb/run-20240812_073955-ikoro1zp/logs/debug-internal.log @@ -0,0 +1,44614 @@ +2024-08-12 07:39:55,687 INFO StreamThr :14795 [internal.py:wandb_internal():86] W&B internal server running at pid: 14795, started at: 2024-08-12 07:39:55.686006 +2024-08-12 07:39:55,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status +2024-08-12 07:39:55,690 INFO WriterThread:14795 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_073955-ikoro1zp/run-ikoro1zp.wandb +2024-08-12 07:39:55,691 DEBUG SenderThread:14795 [sender.py:send():382] send: header +2024-08-12 07:39:55,708 DEBUG SenderThread:14795 [sender.py:send():382] send: run +2024-08-12 07:39:56,198 INFO SenderThread:14795 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_073955-ikoro1zp/files +2024-08-12 07:39:56,198 INFO SenderThread:14795 [sender.py:_start_run_threads():1136] run started: ikoro1zp with start time 1723415995.685329 +2024-08-12 07:39:56,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: check_version +2024-08-12 07:39:56,203 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: check_version +2024-08-12 07:39:56,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: run_start +2024-08-12 07:39:56,301 DEBUG HandlerThread:14795 [system_info.py:__init__():27] System info init +2024-08-12 07:39:56,301 DEBUG HandlerThread:14795 [system_info.py:__init__():42] System info init done +2024-08-12 07:39:56,301 INFO HandlerThread:14795 [system_monitor.py:start():194] Starting system monitor +2024-08-12 07:39:56,301 INFO SystemMonitor:14795 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-12 07:39:56,302 INFO HandlerThread:14795 [system_monitor.py:probe():214] Collecting system info +2024-08-12 07:39:56,302 INFO SystemMonitor:14795 [interfaces.py:start():190] Started cpu monitoring +2024-08-12 07:39:56,302 INFO SystemMonitor:14795 [interfaces.py:start():190] Started disk monitoring +2024-08-12 07:39:56,303 INFO SystemMonitor:14795 [interfaces.py:start():190] Started gpu monitoring +2024-08-12 07:39:56,305 INFO SystemMonitor:14795 [interfaces.py:start():190] Started memory monitoring +2024-08-12 07:39:56,305 INFO SystemMonitor:14795 [interfaces.py:start():190] Started network monitoring +2024-08-12 07:39:56,314 DEBUG HandlerThread:14795 [system_info.py:probe():151] Probing system +2024-08-12 07:39:56,317 DEBUG HandlerThread:14795 [system_info.py:_probe_git():136] Probing git +2024-08-12 07:39:56,329 DEBUG HandlerThread:14795 [system_info.py:_probe_git():144] Probing git done +2024-08-12 07:39:56,329 DEBUG HandlerThread:14795 [system_info.py:probe():199] Probing system done +2024-08-12 07:39:56,329 DEBUG HandlerThread:14795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T22:39:56.314869', 'startedAt': '2024-08-11T22:39:55.672249', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--valid-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--test-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-qwen2-0.5B', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-qwen2-0.5B_train_2024-08-12-07:39:43'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}} +2024-08-12 07:39:56,329 INFO HandlerThread:14795 [system_monitor.py:probe():224] Finished collecting system info +2024-08-12 07:39:56,330 INFO HandlerThread:14795 [system_monitor.py:probe():227] Publishing system info +2024-08-12 07:39:56,331 INFO HandlerThread:14795 [system_monitor.py:probe():229] Finished publishing system info +2024-08-12 07:39:56,337 DEBUG SenderThread:14795 [sender.py:send():382] send: files +2024-08-12 07:39:56,337 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-12 07:39:56,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: python_packages +2024-08-12 07:39:56,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:39:56,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: python_packages +2024-08-12 07:39:56,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:39:56,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:39:56,628 DEBUG SenderThread:14795 [sender.py:send():382] send: telemetry +2024-08-12 07:39:57,005 INFO wandb-upload_0:14795 [upload_job.py:push():131] Uploaded file /tmp/tmptrawtavywandb/54k7mz1e-wandb-metadata.json +2024-08-12 07:39:57,200 INFO Thread-12 :14795 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073955-ikoro1zp/files/requirements.txt +2024-08-12 07:39:57,200 INFO Thread-12 :14795 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:39:57,200 INFO Thread-12 :14795 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-metadata.json +2024-08-12 07:39:59,200 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:40:01,201 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:40:01,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:02,086 DEBUG SenderThread:14795 [sender.py:send():382] send: config +2024-08-12 07:40:02,087 DEBUG SenderThread:14795 [sender.py:send():382] send: config +2024-08-12 07:40:03,202 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:40:07,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:11,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:40:11,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:40:11,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:40:12,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:17,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:22,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:26,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:40:26,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:40:26,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:40:28,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:29,220 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/config.yaml +2024-08-12 07:40:33,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:38,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:41,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:40:41,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:40:41,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:40:44,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:49,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:54,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:40:56,306 DEBUG SystemMonitor:14795 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-12 07:40:56,307 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:40:56,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:40:56,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:40:56,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:41:00,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:05,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:10,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:11,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:41:11,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:41:11,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:41:15,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:17,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:41:19,251 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:41:21,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:26,308 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:41:26,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:41:26,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:41:26,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:41:26,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:31,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:36,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:41,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:41:41,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:41:41,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:41:42,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:47,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:52,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:41:56,309 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:41:56,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:41:56,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:41:56,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:41:58,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:03,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:08,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:11,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:42:11,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:42:11,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:42:13,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:18,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:23,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:26,310 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:42:26,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:42:26,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:42:26,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:42:29,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:32,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:42:32,056 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:42:32,056 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:42:32,058 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:42:32,297 INFO Thread-12 :14795 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:42:33,297 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:42:35,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:40,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:41,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:42:41,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:42:41,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:42:45,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:50,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:55,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:42:56,311 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:42:56,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:42:56,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:42:56,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:43:01,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:06,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:11,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:43:11,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:43:11,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:43:11,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:16,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:21,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:26,312 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:43:26,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:43:26,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:43:26,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:43:27,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:32,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:37,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:41,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:43:41,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:43:41,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:43:42,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:46,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:43:46,729 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:43:46,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:43:46,731 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:43:47,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:43:47,346 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:43:47,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:52,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:43:56,313 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:43:56,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:43:56,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:43:56,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:43:58,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:03,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:08,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:11,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:44:11,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:44:11,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:44:13,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:18,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:23,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:26,314 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:44:26,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:44:26,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:44:26,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:44:29,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:34,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:39,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:41,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:44:41,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:44:41,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:44:44,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:49,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:54,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:44:56,315 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:44:56,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:44:56,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:44:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:45:00,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:01,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:45:01,445 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:45:01,446 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:45:01,447 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:45:02,394 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:45:03,395 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:45:06,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:11,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:45:11,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:45:11,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:45:11,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:16,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:21,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:26,316 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:45:26,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:45:26,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:45:26,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:45:27,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:32,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:37,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:41,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:45:41,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:45:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:45:42,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:47,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:52,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:45:56,317 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:45:56,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:45:56,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:45:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:45:57,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:02,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:07,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:11,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:46:11,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:46:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:46:13,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:16,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:46:16,375 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:46:16,376 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:46:16,377 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:46:16,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:46:17,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:46:19,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:24,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:26,318 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:46:26,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:46:26,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:46:26,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:46:29,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:34,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:39,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:41,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:46:41,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:46:41,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:46:45,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:50,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:55,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:46:56,320 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:46:56,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:46:56,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:46:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:47:01,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:06,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:11,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:47:11,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:47:11,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:47:11,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:16,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:21,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:26,321 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:47:26,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:47:26,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:47:26,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:47:26,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:31,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:47:31,070 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:47:31,071 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:47:31,072 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:47:31,489 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:47:31,489 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:47:32,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:37,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:41,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:47:41,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:47:41,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:47:42,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:47,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:52,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:47:56,322 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:47:56,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:47:56,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:47:56,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:47:57,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:02,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:07,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:11,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:48:11,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:48:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:48:12,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:17,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:22,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:26,323 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:48:26,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:48:26,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:48:26,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:48:28,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:33,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:38,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:41,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:48:41,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:48:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:48:43,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:45,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:48:45,997 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:48:45,997 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:48:45,998 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:48:46,537 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:48:47,538 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:48:49,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:54,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:48:56,323 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:48:56,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:48:56,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:48:56,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:48:59,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:04,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:09,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:11,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:49:11,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:49:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:49:14,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:19,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:24,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:26,325 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:49:26,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:49:26,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:49:26,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:49:29,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:34,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:39,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:41,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:49:41,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:49:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:49:45,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:50,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:55,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:49:56,325 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:49:56,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:49:56,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:49:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:50:00,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:01,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:50:01,082 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:50:01,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:50:01,084 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:50:01,584 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:50:01,584 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:50:06,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:11,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:11,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:50:11,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:50:11,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:50:16,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:21,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:26,326 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:50:26,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:50:26,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:50:26,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:50:26,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:31,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:36,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:41,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:50:41,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:50:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:50:41,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:46,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:51,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:50:56,327 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:50:56,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:50:56,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:50:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:50:57,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:02,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:07,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:11,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:51:11,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:51:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:51:13,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:15,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:51:15,627 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:51:15,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:51:15,628 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:51:15,631 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:51:16,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:51:17,632 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:51:18,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:19,633 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:51:21,635 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:51:24,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:26,328 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:51:26,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:51:26,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:51:26,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:51:29,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:34,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:39,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:41,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:51:41,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:51:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:51:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:50,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:55,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:51:56,330 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:51:56,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:51:56,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:51:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:52:01,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:06,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:11,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:52:11,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:52:11,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:52:11,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:16,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:21,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:26,332 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:52:26,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:52:26,351 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:52:26,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:52:27,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:32,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:35,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:52:35,484 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:52:35,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:52:35,485 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:52:35,682 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:52:37,683 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:52:38,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:41,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:52:41,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:52:41,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:52:43,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:48,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:53,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:52:56,333 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:52:56,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:52:56,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:52:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:52:59,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:04,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:09,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:11,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:53:11,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:53:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:53:15,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:20,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:25,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:26,335 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:53:26,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:53:26,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:53:26,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:53:30,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:35,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:40,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:41,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:53:41,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:53:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:53:46,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:50,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:53:50,361 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:53:50,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:53:50,362 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:53:50,730 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:53:51,731 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:53:52,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:53:56,335 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:53:56,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:53:56,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:53:56,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:53:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:02,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:07,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:11,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:54:11,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:54:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:54:13,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:18,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:23,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:26,339 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:54:26,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:54:26,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:54:26,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:54:28,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:33,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:38,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:41,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:54:41,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:54:41,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:54:44,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:49,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:54,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:54:56,339 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:54:56,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:54:56,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:54:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:54:59,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:04,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:05,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:55:05,335 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:55:05,335 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:55:05,336 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:55:05,779 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:55:05,779 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:55:10,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:11,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:55:11,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:55:11,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:55:15,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:20,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:25,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:26,340 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:55:26,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:55:26,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:55:26,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:55:30,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:35,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:40,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:41,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:55:41,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:55:41,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:55:46,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:51,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:55:56,341 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:55:56,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:55:56,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:55:56,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:55:56,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:01,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:06,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:11,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:56:11,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:56:11,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:56:11,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:16,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:20,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:56:20,440 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:56:20,441 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:56:20,442 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:56:20,827 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:56:21,827 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:56:22,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:26,343 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:56:26,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:56:26,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:56:26,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:56:27,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:32,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:37,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:41,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:56:41,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:56:41,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:56:42,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:47,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:52,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:56:56,344 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:56:56,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:56:56,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:56:56,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:56:58,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:03,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:08,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:11,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:57:11,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:57:11,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:57:13,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:18,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:23,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:26,345 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:57:26,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:57:26,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:57:26,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:57:29,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:34,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:35,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:57:35,283 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:57:35,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:57:35,284 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:57:35,875 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:57:35,876 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:57:40,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:41,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:57:41,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:57:41,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:57:45,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:50,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:55,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:57:56,346 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:57:56,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:57:56,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:57:56,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:58:01,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:06,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:11,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:58:11,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:58:11,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:58:12,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:17,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:22,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:26,347 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:58:26,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:58:26,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:58:26,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:58:27,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:32,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:37,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:41,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:58:41,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:58:41,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:58:42,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:47,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:49,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:58:49,828 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 07:58:49,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:58:49,830 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:58:49,924 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 07:58:49,924 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 07:58:52,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:58:56,348 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:58:56,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:58:56,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:58:56,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:58:58,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:03,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:08,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:11,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:59:11,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:59:11,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:59:13,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:18,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:23,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:26,349 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:59:26,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:59:26,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:59:26,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:59:28,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:33,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:38,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:41,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:59:41,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:59:41,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:59:44,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:49,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:54,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:59:56,350 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 07:59:56,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:59:56,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:59:56,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:00:00,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:03,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:00:03,914 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:00:03,914 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:00:03,916 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:00:03,968 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:00:03,968 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:00:05,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:10,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:11,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:00:11,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:00:11,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:00:16,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:21,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:26,351 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:00:26,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:00:26,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:00:26,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:00:26,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:31,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:36,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:41,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:00:41,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:00:41,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:00:42,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:47,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:52,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:00:56,352 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:00:56,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:00:56,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:00:56,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:00:57,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:02,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:07,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:11,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:01:11,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:01:11,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:01:12,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:17,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:18,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:01:18,073 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:01:18,073 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:01:18,074 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:01:19,013 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:01:21,014 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:01:23,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:26,354 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:01:26,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:01:26,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:01:26,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:01:28,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:33,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:38,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:41,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:01:41,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:01:41,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:01:44,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:49,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:54,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:01:56,353 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:01:56,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:01:56,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:01:56,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:01:59,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:04,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:09,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:11,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:02:11,356 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:02:11,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:02:15,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:20,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:25,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:26,355 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:02:26,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:02:26,356 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:02:26,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:02:30,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:31,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:02:31,909 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:02:31,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:02:31,910 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:02:32,056 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:02:33,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:02:35,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:40,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:41,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:02:41,356 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:02:41,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:02:46,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:51,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:02:56,355 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:02:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:02:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:02:56,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:02:57,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:02,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:07,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:11,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:03:11,356 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:03:11,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:03:12,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:17,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:22,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:26,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:03:26,357 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:03:26,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:03:26,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:03:27,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:32,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:37,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:41,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:03:41,356 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:03:41,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:03:43,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:45,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:03:45,965 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:03:45,965 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:03:45,967 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:03:46,101 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:03:46,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:03:47,101 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:03:49,103 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:03:49,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:51,104 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:03:54,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:03:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:03:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:03:56,357 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:03:56,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:04:00,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:05,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:10,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:11,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:04:11,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:04:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:04:15,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:20,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:25,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:26,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:04:26,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:04:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:04:26,637 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:04:30,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:35,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:40,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:41,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:04:41,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:04:41,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:04:46,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:51,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:04:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:04:56,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:04:56,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:04:56,576 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:04:57,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:02,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:03,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:05:03,885 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:05:03,885 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:05:03,887 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:05:04,149 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:05:05,150 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:05:07,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:11,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:05:11,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:05:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:05:13,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:18,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:23,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:26,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:05:26,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:05:26,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:05:26,528 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:05:29,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:34,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:39,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:41,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:05:41,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:05:41,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:05:44,592 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:49,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:54,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:05:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:05:56,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:05:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:05:56,610 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:05:59,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:04,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:09,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:11,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:06:11,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:06:11,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:06:15,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:17,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:06:17,739 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:06:17,739 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:06:17,741 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:06:18,198 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:06:19,198 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:06:20,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:25,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:26,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:06:26,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:06:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:06:26,573 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:06:31,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:36,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:41,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:06:41,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:06:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:06:41,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:46,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:51,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:06:56,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:06:56,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:06:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:06:56,555 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:06:57,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:02,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:07,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:11,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:07:11,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:07:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:07:12,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:17,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:22,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:26,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:07:26,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:07:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:07:26,623 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:07:27,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:31,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:07:31,727 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:07:31,727 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:07:31,729 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:07:32,247 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:07:32,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:33,248 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:07:37,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:41,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:07:41,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:07:41,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:07:43,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:48,592 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:53,592 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:07:56,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:07:56,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:07:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:07:56,553 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:07:59,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:04,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:09,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:08:11,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:08:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:08:14,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:19,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:24,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:26,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:08:26,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:08:26,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:08:26,537 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:08:30,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:35,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:40,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:41,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:08:41,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:08:41,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:08:45,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:08:45,816 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:08:45,816 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:08:45,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:45,817 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:08:46,295 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:08:47,295 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:08:50,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:55,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:08:56,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:08:56,358 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:08:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:08:56,544 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:09:01,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:06,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:09:11,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:09:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:09:11,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:16,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:21,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:26,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:09:26,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:09:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:09:26,544 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:09:27,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:32,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:37,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:41,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:09:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:09:41,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:09:43,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:48,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:53,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:56,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:09:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:09:56,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:09:56,595 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:09:58,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:09:59,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:09:59,810 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:09:59,810 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:09:59,811 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:10:00,364 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:10:01,365 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:10:03,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:08,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:10:11,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:10:11,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:10:14,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:19,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:24,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:10:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:10:26,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:10:26,536 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:10:30,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:35,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:40,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:10:41,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:10:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:10:45,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:50,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:55,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:10:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:10:56,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:10:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:10:56,568 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:11:01,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:06,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:11,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:11:11,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:11:11,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:11:12,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:13,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:11:13,855 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:11:13,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:11:13,857 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:11:14,412 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:11:15,412 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:11:17,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:22,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:11:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:11:26,359 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:11:26,622 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:11:28,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:33,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:38,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:11:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:11:41,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:11:44,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:49,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:54,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:11:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:11:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:11:56,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:11:56,606 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:12:00,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:05,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:10,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:11,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:12:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:12:11,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:12:16,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:21,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:26,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:12:26,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:12:26,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:12:26,562 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:12:27,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:27,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:12:27,609 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:12:27,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:12:27,611 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:12:28,461 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:12:29,462 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:12:32,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:37,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:41,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:12:41,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:12:41,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:12:42,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:47,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:52,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:12:56,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:12:56,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:12:56,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:12:56,571 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:12:58,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:03,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:08,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:13:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:13:11,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:13:14,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:19,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:24,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:26,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:13:26,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:13:26,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:13:26,577 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:13:29,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:34,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:39,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:41,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:13:41,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:13:41,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:13:41,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:13:41,637 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:13:41,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:13:41,639 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:13:42,511 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:13:43,512 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:13:44,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:49,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:54,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:13:56,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:13:56,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:13:56,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:13:56,584 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:14:00,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:05,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:10,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:14:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:14:11,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:14:15,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:20,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:25,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:26,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:14:26,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:14:26,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:14:26,609 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:14:30,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:35,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:40,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:41,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:14:41,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:14:41,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:14:45,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:50,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:55,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:14:55,341 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:14:55,341 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:14:55,343 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:14:55,560 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:14:56,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:14:56,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:14:56,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:14:56,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:14:56,583 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:14:57,561 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:15:01,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:06,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:11,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:15:11,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:15:11,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:15:11,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:16,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:21,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:26,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:15:26,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:15:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:15:26,581 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:15:27,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:32,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:37,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:41,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:15:41,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:15:41,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:15:42,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:47,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:52,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:15:56,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:15:56,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:15:56,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:15:56,558 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:15:58,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:03,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:08,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:09,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:16:09,215 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:16:09,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:16:09,217 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:16:09,608 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:16:09,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:16:11,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:16:11,361 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:16:11,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:16:11,610 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:16:13,611 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:16:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:15,612 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:16:19,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:24,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:26,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:16:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:16:26,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:16:26,636 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:16:29,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:34,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:39,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:41,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:16:41,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:16:41,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:16:45,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:50,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:55,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:16:56,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:16:56,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:16:56,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:16:56,588 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:17:01,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:06,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:11,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:17:11,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:17:11,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:17:11,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:16,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:21,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:17:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:17:26,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:17:26,604 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:17:27,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:17:27,257 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:17:27,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:17:27,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:27,259 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:17:27,660 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:17:29,661 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:17:32,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:37,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:41,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:17:41,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:17:41,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:17:42,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:47,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:52,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:17:56,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:17:56,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:17:56,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:17:56,551 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:17:58,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:03,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:08,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:11,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:18:11,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:18:11,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:18:13,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:18,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:23,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:18:26,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:18:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:18:26,624 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:18:29,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:34,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:39,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:40,975 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:18:40,977 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:18:40,978 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:18:40,979 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:18:41,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:18:41,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:18:41,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:18:41,703 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:18:41,703 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:18:44,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:49,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:54,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:18:56,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:18:56,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:18:56,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:18:56,552 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:19:00,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:05,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:10,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:11,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:19:11,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:19:11,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:19:15,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:20,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:25,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:19:26,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:19:26,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:19:26,589 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:19:30,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:35,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:40,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:41,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:19:41,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:19:41,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:19:45,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:50,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:54,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:19:54,757 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:19:54,758 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:19:54,759 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:19:55,746 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:19:55,746 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:19:55,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:19:56,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:19:56,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:19:56,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:19:56,577 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:20:01,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:06,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:11,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:20:11,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:20:11,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:20:11,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:16,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:21,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:26,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:20:26,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:20:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:20:26,630 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:20:26,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:31,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:36,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:41,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:20:41,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:20:41,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:20:42,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:47,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:52,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:20:56,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:20:56,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:20:56,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:20:56,571 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:20:58,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:03,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:08,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:21:08,538 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:21:08,538 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:21:08,540 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:21:08,788 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:21:09,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:09,789 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:21:11,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:21:11,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:21:11,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:21:14,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:19,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:24,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:26,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:21:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:21:26,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:21:26,531 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:21:30,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:35,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:40,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:41,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:21:41,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:21:41,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:21:45,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:50,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:55,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:21:56,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:21:56,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:21:56,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:21:56,613 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:22:00,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:05,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:10,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:11,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:22:11,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:22:11,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:22:16,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:21,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:22,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:22:22,550 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:22:22,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:22:22,552 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:22:22,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:22:23,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:22:26,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:22:26,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:22:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:22:26,605 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:22:26,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:31,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:36,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:41,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:22:41,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:22:41,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:22:42,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:47,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:52,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:22:56,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:22:56,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:22:56,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:22:56,627 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:22:57,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:02,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:07,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:11,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:23:11,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:23:11,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:23:13,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:18,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:23,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:23:26,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:23:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:23:26,627 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:23:28,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:33,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:36,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:23:36,495 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:23:36,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:23:36,496 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:23:36,878 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:23:37,878 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:23:39,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:41,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:23:41,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:23:41,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:23:44,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:49,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:54,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:23:56,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:23:56,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:23:56,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:23:56,565 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:24:00,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:05,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:10,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:11,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:24:11,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:24:11,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:24:16,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:21,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:26,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:24:26,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:24:26,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:24:26,612 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:24:26,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:31,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:36,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:41,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:24:41,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:24:41,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:24:42,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:47,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:50,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:24:50,433 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:24:50,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:24:50,434 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:24:50,923 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:24:51,923 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:24:53,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:24:56,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:24:56,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:24:56,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:24:56,616 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:24:58,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:03,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:08,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:11,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:25:11,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:25:11,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:25:14,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:19,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:24,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:26,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:25:26,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:25:26,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:25:26,612 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:25:29,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:34,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:39,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:41,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:25:41,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:25:41,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:25:45,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:50,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:55,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:25:56,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:25:56,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:25:56,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:25:56,625 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:26:00,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:04,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:26:04,302 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:26:04,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:26:04,304 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:26:04,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:26:05,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:26:06,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:11,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:11,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:26:11,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:26:11,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:26:16,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:21,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:26,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:26:26,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:26:26,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:26:26,627 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:26:26,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:31,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:36,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:41,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:26:41,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:26:41,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:26:41,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:46,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:51,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:26:56,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:26:56,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:26:56,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:26:56,570 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:26:57,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:02,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:07,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:11,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:27:11,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:27:11,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:27:13,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:18,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:27:18,121 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:27:18,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:27:18,124 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:27:19,009 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:27:19,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:20,010 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:27:24,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:26,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:27:26,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:27:26,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:27:26,626 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:27:29,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:34,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:39,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:41,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:27:41,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:27:41,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:27:44,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:49,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:54,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:27:56,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:27:56,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:27:56,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:27:56,630 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:28:00,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:05,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:10,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:11,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:28:11,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:28:11,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:28:16,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:21,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:26,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:28:26,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:28:26,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:28:26,579 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:28:27,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:32,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:28:32,017 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:28:32,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:28:32,020 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:28:32,053 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:28:32,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:32,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:28:34,054 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:28:36,056 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:28:37,056 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:28:37,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:38,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:28:41,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:28:41,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:28:41,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:28:43,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:48,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:53,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:28:56,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:28:56,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:28:56,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:28:56,585 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:28:59,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:04,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:09,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:11,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:29:11,366 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:29:11,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:29:15,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:20,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:25,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:26,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:29:26,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:29:26,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:29:26,580 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:29:31,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:36,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:41,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:29:41,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:29:41,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:29:41,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:46,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:49,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:29:49,973 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:29:49,974 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:29:49,975 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:29:50,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:29:51,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:29:52,100 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:29:56,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:29:56,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:29:56,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:29:56,545 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:29:57,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:02,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:07,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:11,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:30:11,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:30:11,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:30:12,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:17,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:22,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:26,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:30:26,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:30:26,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:30:26,578 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:30:27,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:32,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:37,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:41,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:30:41,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:30:41,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:30:43,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:48,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:53,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:30:56,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:30:56,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:30:56,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:30:56,632 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:30:58,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:03,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:03,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:31:03,897 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:31:03,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:31:03,899 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:31:04,143 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:31:06,144 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:31:08,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:11,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:31:11,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:31:11,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:31:14,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:19,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:24,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:26,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:31:26,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:31:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:31:26,561 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:31:29,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:34,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:39,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:41,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:31:41,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:31:41,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:31:45,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:50,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:55,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:31:56,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:31:56,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:31:56,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:31:56,636 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:32:00,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:05,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:10,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:11,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:32:11,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:32:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:32:16,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:17,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:32:17,744 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:32:17,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:32:17,746 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:32:18,186 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:32:18,186 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:32:21,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:26,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:32:26,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:32:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:32:26,582 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:32:27,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:32,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:37,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:41,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:32:41,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:32:41,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:32:42,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:47,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:52,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:32:56,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:32:56,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:32:56,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:32:56,618 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:32:58,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:03,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:08,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:11,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:33:11,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:33:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:33:13,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:18,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:23,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:26,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:33:26,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:33:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:33:26,589 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:33:29,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:31,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:33:31,702 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:33:31,702 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:33:31,704 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:33:32,230 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:33:32,230 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:33:34,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:39,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:41,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:33:41,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:33:41,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:33:45,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:50,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:55,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:33:56,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:33:56,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:33:56,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:33:56,593 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:34:01,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:06,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:11,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:34:11,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:34:11,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:34:11,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:16,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:21,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:26,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:34:26,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:34:26,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:34:26,611 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:34:27,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:32,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:37,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:41,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:34:41,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:34:41,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:34:43,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:45,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:34:45,473 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:34:45,474 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:34:45,475 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:34:46,273 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:34:46,274 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:34:49,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:54,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:34:56,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:34:56,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:34:56,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:34:56,566 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:34:59,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:04,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:09,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:11,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:35:11,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:35:11,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:35:14,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:19,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:24,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:26,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:35:26,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:35:26,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:35:26,613 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:35:30,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:35,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:40,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:41,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:35:41,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:35:41,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:35:46,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:51,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:56,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:35:56,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:35:56,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:35:56,597 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:35:56,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:35:59,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:35:59,366 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:35:59,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:35:59,368 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:36:00,318 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:36:00,318 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:36:02,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:07,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:11,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:36:11,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:36:11,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:36:12,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:17,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:22,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:26,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:36:26,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:36:26,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:36:26,647 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:36:27,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:32,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:37,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:41,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:36:41,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:36:41,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:36:43,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:48,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:53,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:36:56,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:36:56,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:36:56,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:36:56,651 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:36:58,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:03,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:08,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:37:11,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:37:11,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:37:13,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:37:13,367 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:37:13,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:37:13,368 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:37:14,362 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:37:14,362 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:37:14,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:19,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:24,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:37:26,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:37:26,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:37:26,577 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:37:29,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:34,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:39,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:41,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:37:41,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:37:41,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:37:44,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:49,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:54,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:37:56,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:37:56,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:37:56,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:37:56,598 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:38:00,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:05,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:10,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:38:11,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:38:11,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:38:15,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:20,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:25,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:38:26,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:38:26,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:38:26,575 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:38:27,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:38:27,042 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:38:27,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:38:27,045 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:38:27,404 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:38:28,405 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:38:31,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:36,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:41,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:41,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:38:41,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:38:41,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:38:46,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:51,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:38:56,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:38:56,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:38:56,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:38:56,598 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:38:57,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:02,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:07,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:39:11,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:39:11,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:39:12,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:17,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:22,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:39:26,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:39:26,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:39:26,641 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:39:28,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:33,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:38,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:40,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:39:40,922 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:39:40,923 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:39:40,924 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:39:41,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:39:41,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:39:41,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:39:41,449 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:39:43,450 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:39:44,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:49,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:54,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:39:56,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:39:56,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:39:56,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:39:56,644 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:39:59,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:04,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:09,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:11,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:40:11,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:40:11,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:40:14,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:19,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:24,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:26,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:40:26,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:40:26,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:40:26,610 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:40:30,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:35,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:40,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:41,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:40:41,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:40:41,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:40:46,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:51,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:54,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:40:54,846 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:40:54,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:40:54,848 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:40:55,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:40:55,491 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:40:56,425 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:40:56,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:40:56,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:40:56,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:40:56,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:40:57,493 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:40:59,494 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:41:01,495 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:41:01,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:06,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:11,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:41:11,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:41:11,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:41:12,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:17,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:22,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:26,426 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:41:26,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:41:26,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:41:26,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:41:28,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:33,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:38,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:41,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:41:41,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:41:41,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:41:43,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:48,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:53,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:41:56,427 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:41:56,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:41:56,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:41:56,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:41:59,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:04,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:09,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:11,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:42:11,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:42:11,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:42:12,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:42:12,739 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:42:12,739 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:42:12,740 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:42:13,538 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:42:15,539 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:42:15,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:20,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:25,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:26,428 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:42:26,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:42:26,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:42:26,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:42:30,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:35,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:40,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:41,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:42:41,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:42:41,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:42:46,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:51,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:42:56,429 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:42:56,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:42:56,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:42:56,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:42:56,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:01,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:06,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:11,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:43:11,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:43:11,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:43:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:16,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:21,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:26,430 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:43:26,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:43:26,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:43:26,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:43:26,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:43:26,841 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:43:26,842 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:43:26,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:26,843 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:43:27,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:43:29,582 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:43:31,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:36,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:41,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:43:41,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:43:41,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:43:42,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:47,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:52,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:43:56,430 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:43:56,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:43:56,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:43:56,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:43:57,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:02,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:07,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:11,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:44:11,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:44:11,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:44:13,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:18,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:23,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:26,432 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:44:26,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:44:26,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:44:26,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:44:28,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:33,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:38,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:40,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:44:40,770 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:44:40,770 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:44:40,773 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:44:41,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:44:41,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:44:41,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:44:41,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:44:43,625 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:44:44,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:49,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:54,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:44:56,432 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:44:56,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:44:56,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:44:56,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:44:59,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:04,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:09,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:11,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:45:11,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:45:11,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:45:15,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:20,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:25,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:26,433 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:45:26,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:45:26,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:45:26,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:45:31,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:36,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:41,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:45:41,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:45:41,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:45:42,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:47,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:52,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:45:54,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:45:54,863 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:45:54,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:45:54,865 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:45:55,670 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:45:56,434 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:45:56,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:45:56,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:45:56,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:45:57,671 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:45:57,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:02,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:07,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:11,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:46:11,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:46:11,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:46:12,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:17,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:22,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:26,435 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:46:26,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:46:26,544 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:46:26,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:46:28,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:33,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:38,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:41,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:46:41,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:46:41,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:46:43,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:48,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:53,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:46:56,436 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:46:56,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:46:56,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:46:56,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:46:59,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:04,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:08,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:47:08,951 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:47:08,952 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:47:08,953 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:47:09,714 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:47:09,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:11,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:47:11,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:47:11,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:47:11,715 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:47:15,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:20,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:25,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:26,437 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:47:26,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:47:26,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:47:26,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:47:31,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:36,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:41,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:47:41,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:47:41,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:47:42,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:47,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:52,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:47:56,438 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:47:56,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:47:56,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:47:56,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:47:57,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:02,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:07,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:11,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:48:11,546 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:48:11,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:48:13,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:18,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:22,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:48:22,897 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:48:22,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:48:22,899 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:48:23,757 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:48:23,757 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:48:23,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:26,439 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:48:26,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:48:26,546 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:48:26,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:48:29,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:34,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:39,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:41,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:48:41,546 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:48:41,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:48:44,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:49,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:54,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:48:56,440 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:48:56,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:48:56,546 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:48:56,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:49:00,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:05,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:10,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:11,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:49:11,546 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:49:11,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:49:15,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:20,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:25,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:26,441 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:49:26,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:49:26,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:49:26,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:49:31,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:36,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:37,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:49:37,172 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:49:37,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:49:37,174 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:49:37,800 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:49:39,801 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:49:41,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:49:41,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:49:41,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:49:42,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:47,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:52,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:49:56,442 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:49:56,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:49:56,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:49:56,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:49:57,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:02,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:07,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:11,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:50:11,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:50:11,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:50:12,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:17,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:22,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:26,443 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:50:26,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:50:26,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:50:26,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:50:27,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:32,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:37,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:41,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:50:41,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:50:41,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:50:43,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:48,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:51,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:50:51,037 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:50:51,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:50:51,039 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:50:51,843 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:50:51,843 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:50:54,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:50:56,444 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:50:56,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:50:56,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:50:56,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:50:59,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:04,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:09,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:11,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:51:11,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:51:11,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:51:14,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:19,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:24,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:26,445 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:51:26,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:51:26,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:51:26,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:51:30,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:35,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:40,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:41,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:51:41,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:51:41,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:51:46,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:51,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:51:56,446 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:51:56,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:51:56,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:51:56,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:51:56,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:01,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:04,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:52:04,728 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:52:04,728 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:52:04,730 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:52:04,887 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:52:05,888 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:52:07,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:11,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:52:11,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:52:11,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:52:12,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:17,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:22,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:26,446 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:52:26,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:52:26,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:52:26,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:52:27,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:32,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:37,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:41,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:52:41,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:52:41,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:52:43,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:48,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:53,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:52:56,447 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:52:56,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:52:56,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:52:56,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:52:58,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:03,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:08,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:11,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:53:11,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:53:11,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:53:13,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:18,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:53:18,680 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:53:18,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:53:18,682 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:53:18,931 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:53:19,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:19,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:53:19,932 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:53:21,933 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:53:22,934 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:53:23,935 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:53:25,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:26,448 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:53:26,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:53:26,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:53:26,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:53:30,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:35,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:40,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:41,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:53:41,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:53:41,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:53:45,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:50,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:55,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:53:56,449 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:53:56,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:53:56,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:53:56,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:54:01,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:06,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:11,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:54:11,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:54:11,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:54:11,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:16,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:21,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:26,450 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:54:26,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:54:26,548 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:54:26,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:54:27,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:32,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:37,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:54:37,017 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:54:37,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:54:37,018 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:54:37,977 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:54:37,977 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:54:38,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:41,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:54:41,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:54:41,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:54:43,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:48,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:53,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:54:56,451 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:54:56,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:54:56,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:54:56,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:54:59,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:04,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:09,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:11,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:55:11,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:55:11,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:55:15,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:20,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:25,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:26,452 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:55:26,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:55:26,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:55:26,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:55:31,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:36,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:41,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:55:41,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:55:41,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:55:42,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:47,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:50,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:55:50,868 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:55:50,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:55:50,870 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:55:51,018 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:55:52,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:55:52,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:55:56,453 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:55:56,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:55:56,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:55:56,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:55:58,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:03,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:08,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:11,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:56:11,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:56:11,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:56:14,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:19,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:24,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:26,454 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:56:26,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:56:26,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:56:26,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:56:29,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:34,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:39,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:41,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:56:41,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:56:41,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:56:45,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:50,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:55,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:56:56,455 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:56:56,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:56:56,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:56:56,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:57:01,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:04,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:57:04,661 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:57:04,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:57:04,663 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:57:05,061 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:57:06,062 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:57:07,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:11,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:57:11,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:57:11,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:57:12,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:17,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:22,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:26,456 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:57:26,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:57:26,549 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:57:26,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:57:28,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:33,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:38,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:41,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:57:41,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:57:41,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:57:43,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:48,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:53,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:57:56,457 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:57:56,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:57:56,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:57:56,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:57:59,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:04,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:09,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:11,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:58:11,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:58:11,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:58:14,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:18,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:58:18,470 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:58:18,471 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:58:18,473 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:58:19,105 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:58:20,106 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:58:20,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:25,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:26,457 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:58:26,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:58:26,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:58:26,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:58:30,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:35,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:40,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:41,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:58:41,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:58:41,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:58:45,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:50,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:55,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:58:56,458 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:58:56,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:58:56,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:58:56,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:59:01,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:06,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:11,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:59:11,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:59:11,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:59:12,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:17,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:22,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:26,459 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:59:26,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:59:26,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:59:26,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:59:28,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:32,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 08:59:32,349 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 08:59:32,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 08:59:32,351 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 08:59:33,148 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 08:59:34,149 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 08:59:34,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:39,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:41,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:59:41,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:59:41,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 08:59:44,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:49,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:54,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 08:59:56,460 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 08:59:56,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 08:59:56,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 08:59:56,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:00:00,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:05,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:10,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:11,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:00:11,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:00:11,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:00:15,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:20,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:25,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:26,461 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:00:26,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:00:26,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:00:26,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:00:30,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:35,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:40,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:41,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:00:41,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:00:41,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:00:45,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:46,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:00:46,436 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:00:46,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:00:46,438 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:00:47,191 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:00:48,192 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:00:51,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:56,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:00:56,462 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:00:56,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:00:56,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:00:56,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:01:01,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:06,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:11,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:01:11,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:01:11,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:01:12,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:17,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:22,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:26,463 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:01:26,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:01:26,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:01:26,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:01:28,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:33,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:38,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:41,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:01:41,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:01:41,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:01:43,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:48,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:53,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:01:56,464 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:01:56,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:01:56,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:01:56,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:01:58,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:00,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:02:00,341 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:02:00,342 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:02:00,343 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:02:01,236 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:02:02,237 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:02:04,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:09,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:11,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:02:11,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:02:11,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:02:14,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:19,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:24,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:26,465 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:02:26,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:02:26,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:02:26,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:02:30,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:35,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:40,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:41,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:02:41,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:02:41,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:02:45,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:50,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:55,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:02:56,467 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:02:56,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:02:56,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:02:56,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:03:01,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:06,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:11,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:03:11,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:03:11,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:03:12,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:14,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:03:14,188 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:03:14,188 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:03:14,189 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:03:14,279 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:03:16,280 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:03:18,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:23,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:26,466 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:03:26,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:03:26,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:03:26,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:03:28,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:33,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:38,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:41,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:03:41,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:03:41,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:03:43,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:48,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:53,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:03:56,468 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:03:56,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:03:56,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:03:56,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:03:59,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:04,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:09,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:11,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:04:11,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:04:11,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:04:14,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:19,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:24,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:26,468 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:04:26,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:04:26,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:04:26,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:04:28,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:04:28,237 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:04:28,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:04:28,238 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:04:28,322 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:04:30,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:30,323 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:04:35,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:40,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:41,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:04:41,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:04:41,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:04:45,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:50,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:55,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:04:56,469 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:04:56,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:04:56,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:04:56,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:05:00,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:05,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:10,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:11,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:05:11,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:05:11,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:05:16,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:21,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:26,470 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:05:26,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:05:26,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:05:26,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:05:26,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:31,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:36,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:41,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:05:41,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:05:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:05:42,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:05:42,481 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:05:42,481 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:05:42,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:42,483 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:05:43,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:05:43,366 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:05:44,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:05:45,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:05:46,368 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:05:47,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:48,369 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:05:52,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:05:56,471 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:05:56,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:05:56,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:05:56,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:05:57,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:02,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:07,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:11,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:06:11,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:06:11,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:06:12,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:17,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:22,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:26,472 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:06:26,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:06:26,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:06:26,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:06:28,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:33,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:38,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:41,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:06:41,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:06:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:06:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:48,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:53,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:06:56,473 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:06:56,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:06:56,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:06:56,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:06:59,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:00,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:07:00,612 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:07:00,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:07:00,613 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:07:01,412 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:07:02,413 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:07:05,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:10,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:11,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:07:11,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:07:11,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:07:15,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:20,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:25,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:26,474 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:07:26,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:07:26,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:07:26,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:07:30,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:35,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:40,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:41,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:07:41,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:07:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:07:46,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:51,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:07:56,475 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:07:56,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:07:56,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:07:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:07:56,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:01,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:06,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:11,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:08:11,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:08:11,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:08:12,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:14,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:08:14,553 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:08:14,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:08:14,554 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:08:15,456 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:08:16,457 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:08:18,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:23,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:26,476 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:08:26,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:08:26,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:08:26,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:08:28,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:33,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:38,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:41,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:08:41,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:08:41,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:08:44,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:49,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:54,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:08:56,477 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:08:56,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:08:56,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:08:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:08:59,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:04,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:09,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:11,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:09:11,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:09:11,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:09:15,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:20,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:25,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:26,478 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:09:26,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:09:26,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:09:26,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:09:28,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:09:28,523 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:09:28,524 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:09:28,525 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:09:29,500 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:09:30,500 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:09:31,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:36,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:41,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:41,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:09:41,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:09:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:09:46,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:51,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:09:56,479 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:09:56,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:09:56,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:09:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:09:56,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:01,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:06,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:11,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:10:11,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:10:11,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:10:11,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:16,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:21,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:26,481 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:10:26,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:10:26,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:10:26,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:10:27,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:32,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:37,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:41,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:10:41,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:10:41,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:10:42,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:10:42,375 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:10:42,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:10:42,376 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:10:42,543 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:10:43,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:44,545 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:10:48,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:53,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:10:56,481 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:10:56,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:10:56,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:10:56,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:10:58,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:03,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:08,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:11,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:11:11,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:11:11,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:11:14,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:19,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:24,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:26,483 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:11:26,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:11:26,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:11:26,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:11:29,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:34,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:39,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:41,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:11:41,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:11:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:11:45,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:50,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:55,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:11:56,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:11:56,360 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:11:56,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:11:56,361 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:11:56,483 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:11:56,587 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:11:56,587 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:11:56,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:11:56,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:11:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:12:00,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:05,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:10,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:11,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:12:11,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:12:11,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:12:16,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:21,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:26,484 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:12:26,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:12:26,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:12:26,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:12:27,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:32,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:37,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:41,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:12:41,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:12:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:12:42,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:47,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:52,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:12:56,485 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:12:56,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:12:56,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:12:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:12:58,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:03,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:08,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:10,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:13:10,314 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:13:10,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:13:10,316 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:13:10,632 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:13:10,632 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:13:11,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:13:11,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:13:11,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:13:14,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:19,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:24,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:26,486 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:13:26,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:13:26,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:13:26,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:13:29,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:34,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:39,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:41,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:13:41,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:13:41,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:13:45,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:50,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:55,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:13:56,487 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:13:56,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:13:56,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:13:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:14:01,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:06,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:11,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:14:11,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:14:11,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:14:11,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:16,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:21,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:24,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:14:24,172 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:14:24,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:14:24,173 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:14:24,676 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:14:24,676 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:14:26,488 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:14:26,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:14:26,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:14:26,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:14:26,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:31,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:36,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:41,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:14:41,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:14:41,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:14:41,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:46,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:51,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:14:56,489 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:14:56,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:14:56,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:14:56,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:14:57,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:02,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:07,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:11,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:15:11,641 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:15:11,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:15:12,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:17,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:22,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:26,490 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:15:26,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:15:26,641 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:15:26,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:15:28,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:33,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:38,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:15:38,010 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:15:38,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:15:38,012 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:15:38,720 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:15:39,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:39,721 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:15:41,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:15:41,642 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:15:41,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:15:44,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:49,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:54,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:15:56,491 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:15:56,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:15:56,642 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:15:56,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:15:59,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:04,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:09,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:11,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:16:11,642 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:16:11,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:16:14,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:19,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:24,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:26,492 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:16:26,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:16:26,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:16:26,656 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:16:29,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:34,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:39,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:41,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:16:41,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:16:41,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:16:45,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:50,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:51,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:16:51,775 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:16:51,776 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:16:51,777 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:16:52,765 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:16:53,765 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:16:56,493 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:16:56,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:16:56,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:16:56,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:16:56,656 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:17:01,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:06,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:11,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:17:11,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:17:11,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:17:11,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:16,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:21,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:26,494 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:17:26,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:17:26,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:17:26,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:17:27,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:32,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:37,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:41,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:17:41,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:17:41,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:17:42,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:47,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:52,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:17:56,495 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:17:56,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:17:56,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:17:56,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:17:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:03,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:05,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:18:05,727 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:18:05,728 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:18:05,730 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:18:05,807 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:18:06,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:18:07,808 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:18:08,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:09,810 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:18:10,811 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:18:11,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:18:11,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:18:11,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:18:11,812 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:18:14,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:19,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:24,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:26,496 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:18:26,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:18:26,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:18:26,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:18:29,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:34,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:39,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:41,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:18:41,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:18:41,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:18:44,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:49,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:54,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:18:56,497 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:18:56,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:18:56,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:18:56,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:19:00,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:05,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:10,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:11,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:19:11,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:19:11,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:19:15,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:20,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:24,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:19:24,114 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:19:24,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:19:24,116 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:19:24,855 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:19:25,856 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:19:26,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:26,498 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:19:26,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:19:26,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:19:26,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:19:31,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:36,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:41,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:19:41,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:19:41,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:19:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:46,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:51,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:19:56,499 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:19:56,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:19:56,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:19:56,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:19:57,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:02,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:07,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:11,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:20:11,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:20:11,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:20:12,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:17,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:22,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:26,500 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:20:26,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:20:26,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:20:26,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:20:27,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:32,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:37,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:37,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:20:37,944 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:20:37,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:20:37,946 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:20:38,901 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:20:39,901 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:20:41,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:20:41,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:20:41,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:20:42,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:47,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:52,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:20:56,501 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:20:56,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:20:56,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:20:56,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:20:58,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:03,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:08,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:11,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:21:11,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:21:11,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:21:13,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:18,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:23,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:26,502 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:21:26,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:21:26,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:21:26,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:21:29,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:34,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:39,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:41,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:21:41,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:21:41,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:21:44,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:49,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:51,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:21:51,817 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:21:51,818 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:21:51,819 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:21:51,944 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:21:53,945 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:21:55,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:21:56,502 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:21:56,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:21:56,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:21:56,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:22:00,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:05,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:10,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:11,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:22:11,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:22:11,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:22:15,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:20,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:25,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:26,503 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:22:26,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:22:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:22:26,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:22:31,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:36,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:41,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:22:41,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:22:41,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:22:41,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:46,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:51,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:22:56,505 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:22:56,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:22:56,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:22:56,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:22:57,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:02,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:05,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:23:05,708 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:23:05,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:23:05,711 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:23:05,987 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:23:07,989 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:23:08,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:11,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:23:11,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:23:11,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:23:13,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:18,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:23,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:26,506 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:23:26,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:23:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:23:26,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:23:29,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:34,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:39,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:41,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:23:41,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:23:41,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:23:44,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:49,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:54,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:23:56,506 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:23:56,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:23:56,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:23:56,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:23:59,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:04,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:09,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:11,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:24:11,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:24:11,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:24:14,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:19,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:24:19,384 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:24:19,385 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:24:19,387 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:24:20,031 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:24:20,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:22,033 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:24:25,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:26,507 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:24:26,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:24:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:24:26,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:24:30,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:35,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:40,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:41,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:24:41,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:24:41,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:24:45,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:50,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:55,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:24:56,508 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:24:56,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:24:56,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:24:56,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:25:01,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:06,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:11,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:25:11,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:25:11,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:25:11,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:16,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:21,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:26,509 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:25:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:25:26,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:25:26,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:25:26,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:31,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:33,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:25:33,445 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:25:33,446 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:25:33,448 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:25:34,075 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:25:36,077 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:25:37,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:41,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:25:41,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:25:41,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:25:42,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:47,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:52,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:25:56,510 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:25:56,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:25:56,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:25:56,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:25:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:03,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:08,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:11,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:26:11,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:26:11,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:26:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:18,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:23,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:26,511 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:26:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:26:26,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:26:26,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:26:29,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:34,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:39,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:41,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:26:41,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:26:41,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:26:45,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:47,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:26:47,299 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:26:47,300 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:26:47,302 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:26:48,119 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:26:48,119 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:26:51,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:56,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:26:56,512 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:26:56,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:26:56,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:26:56,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:27:01,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:06,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:11,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:27:11,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:27:11,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:27:11,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:16,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:21,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:26,513 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:27:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:27:26,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:27:26,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:27:27,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:32,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:37,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:41,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:27:41,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:27:41,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:27:42,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:47,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:52,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:27:56,514 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:27:56,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:27:56,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:27:56,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:27:58,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:01,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:28:01,193 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:28:01,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:28:01,195 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:28:02,162 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:28:02,163 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:28:04,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:09,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:11,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:28:11,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:28:11,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:28:14,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:19,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:24,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:26,515 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:28:26,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:28:26,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:28:26,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:28:29,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:34,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:39,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:41,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:28:41,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:28:41,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:28:45,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:50,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:55,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:28:56,516 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:28:56,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:28:56,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:28:56,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:29:00,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:05,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:10,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:11,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:29:11,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:29:11,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:29:15,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:29:15,091 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:29:15,092 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:29:15,093 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:29:15,205 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:29:16,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:16,206 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:29:21,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:26,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:26,517 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:29:26,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:29:26,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:29:26,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:29:31,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:36,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:41,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:29:41,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:29:41,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:29:41,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:46,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:51,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:29:56,518 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:29:56,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:29:56,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:29:56,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:29:57,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:02,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:07,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:11,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:30:11,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:30:11,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:30:12,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:17,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:22,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:26,519 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:30:26,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:30:26,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:30:26,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:30:28,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:29,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:30:29,009 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:30:29,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:30:29,011 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:30:29,250 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:30:29,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:30:30,250 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:30:32,252 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:30:33,253 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:30:33,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:34,254 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:30:38,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:41,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:30:41,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:30:41,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:30:44,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:49,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:54,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:30:56,520 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:30:56,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:30:56,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:30:56,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:30:59,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:04,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:09,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:11,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:31:11,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:31:11,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:31:14,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:19,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:24,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:26,521 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:31:26,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:31:26,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:31:26,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:31:30,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:35,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:40,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:41,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:31:41,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:31:41,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:31:46,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:31:46,700 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:31:46,700 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:31:46,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:46,702 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:31:47,298 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:31:48,299 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:31:51,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:56,522 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:31:56,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:31:56,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:31:56,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:31:56,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:32:02,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:07,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:11,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:32:11,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:32:11,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:32:12,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:17,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:22,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:26,523 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:32:26,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:32:26,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:32:26,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:32:28,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:33,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:38,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:41,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:32:41,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:32:41,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:32:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:49,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:54,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:32:56,524 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:32:56,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:32:56,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:32:56,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:32:59,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:00,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:33:00,572 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:33:00,572 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:33:00,574 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:33:01,341 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:33:02,342 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:33:04,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:09,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:11,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:33:11,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:33:11,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:33:14,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:19,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:24,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:26,525 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:33:26,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:33:26,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:33:26,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:33:30,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:35,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:40,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:41,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:33:41,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:33:41,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:33:45,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:50,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:55,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:33:56,526 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:33:56,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:33:56,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:33:56,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:34:00,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:05,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:10,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:11,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:34:11,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:34:11,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:34:14,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:34:14,435 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:34:14,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:34:14,436 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:34:15,385 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:34:16,385 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:34:16,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:21,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:26,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:26,527 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:34:26,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:34:26,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:34:26,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:34:32,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:37,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:41,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:34:41,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:34:41,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:34:42,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:47,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:52,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:34:56,528 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:34:56,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:34:56,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:34:56,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:34:57,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:02,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:07,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:11,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:35:11,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:35:11,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:35:12,997 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:17,997 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:22,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:26,529 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:35:26,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:35:26,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:35:26,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:35:28,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:28,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:35:28,264 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:35:28,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:35:28,265 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:35:28,428 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:35:30,429 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:35:33,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:38,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:41,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:35:41,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:35:41,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:35:43,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:48,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:53,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:35:56,530 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:35:56,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:35:56,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:35:56,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:35:59,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:04,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:09,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:11,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:36:11,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:36:11,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:36:14,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:19,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:24,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:26,531 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:36:26,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:36:26,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:36:26,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:36:29,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:34,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:39,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:41,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:36:41,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:36:41,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:36:42,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:36:42,089 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:36:42,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:36:42,090 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:36:42,472 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:36:44,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:36:45,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:50,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:55,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:36:56,532 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:36:56,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:36:56,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:36:56,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:37:01,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:06,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:11,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:11,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:37:11,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:37:11,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:37:16,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:21,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:26,533 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:37:26,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:37:26,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:37:26,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:37:27,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:32,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:37,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:41,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:37:41,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:37:41,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:37:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:47,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:52,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:37:56,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:37:56,120 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:37:56,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:37:56,121 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:37:56,515 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:37:56,533 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:37:56,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:37:56,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:37:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:37:58,517 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:37:58,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:03,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:08,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:11,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:38:11,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:38:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:38:13,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:18,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:23,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:26,534 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:38:26,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:38:26,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:38:26,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:38:29,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:34,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:39,003 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:41,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:38:41,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:38:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:38:44,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:49,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:54,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:38:56,535 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:38:56,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:38:56,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:38:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:39:00,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:05,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:09,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:39:09,927 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:39:09,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:39:09,928 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:39:10,559 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:39:10,559 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:39:10,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:11,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:39:11,764 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:39:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:39:16,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:21,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:26,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:26,536 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:39:26,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:39:26,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:39:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:39:31,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:36,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:41,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:41,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:39:41,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:39:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:39:47,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:52,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:39:56,537 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:39:56,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:39:56,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:39:56,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:39:57,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:02,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:07,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:11,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:40:11,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:40:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:40:13,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:18,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:23,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:23,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:40:23,751 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:40:23,752 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:40:23,753 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:40:24,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:40:24,603 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:40:26,538 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:40:26,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:40:26,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:40:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:40:29,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:34,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:39,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:41,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:40:41,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:40:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:40:44,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:49,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:54,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:40:56,539 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:40:56,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:40:56,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:40:56,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:40:59,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:04,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:09,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:11,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:41:11,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:41:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:41:15,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:20,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:25,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:26,541 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:41:26,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:41:26,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:41:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:41:31,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:36,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:37,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:41:37,716 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:41:37,717 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:41:37,718 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:41:38,647 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:41:38,647 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:41:41,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:41,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:41:41,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:41:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:41:47,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:52,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:41:56,541 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:41:56,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:41:56,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:41:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:41:57,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:02,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:07,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:11,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:42:11,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:42:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:42:13,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:18,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:23,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:26,542 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:42:26,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:42:26,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:42:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:42:28,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:33,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:38,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:41,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:42:41,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:42:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:42:43,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:48,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:51,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:42:51,434 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:42:51,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:42:51,435 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:42:51,690 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:42:52,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:42:52,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:42:53,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:42:54,693 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:42:56,543 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:42:56,694 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:42:56,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:42:56,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:42:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:42:58,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:03,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:08,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:11,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:43:11,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:43:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:43:13,993 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:18,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:23,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:26,544 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:43:26,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:43:26,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:43:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:43:29,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:34,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:39,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:41,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:43:41,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:43:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:43:44,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:49,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:54,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:43:56,544 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:43:56,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:43:56,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:43:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:44:00,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:05,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:09,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:44:09,564 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:44:09,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:44:09,566 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:44:09,736 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:44:10,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:10,737 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:44:11,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:44:11,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:44:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:44:15,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:20,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:25,993 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:26,545 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:44:26,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:44:26,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:44:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:44:31,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:36,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:41,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:44:41,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:44:41,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:44:41,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:46,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:51,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:44:56,546 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:44:56,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:44:56,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:44:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:44:57,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:02,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:07,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:11,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:45:11,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:45:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:45:12,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:17,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:22,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:23,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:45:23,273 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:45:23,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:45:23,275 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:45:23,781 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:45:24,782 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:45:26,547 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:45:26,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:45:26,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:45:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:45:27,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:32,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:37,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:41,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:45:41,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:45:41,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:45:43,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:48,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:53,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:45:56,548 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:45:56,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:45:56,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:45:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:45:58,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:04,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:09,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:11,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:46:11,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:46:11,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:46:14,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:19,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:24,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:26,549 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:46:26,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:46:26,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:46:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:46:30,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:35,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:37,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:46:37,169 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:46:37,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:46:37,171 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:46:37,825 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:46:38,826 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:46:40,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:41,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:46:41,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:46:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:46:46,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:51,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:56,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:46:56,550 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:46:56,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:46:56,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:46:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:47:02,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:07,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:11,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:47:11,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:47:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:47:12,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:17,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:22,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:26,551 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:47:26,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:47:26,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:47:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:47:27,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:32,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:37,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:41,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:47:41,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:47:41,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:47:43,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:48,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:51,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:47:51,043 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:47:51,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:47:51,045 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:47:51,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:47:52,871 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:47:53,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:47:56,552 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:47:56,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:47:56,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:47:56,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:47:59,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:04,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:09,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:11,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:48:11,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:48:11,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:48:15,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:20,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:25,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:26,554 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:48:26,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:48:26,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:48:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:48:30,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:35,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:40,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:41,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:48:41,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:48:41,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:48:46,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:51,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:56,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:48:56,554 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:48:56,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:48:56,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:48:56,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:49:01,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:04,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:49:04,878 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:49:04,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:49:04,881 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:49:04,913 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:49:06,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:06,914 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:49:11,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:49:11,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:49:11,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:49:12,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:17,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:22,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:26,554 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:49:26,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:49:26,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:49:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:49:27,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:32,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:37,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:41,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:49:41,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:49:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:49:43,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:48,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:53,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:49:56,556 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:49:56,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:49:56,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:49:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:49:58,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:03,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:08,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:11,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:50:11,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:50:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:50:14,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:18,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:50:18,824 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:50:18,825 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:50:18,826 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:50:18,958 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:50:19,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:19,959 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:50:24,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:26,556 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:50:26,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:50:26,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:50:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:50:30,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:35,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:40,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:41,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:50:41,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:50:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:50:46,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:51,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:56,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:50:56,557 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:50:56,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:50:56,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:50:56,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:51:01,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:06,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:11,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:11,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:51:11,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:51:11,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:51:17,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:22,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:26,558 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:51:26,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:51:26,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:51:26,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:51:28,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:32,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:51:32,779 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:51:32,779 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:51:32,781 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:51:33,002 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:51:33,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:34,003 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:51:38,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:41,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:51:41,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:51:41,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:51:44,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:49,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:54,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:51:56,559 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:51:56,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:51:56,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:51:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:51:59,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:04,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:09,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:11,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:52:11,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:52:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:52:15,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:20,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:25,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:26,560 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:52:26,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:52:26,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:52:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:52:30,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:35,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:40,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:41,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:52:41,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:52:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:52:45,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:46,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:52:46,468 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:52:46,468 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:52:46,469 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:52:47,047 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:52:48,047 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:52:50,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:55,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:52:56,561 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:52:56,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:52:56,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:52:56,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:53:01,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:06,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:11,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:11,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:53:11,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:53:11,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:53:17,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:22,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:26,562 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:53:26,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:53:26,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:53:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:53:27,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:32,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:37,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:41,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:53:41,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:53:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:53:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:48,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:53,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:53:56,563 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:53:56,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:53:56,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:53:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:53:59,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:00,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:54:00,344 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:54:00,344 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:54:00,345 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:54:01,091 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:54:02,092 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:54:04,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:09,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:11,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:54:11,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:54:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:54:15,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:20,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:25,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:26,565 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:54:26,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:54:26,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:54:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:54:31,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:36,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:41,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:41,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:54:41,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:54:41,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:54:47,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:52,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:54:56,566 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:54:56,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:54:56,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:54:56,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:54:58,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:03,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:08,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:11,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:55:11,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:55:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:55:13,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:14,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:55:14,241 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:55:14,241 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:55:14,243 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:55:14,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:55:15,136 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:55:16,137 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:55:18,138 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:55:18,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:19,139 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:55:20,139 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:55:23,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:26,567 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:55:26,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:55:26,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:55:26,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:55:29,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:34,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:39,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:41,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:55:41,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:55:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:55:44,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:49,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:54,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:55:56,569 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:55:56,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:55:56,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:55:56,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:56:00,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:05,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:10,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:11,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:56:11,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:56:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:56:15,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:20,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:25,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:26,569 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:56:26,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:56:26,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:56:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:56:31,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:32,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:56:32,039 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:56:32,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:56:32,041 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:56:32,183 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:56:34,184 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:56:36,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:41,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:41,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:56:41,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:56:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:56:46,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:51,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:56,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:56:56,570 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:56:56,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:56:56,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:56:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:57:02,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:07,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:11,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:57:11,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:57:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:57:12,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:17,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:22,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:26,572 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:57:26,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:57:26,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:57:26,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:57:27,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:32,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:37,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:41,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:57:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:57:41,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:57:43,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:45,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:57:45,730 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:57:45,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:57:45,736 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:57:46,227 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:57:48,228 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:57:48,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:53,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:57:56,572 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:57:56,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:57:56,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:57:56,875 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:57:59,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:04,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:09,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:11,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:58:11,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:58:11,875 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:58:15,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:20,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:25,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:26,573 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:58:26,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:58:26,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:58:26,875 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:58:30,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:35,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:40,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:41,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:58:41,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:58:41,875 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:58:46,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:51,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:56,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:58:56,574 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:58:56,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:58:56,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:58:56,875 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:58:59,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 09:58:59,457 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 09:58:59,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 09:58:59,459 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 09:59:00,270 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 09:59:00,270 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 09:59:01,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:06,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:11,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:11,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:59:11,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:59:11,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:59:17,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:22,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:26,575 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:59:26,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:59:26,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:59:26,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:59:27,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:32,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:37,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:41,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:59:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:59:41,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:59:43,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:48,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:53,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 09:59:56,575 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 09:59:56,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 09:59:56,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 09:59:56,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 09:59:59,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:04,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:09,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:11,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:00:11,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:00:11,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:00:13,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:00:13,166 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:00:13,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:00:13,168 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:00:13,312 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:00:14,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:14,313 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:00:19,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:24,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:26,576 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:00:26,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:00:26,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:00:26,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:00:30,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:35,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:40,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:41,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:00:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:00:41,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:00:45,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:50,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:55,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:00:56,577 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:00:56,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:00:56,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:00:56,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:01:01,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:06,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:11,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:11,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:01:11,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:01:11,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:01:16,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:21,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:26,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:26,578 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:01:26,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:01:26,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:01:26,877 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:01:27,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:01:27,164 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:01:27,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:01:27,166 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:01:27,355 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:01:28,356 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:01:31,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:36,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:41,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:01:41,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:01:41,877 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:01:47,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:52,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:01:56,579 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:01:56,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:01:56,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:01:56,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:01:58,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:03,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:08,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:11,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:02:11,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:02:11,877 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:02:13,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:18,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:23,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:26,580 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:02:26,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:02:26,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:02:26,877 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:02:29,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:34,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:39,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:40,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:02:40,638 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:02:40,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:02:40,640 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:02:41,397 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:02:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:02:41,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:02:41,877 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:02:42,398 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:02:45,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:50,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:55,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:02:56,581 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:02:56,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:02:56,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:02:56,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:03:01,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:06,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:11,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:11,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:03:11,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:03:11,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:03:16,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:21,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:26,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:26,582 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:03:26,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:03:26,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:03:26,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:03:32,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:37,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:03:41,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:03:41,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:03:43,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:48,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:53,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:03:54,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:03:54,379 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:03:54,380 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:03:54,381 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:03:54,440 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:03:56,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:03:56,583 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:03:56,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:03:56,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:03:56,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:03:59,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:04,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:09,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:11,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:04:11,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:04:11,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:04:15,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:20,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:25,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:26,584 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:04:26,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:04:26,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:04:26,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:04:30,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:35,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:40,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:41,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:04:41,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:04:41,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:04:46,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:51,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:56,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:04:56,585 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:04:56,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:04:56,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:04:56,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:05:01,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:06,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:08,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:05:08,438 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:05:08,439 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:05:08,440 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:05:08,483 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:05:10,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:05:11,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:11,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:05:11,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:05:11,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:05:17,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:22,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:26,586 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:05:26,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:05:26,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:05:26,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:05:28,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:33,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:38,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:41,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:05:41,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:05:41,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:05:43,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:48,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:53,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:05:56,586 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:05:56,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:05:56,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:05:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:05:58,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:03,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:08,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:11,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:06:11,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:06:11,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:06:14,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:19,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:22,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:06:22,141 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:06:22,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:06:22,142 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:06:22,527 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:06:24,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:24,529 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:06:26,587 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:06:26,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:06:26,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:06:26,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:06:30,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:35,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:40,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:41,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:06:41,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:06:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:06:46,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:51,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:56,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:06:56,588 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:06:56,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:06:56,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:06:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:07:01,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:06,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:11,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:11,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:07:11,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:07:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:07:16,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:21,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:26,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:26,589 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:07:26,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:07:26,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:07:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:07:32,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:36,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:07:36,072 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:07:36,073 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:07:36,074 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:07:36,571 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:07:36,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:07:37,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:38,573 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:07:39,574 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:07:40,575 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:07:41,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:07:41,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:07:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:07:42,576 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:07:43,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:48,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:53,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:07:56,590 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:07:56,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:07:56,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:07:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:07:59,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:04,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:09,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:11,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:08:11,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:08:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:08:15,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:20,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:25,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:26,591 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:08:26,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:08:26,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:08:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:08:31,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:36,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:41,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:41,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:08:41,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:08:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:08:46,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:51,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:54,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:08:54,150 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:08:54,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:08:54,151 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:08:54,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:08:56,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:08:56,592 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:08:56,619 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:08:56,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:08:56,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:08:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:09:02,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:07,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:11,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:09:11,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:09:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:09:12,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:17,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:22,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:26,593 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:09:26,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:09:26,889 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:09:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:09:28,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:33,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:38,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:41,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:09:41,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:09:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:09:43,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:48,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:53,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:09:56,594 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:09:56,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:09:56,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:09:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:09:59,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:04,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:07,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:10:07,969 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:10:07,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:10:07,970 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:10:08,664 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:10:08,664 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:10:10,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:11,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:10:11,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:10:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:10:15,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:20,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:25,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:26,595 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:10:26,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:10:26,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:10:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:10:30,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:35,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:40,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:41,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:10:41,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:10:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:10:46,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:51,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:56,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:10:56,596 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:10:56,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:10:56,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:10:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:11:02,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:07,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:11,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:11:11,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:11:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:11:12,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:17,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:21,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:11:21,921 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:11:21,921 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:11:21,923 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:11:22,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:11:22,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:11:22,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:26,597 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:11:26,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:11:26,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:11:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:11:28,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:33,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:38,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:41,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:11:41,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:11:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:11:44,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:49,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:54,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:11:56,598 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:11:56,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:11:56,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:11:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:11:59,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:04,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:09,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:11,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:12:11,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:12:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:12:15,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:20,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:25,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:26,599 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:12:26,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:12:26,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:12:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:12:30,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:35,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:35,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:12:35,782 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:12:35,782 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:12:35,783 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:12:36,754 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:12:36,755 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:12:40,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:41,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:12:41,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:12:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:12:46,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:51,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:56,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:12:56,599 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:12:56,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:12:56,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:12:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:13:01,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:06,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:11,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:11,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:13:11,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:13:11,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:13:17,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:22,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:26,600 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:13:26,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:13:26,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:13:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:13:27,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:32,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:37,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:41,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:13:41,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:13:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:13:43,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:48,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:49,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:13:49,674 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:13:49,674 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:13:49,675 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:13:49,800 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:13:50,801 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:13:53,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:13:56,601 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:13:56,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:13:56,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:13:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:13:59,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:04,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:09,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:11,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:14:11,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:14:11,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:14:14,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:19,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:24,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:26,602 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:14:26,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:14:26,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:14:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:14:30,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:35,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:40,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:41,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:14:41,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:14:41,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:14:46,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:51,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:56,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:14:56,603 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:14:56,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:14:56,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:14:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:15:01,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:03,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:15:03,581 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:15:03,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:15:03,583 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:15:03,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:15:04,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:15:06,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:11,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:11,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:15:11,893 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:15:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:15:17,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:22,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:26,604 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:15:26,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:15:26,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:15:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:15:28,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:33,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:38,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:41,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:15:41,893 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:15:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:15:43,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:48,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:53,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:15:56,605 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:15:56,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:15:56,893 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:15:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:15:59,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:04,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:09,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:11,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:16:11,893 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:16:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:16:15,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:17,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:16:17,400 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:16:17,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:16:17,402 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:16:17,891 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:16:18,892 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:16:20,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:25,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:26,606 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:16:26,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:16:26,893 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:16:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:16:31,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:36,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:41,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:41,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:16:41,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:16:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:16:46,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:51,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:56,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:16:56,607 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:16:56,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:16:56,893 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:16:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:17:02,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:07,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:11,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:17:11,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:17:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:17:12,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:17,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:22,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:26,608 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:17:26,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:17:26,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:17:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:17:28,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:31,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:17:31,323 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:17:31,323 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:17:31,324 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:17:31,935 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:17:32,936 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:17:33,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:38,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:41,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:17:41,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:17:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:17:44,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:49,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:54,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:17:56,609 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:17:56,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:17:56,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:17:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:18:00,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:05,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:10,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:11,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:18:11,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:18:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:18:16,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:21,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:26,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:26,610 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:18:26,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:18:26,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:18:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:18:31,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:36,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:41,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:41,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:18:41,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:18:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:18:45,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:18:45,299 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:18:45,299 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:18:45,300 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:18:45,980 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:18:46,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:46,981 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:18:51,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:56,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:18:56,612 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:18:56,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:18:56,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:18:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:19:02,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:07,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:11,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:19:11,894 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:19:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:19:13,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:18,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:23,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:26,615 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:19:26,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:19:26,895 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:19:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:19:28,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:33,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:38,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:41,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:19:41,895 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:19:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:19:44,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:49,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:54,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:56,617 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:19:56,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:19:56,895 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:19:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:19:59,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:19:59,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:19:59,198 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:19:59,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:19:59,199 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:19:59,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:20:00,026 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:20:01,026 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:20:03,028 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:20:04,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:05,029 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:20:09,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:11,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:20:11,895 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:20:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:20:15,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:20,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:25,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:26,615 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:20:26,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:20:26,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:20:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:20:31,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:36,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:41,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:41,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:20:41,895 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:20:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:20:47,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:52,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:20:56,617 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:20:56,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:20:56,895 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:20:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:20:58,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:03,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:08,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:11,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:21:11,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:21:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:21:14,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:17,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:21:17,187 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:21:17,188 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:21:17,189 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:21:18,075 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:21:19,075 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:21:19,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:24,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:26,617 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:21:26,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:21:26,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:21:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:21:30,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:35,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:40,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:41,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:21:41,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:21:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:21:46,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:51,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:56,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:21:56,618 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:21:56,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:21:56,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:21:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:22:01,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:06,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:11,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:11,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:22:11,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:22:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:22:17,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:22,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:26,619 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:22:26,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:22:26,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:22:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:22:28,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:31,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:22:31,216 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:22:31,216 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:22:31,217 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:22:32,120 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:22:33,121 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:22:33,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:38,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:41,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:22:41,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:22:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:22:44,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:49,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:54,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:22:56,620 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:22:56,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:22:56,896 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:22:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:23:00,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:05,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:10,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:11,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:23:11,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:23:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:23:15,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:20,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:25,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:26,622 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:23:26,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:23:26,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:23:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:23:31,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:36,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:41,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:41,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:23:41,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:23:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:23:45,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:23:45,147 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:23:45,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:23:45,149 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:23:45,166 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:23:46,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:47,167 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:23:51,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:56,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:23:56,622 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:23:56,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:23:56,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:23:56,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:24:02,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:07,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:11,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:24:11,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:24:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:24:13,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:18,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:23,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:26,623 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:24:26,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:24:26,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:24:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:24:29,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:34,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:39,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:41,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:24:41,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:24:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:24:44,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:49,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:54,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:24:56,624 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:24:56,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:24:56,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:24:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:24:58,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:24:58,945 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:24:58,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:24:58,947 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:24:59,211 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:24:59,211 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:24:59,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:04,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:09,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:11,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:25:11,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:25:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:25:15,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:20,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:25,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:26,625 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:25:26,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:25:26,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:25:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:25:31,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:36,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:41,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:41,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:25:41,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:25:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:25:46,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:51,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:56,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:25:56,626 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:25:56,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:25:56,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:25:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:26:02,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:07,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:11,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:26:11,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:26:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:26:13,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:26:13,035 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:26:13,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:26:13,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:13,037 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:26:13,256 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:26:14,256 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:26:18,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:23,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:26,627 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:26:26,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:26:26,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:26:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:26:28,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:33,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:38,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:41,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:26:41,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:26:41,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:26:44,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:49,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:54,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:26:56,629 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:26:56,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:26:56,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:26:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:26:59,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:04,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:09,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:11,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:27:11,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:27:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:27:15,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:20,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:25,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:26,629 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:27:26,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:27:26,786 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:27:26,786 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:27:26,787 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:27:26,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:27:26,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:27:26,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:27:27,301 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:27:28,302 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:27:31,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:36,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:41,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:41,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:27:41,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:27:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:27:47,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:52,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:27:56,630 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:27:56,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:27:56,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:27:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:27:58,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:03,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:08,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:11,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:28:11,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:28:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:28:13,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:18,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:23,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:26,632 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:28:26,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:28:26,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:28:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:28:29,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:34,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:39,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:40,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:28:40,606 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:28:40,606 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:28:40,607 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:28:41,346 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:28:41,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:28:41,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:28:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:28:42,347 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:28:44,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:49,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:54,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:28:56,632 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:28:56,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:28:56,899 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:28:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:28:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:04,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:09,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:11,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:29:11,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:29:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:29:15,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:20,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:25,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:26,633 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:29:26,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:29:26,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:29:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:29:31,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:36,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:41,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:41,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:29:41,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:29:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:29:46,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:51,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:54,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:29:54,414 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:29:54,414 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:29:54,415 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:29:55,391 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:29:56,391 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:29:56,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:29:56,634 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:29:56,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:29:56,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:29:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:30:02,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:07,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:11,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:30:11,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:30:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:30:13,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:18,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:23,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:26,635 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:30:26,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:30:26,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:30:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:30:28,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:33,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:38,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:41,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:30:41,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:30:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:30:43,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:48,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:53,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:30:56,639 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:30:56,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:30:56,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:30:56,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:30:58,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:03,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:08,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:31:08,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:08,169 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:31:08,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:31:08,171 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:31:08,435 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:31:10,436 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:31:11,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:31:11,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:31:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:31:14,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:19,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:24,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:26,640 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:31:26,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:31:26,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:31:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:31:30,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:35,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:40,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:41,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:31:41,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:31:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:31:46,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:51,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:56,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:31:56,641 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:31:56,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:31:56,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:31:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:32:02,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:07,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:11,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:32:11,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:32:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:32:12,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:17,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:21,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:32:21,939 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:32:21,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:32:21,941 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:32:22,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:32:22,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:22,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:32:24,482 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:32:25,483 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:32:26,642 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:32:26,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:32:26,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:32:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:32:28,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:28,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:32:33,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:38,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:41,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:32:41,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:32:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:32:43,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:48,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:53,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:32:56,643 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:32:56,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:32:56,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:32:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:32:59,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:04,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:09,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:11,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:33:11,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:33:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:33:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:19,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:24,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:26,644 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:33:26,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:33:26,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:33:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:33:30,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:35,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:39,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:33:39,788 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:33:39,789 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:33:39,790 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:33:40,527 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:33:40,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:41,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:33:41,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:33:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:33:42,528 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:33:46,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:51,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:56,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:33:56,646 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:33:56,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:33:56,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:33:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:34:01,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:06,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:11,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:11,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:34:11,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:34:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:34:17,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:22,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:26,646 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:34:26,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:34:26,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:34:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:34:27,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:32,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:37,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:41,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:34:41,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:34:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:34:43,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:48,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:53,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:34:53,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:34:53,803 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:34:53,803 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:34:53,805 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:34:54,570 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:34:56,571 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:34:56,647 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:34:56,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:34:56,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:34:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:34:59,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:04,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:09,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:11,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:35:11,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:35:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:35:14,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:19,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:24,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:26,648 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:35:26,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:35:26,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:35:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:35:29,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:34,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:39,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:41,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:35:41,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:35:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:35:44,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:49,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:54,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:35:56,649 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:35:56,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:35:56,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:35:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:36:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:05,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:07,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:36:07,559 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:36:07,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:36:07,560 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:36:07,612 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:36:08,613 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:36:10,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:11,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:36:11,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:36:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:36:16,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:21,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:26,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:26,650 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:36:26,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:36:26,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:36:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:36:32,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:37,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:41,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:36:41,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:36:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:36:42,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:47,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:52,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:36:56,651 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:36:56,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:36:56,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:36:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:36:57,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:02,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:07,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:11,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:37:11,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:37:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:37:13,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:18,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:21,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:37:21,432 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:37:21,432 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:37:21,433 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:37:21,656 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:37:22,656 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:37:23,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:26,652 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:37:26,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:37:26,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:37:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:37:29,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:34,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:39,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:41,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:37:41,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:37:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:37:44,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:49,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:54,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:37:56,653 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:37:56,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:37:56,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:37:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:37:59,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:04,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:09,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:11,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:38:11,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:38:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:38:15,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:20,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:25,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:26,654 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:38:26,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:38:26,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:38:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:38:31,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:35,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:38:35,131 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:38:35,132 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:38:35,133 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:38:35,699 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:38:36,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:36,700 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:38:41,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:41,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:38:41,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:38:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:38:47,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:52,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:38:56,655 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:38:56,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:38:56,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:38:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:38:58,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:03,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:08,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:11,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:39:11,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:39:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:39:14,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:19,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:24,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:26,656 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:39:26,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:39:26,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:39:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:39:30,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:35,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:40,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:41,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:39:41,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:39:41,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:39:45,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:49,005 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:39:49,007 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:39:49,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:39:49,008 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:39:49,742 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:39:50,743 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:39:51,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:56,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:39:56,656 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:39:56,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:39:56,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:39:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:40:01,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:06,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:11,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:11,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:40:11,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:40:11,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:40:16,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:21,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:26,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:26,657 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:40:26,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:40:26,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:40:26,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:40:32,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:37,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:41,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:40:41,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:40:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:40:42,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:47,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:52,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:40:56,658 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:40:56,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:40:56,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:40:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:40:58,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:03,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:41:03,055 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:41:03,055 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:41:03,056 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:41:03,788 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:41:04,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:04,789 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:41:09,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:11,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:41:11,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:41:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:41:14,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:19,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:24,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:26,659 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:41:26,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:41:26,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:41:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:41:30,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:35,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:40,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:41,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:41:41,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:41:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:41:45,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:50,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:55,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:41:56,661 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:41:56,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:41:56,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:41:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:42:01,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:06,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:11,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:11,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:42:11,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:42:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:42:16,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:16,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:42:16,821 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:42:16,821 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:42:16,823 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:42:16,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:42:18,833 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:42:21,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:26,661 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:42:26,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:42:26,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:42:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:42:27,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:32,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:37,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:41,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:42:41,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:42:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:42:42,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:47,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:52,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:42:56,662 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:42:56,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:42:56,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:42:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:42:58,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:03,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:08,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:11,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:43:11,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:43:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:43:13,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:18,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:23,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:26,663 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:43:26,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:43:26,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:43:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:43:29,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:30,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:43:30,524 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:43:30,524 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:43:30,525 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:43:30,876 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:43:32,877 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:43:34,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:39,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:41,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:43:41,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:43:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:43:45,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:50,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:55,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:43:56,664 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:43:56,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:43:56,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:43:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:44:00,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:05,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:10,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:11,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:44:11,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:44:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:44:16,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:21,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:26,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:26,665 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:44:26,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:44:26,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:44:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:44:31,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:36,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:41,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:41,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:44:41,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:44:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:44:44,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:44:44,343 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:44:44,344 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:44:44,345 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:44:44,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:44:44,921 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:44:44,922 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:44:46,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:46,923 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:44:48,924 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:44:50,925 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:44:51,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:56,666 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:44:56,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:44:56,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:44:56,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:44:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:45:02,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:07,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:11,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:45:11,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:45:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:45:13,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:18,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:23,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:26,667 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:45:26,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:45:26,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:45:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:45:29,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:34,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:39,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:41,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:45:41,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:45:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:45:44,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:49,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:54,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:45:56,668 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:45:56,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:45:56,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:45:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:46:00,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:02,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:46:02,653 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:46:02,654 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:46:02,655 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:46:02,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:46:04,969 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:46:05,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:10,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:11,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:46:11,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:46:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:46:16,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:21,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:26,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:26,669 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:46:26,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:46:26,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:46:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:46:31,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:36,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:41,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:41,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:46:41,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:46:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:46:46,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:51,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:56,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:46:56,670 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:46:56,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:46:56,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:46:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:47:02,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:07,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:11,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:47:11,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:47:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:47:12,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:16,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:47:16,596 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:47:16,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:47:16,598 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:47:17,013 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:47:17,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:19,014 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:47:22,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:26,671 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:47:26,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:47:26,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:47:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:47:28,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:33,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:38,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:41,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:47:41,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:47:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:47:43,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:48,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:53,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:47:56,672 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:47:56,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:47:56,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:47:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:47:59,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:04,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:09,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:11,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:48:11,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:48:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:48:15,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:20,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:25,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:26,698 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:48:26,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:48:26,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:48:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:48:30,696 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:48:30,698 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:48:30,698 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:48:30,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:30,699 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:48:31,058 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:48:33,059 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:48:35,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:40,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:41,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:48:41,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:48:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:48:46,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:51,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:56,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:48:56,699 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:48:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:48:56,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:48:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:49:02,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:07,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:11,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:49:11,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:49:11,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:49:12,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:17,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:22,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:26,701 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:49:26,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:49:26,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:49:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:49:27,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:32,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:37,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:49:41,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:49:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:49:43,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:44,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:49:44,661 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:49:44,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:49:44,663 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:49:45,102 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:49:45,102 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:49:48,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:53,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:49:56,701 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:49:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:49:56,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:49:56,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:49:59,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:04,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:09,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:50:11,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:50:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:50:15,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:20,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:25,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:26,702 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:50:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:50:26,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:50:26,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:50:30,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:35,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:40,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:50:41,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:50:41,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:50:45,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:50,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:55,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:50:56,702 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:50:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:50:56,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:50:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:50:58,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:50:58,571 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:50:58,571 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:50:58,572 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:50:59,145 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:50:59,145 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:51:00,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:05,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:10,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:11,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:51:11,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:51:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:51:16,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:21,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:26,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:26,703 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:51:26,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:51:26,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:51:26,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:51:31,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:36,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:41,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:51:41,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:51:41,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:51:47,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:52,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:51:56,704 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:51:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:51:56,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:51:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:51:57,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:02,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:07,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:11,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:52:11,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:52:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:52:12,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:52:12,378 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:52:12,378 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:52:12,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:12,379 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:52:13,190 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:52:13,191 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:52:17,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:22,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:26,705 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:52:26,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:52:26,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:52:26,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:52:28,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:33,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:38,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:41,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:52:41,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:52:41,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:52:44,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:49,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:54,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:52:56,707 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:52:56,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:52:56,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:52:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:52:59,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:04,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:09,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:11,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:53:11,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:53:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:53:15,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:20,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:25,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:26,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:53:26,154 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:53:26,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:53:26,155 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:53:26,230 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:53:26,707 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:53:26,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:53:26,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:53:26,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:53:27,231 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:53:31,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:36,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:41,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:41,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:53:41,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:53:41,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:53:47,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:52,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:53:56,709 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:53:56,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:53:56,909 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:53:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:53:57,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:02,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:07,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:11,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:54:11,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:54:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:54:13,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:18,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:23,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:26,709 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:54:26,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:54:26,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:54:26,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:54:28,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:33,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:38,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:40,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:54:40,182 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:54:40,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:54:40,183 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:54:40,273 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:54:41,274 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:54:41,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:54:41,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:54:41,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:54:43,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:48,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:53,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:54:56,710 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:54:56,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:54:56,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:54:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:54:59,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:04,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:09,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:11,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:55:11,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:55:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:55:14,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:19,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:24,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:26,711 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:55:26,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:55:26,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:55:26,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:55:29,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:34,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:39,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:41,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:55:41,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:55:41,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:55:44,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:49,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:54,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:55:54,151 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:55:54,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:55:54,153 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:55:54,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:55:54,318 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:55:55,318 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:55:56,712 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:55:56,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:55:56,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:55:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:56:00,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:05,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:10,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:11,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:56:11,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:56:11,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:56:15,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:20,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:25,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:26,713 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:56:26,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:56:26,910 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:56:26,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:56:30,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:35,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:40,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:41,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:56:41,911 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:56:41,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:56:45,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:50,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:55,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:56:56,714 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:56:56,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:56:56,911 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:56:56,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:57:01,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:06,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:08,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:57:08,011 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:57:08,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:57:08,013 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:57:08,360 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:57:08,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:57:09,361 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:57:11,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:11,362 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:57:11,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:57:11,957 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:57:12,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:57:13,364 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:57:17,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:22,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:26,715 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:57:26,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:57:26,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:57:26,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:57:28,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:33,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:38,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:41,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:57:41,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:57:41,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:57:43,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:48,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:53,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:57:56,716 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:57:56,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:57:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:57:56,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:57:59,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:04,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:09,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:11,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:58:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:58:11,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:58:14,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:19,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:24,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:25,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:58:25,742 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:58:25,742 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:58:25,744 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:58:26,429 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:58:26,717 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:58:26,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:58:26,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:58:26,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:58:27,430 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:58:30,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:35,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:40,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:41,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:58:41,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:58:41,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:58:45,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:50,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:55,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:58:56,718 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:58:56,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:58:56,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:58:56,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:59:00,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:05,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:10,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:11,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:59:11,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:59:11,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:59:16,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:21,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:26,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:26,719 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:59:26,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:59:26,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:59:26,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:59:32,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:37,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:39,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 10:59:39,460 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 10:59:39,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 10:59:39,462 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 10:59:39,472 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 10:59:41,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 10:59:41,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:59:41,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:59:41,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:59:43,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:48,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:53,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 10:59:56,720 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 10:59:56,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 10:59:56,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 10:59:56,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 10:59:58,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:03,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:08,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:11,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:00:11,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:00:11,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:00:13,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:18,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:23,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:26,721 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:00:26,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:00:26,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:00:26,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:00:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:34,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:39,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:41,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:00:41,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:00:41,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:00:45,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:50,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:53,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:00:53,127 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:00:53,128 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:00:53,130 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:00:53,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:00:53,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:00:56,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:00:56,722 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:00:56,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:00:56,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:00:56,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:01:01,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:06,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:11,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:11,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:01:11,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:01:11,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:01:16,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:21,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:26,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:26,722 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:01:26,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:01:26,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:01:26,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:01:32,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:37,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:41,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:01:41,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:01:41,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:01:42,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:47,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:52,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:01:56,724 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:01:56,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:01:56,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:01:56,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:01:58,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:03,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:07,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:02:07,143 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:02:07,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:02:07,145 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:02:07,561 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:02:08,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:08,562 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:02:11,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:02:11,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:02:11,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:02:13,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:18,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:23,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:26,724 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:02:26,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:02:26,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:02:26,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:02:28,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:33,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:38,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:41,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:02:41,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:02:41,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:02:44,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:49,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:54,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:02:56,725 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:02:56,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:02:56,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:02:56,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:03:00,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:05,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:10,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:11,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:03:11,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:03:11,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:03:16,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:21,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:03:21,036 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:03:21,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:03:21,039 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:03:21,605 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:03:22,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:22,606 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:03:26,726 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:03:26,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:03:26,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:03:26,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:03:27,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:32,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:37,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:41,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:03:41,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:03:41,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:03:42,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:47,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:52,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:03:56,727 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:03:56,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:03:56,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:03:56,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:03:58,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:03,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:08,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:11,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:04:11,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:04:11,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:04:14,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:19,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:24,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:26,728 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:04:26,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:04:26,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:04:26,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:04:29,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:34,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:34,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:04:34,905 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:04:34,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:04:34,907 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:04:35,650 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:04:36,650 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:04:39,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:41,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:04:41,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:04:41,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:04:45,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:50,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:55,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:04:56,730 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:04:56,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:04:56,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:04:56,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:05:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:05,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:10,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:11,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:05:11,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:05:11,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:05:15,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:20,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:25,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:26,730 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:05:26,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:05:26,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:05:26,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:05:31,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:36,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:41,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:41,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:05:41,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:05:41,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:05:46,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:48,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:05:48,850 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:05:48,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:05:48,853 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:05:49,694 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:05:50,695 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:05:51,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:56,731 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:05:56,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:05:56,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:05:56,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:05:56,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:06:02,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:07,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:11,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:06:11,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:06:11,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:06:12,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:17,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:22,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:26,733 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:06:26,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:06:26,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:06:26,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:06:28,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:33,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:38,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:41,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:06:41,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:06:41,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:06:43,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:48,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:53,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:06:56,733 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:06:56,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:06:56,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:06:56,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:06:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:02,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:07:02,620 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:07:02,620 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:07:02,622 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:07:02,735 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:07:04,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:04,737 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:07:09,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:11,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:07:11,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:07:11,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:07:15,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:20,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:25,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:26,734 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:07:26,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:07:26,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:07:26,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:07:31,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:36,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:41,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:41,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:07:41,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:07:41,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:07:46,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:51,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:56,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:07:56,735 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:07:56,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:07:56,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:07:56,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:08:02,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:07,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:11,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:08:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:08:11,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:08:12,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:16,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:08:16,505 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:08:16,505 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:08:16,507 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:08:16,778 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:08:17,507 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:18,780 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:08:22,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:26,736 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:08:26,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:08:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:08:26,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:08:28,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:33,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:38,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:41,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:08:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:08:41,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:08:44,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:49,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:54,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:08:56,737 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:08:56,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:08:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:08:56,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:08:59,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:04,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:09,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:11,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:09:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:09:11,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:09:15,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:20,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:25,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:26,738 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:09:26,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:09:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:09:26,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:09:30,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:30,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:09:30,363 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:09:30,364 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:09:30,365 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:09:30,823 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:09:30,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:09:32,825 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:09:34,826 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:09:35,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:36,827 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:09:40,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:41,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:09:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:09:41,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:09:46,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:51,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:56,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:09:56,738 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:09:56,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:09:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:09:56,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:10:02,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:07,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:11,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:10:11,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:10:11,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:10:12,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:17,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:22,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:26,739 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:10:26,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:10:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:10:26,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:10:28,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:33,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:38,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:41,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:10:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:10:41,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:10:43,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:48,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:48,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:10:48,216 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:10:48,217 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:10:48,218 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:10:48,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:10:50,871 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:10:53,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:10:56,740 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:10:56,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:10:56,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:10:56,958 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:10:59,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:04,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:09,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:11,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:11:11,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:11:11,958 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:11:14,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:19,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:24,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:26,741 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:11:26,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:11:26,958 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:11:27,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:11:29,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:34,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:39,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:41,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:11:41,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:11:41,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:11:45,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:50,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:55,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:11:56,742 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:11:56,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:11:56,958 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:11:57,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:12:00,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:02,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:12:02,191 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:12:02,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:12:02,192 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:12:02,913 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:12:04,914 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:12:05,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:10,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:11,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:12:11,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:12:11,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:12:16,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:21,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:26,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:26,743 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:12:26,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:12:26,958 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:12:27,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:12:31,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:36,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:41,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:41,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:12:41,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:12:42,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:12:46,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:51,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:56,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:12:56,744 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:12:56,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:12:56,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:12:57,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:13:02,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:07,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:11,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:13:11,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:13:12,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:13:12,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:16,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:13:16,059 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:13:16,059 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:13:16,060 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:13:16,957 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:13:16,957 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:13:18,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:23,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:26,745 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:13:26,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:13:26,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:13:26,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:13:28,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:33,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:38,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:41,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:13:41,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:13:42,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:13:43,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:48,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:53,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:13:56,746 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:13:56,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:13:56,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:13:57,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:13:59,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:04,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:09,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:11,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:14:11,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:14:12,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:14:15,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:20,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:25,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:26,747 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:14:26,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:14:26,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:14:27,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:14:29,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:14:29,825 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:14:29,825 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:14:29,826 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:14:29,999 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:14:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:31,000 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:14:35,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:40,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:41,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:14:41,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:14:41,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:14:46,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:51,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:56,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:14:56,748 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:14:56,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:14:56,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:14:57,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:15:02,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:07,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:11,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:15:11,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:15:12,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:15:13,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:18,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:23,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:26,749 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:15:26,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:15:26,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:15:27,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:15:29,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:34,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:39,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:41,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:15:41,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:15:42,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:15:43,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:15:43,511 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:15:43,511 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:15:43,512 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:15:44,043 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:15:44,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:45,044 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:15:49,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:54,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:15:56,750 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:15:56,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:15:56,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:15:57,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:16:00,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:05,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:10,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:11,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:16:11,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:16:12,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:16:16,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:21,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:26,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:26,751 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:16:26,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:16:26,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:16:27,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:16:32,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:37,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:41,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:16:41,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:16:42,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:16:42,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:47,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:52,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:56,752 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:16:56,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:16:56,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:16:57,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:16:57,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:16:57,379 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:16:57,379 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:16:57,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:16:57,381 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:16:58,087 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:16:59,088 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:17:02,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:07,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:11,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:17:11,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:17:11,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:17:13,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:18,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:23,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:26,753 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:17:26,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:17:26,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:17:27,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:17:29,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:34,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:39,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:41,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:17:41,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:17:42,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:17:44,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:49,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:54,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:17:56,754 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:17:56,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:17:56,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:17:57,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:17:59,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:04,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:09,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:11,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:18:11,285 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:18:11,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:18:11,287 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:18:11,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:18:11,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:18:11,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:18:12,130 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:18:13,131 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:18:15,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:20,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:25,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:26,755 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:18:26,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:18:26,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:18:27,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:18:31,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:36,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:41,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:41,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:18:41,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:18:42,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:18:46,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:51,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:56,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:18:56,756 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:18:56,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:18:56,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:18:57,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:19:02,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:07,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:11,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:19:11,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:19:12,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:19:13,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:18,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:23,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:25,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:19:25,117 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:19:25,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:19:25,119 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:19:25,174 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:19:26,756 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:19:26,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:19:26,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:19:26,963 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:19:27,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:19:28,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:33,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:38,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:41,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:19:41,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:19:41,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:19:43,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:48,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:53,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:19:56,757 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:19:56,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:19:56,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:19:57,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:19:59,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:04,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:09,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:12,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:20:12,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:20:12,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:20:14,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:19,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:24,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:26,760 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:20:27,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:20:27,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:20:27,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:20:30,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:35,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:39,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:20:39,071 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:20:39,072 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:20:39,073 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:20:39,218 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:20:41,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:41,219 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:20:42,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:20:42,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:20:42,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:20:46,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:51,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:56,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:20:56,759 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:20:57,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:20:57,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:20:57,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:21:01,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:06,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:11,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:12,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:21:12,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:21:12,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:21:17,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:22,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:26,761 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:21:27,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:21:27,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:21:27,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:21:27,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:32,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:37,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:42,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:21:42,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:21:42,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:21:43,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:48,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:52,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:21:52,782 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:21:52,783 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:21:52,785 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:21:53,261 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:21:53,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:21:53,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:21:55,263 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:21:56,761 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:21:57,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:21:57,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:21:57,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:21:57,264 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:21:59,265 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:21:59,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:04,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:09,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:12,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:22:12,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:22:12,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:22:15,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:20,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:25,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:26,762 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:22:27,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:22:27,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:22:27,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:22:31,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:36,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:41,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:42,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:22:42,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:22:42,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:22:46,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:51,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:56,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:22:56,764 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:22:57,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:22:57,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:22:57,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:23:02,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:07,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:10,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:23:10,811 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:23:10,811 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:23:10,812 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:23:11,307 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:23:12,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:23:12,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:23:12,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:23:12,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:13,309 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:23:17,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:22,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:26,765 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:23:27,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:23:27,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:23:27,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:23:27,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:32,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:37,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:42,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:23:42,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:23:42,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:23:43,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:48,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:53,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:23:56,766 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:23:57,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:23:57,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:23:57,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:23:59,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:04,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:09,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:12,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:24:12,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:24:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:24:15,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:20,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:24,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:24:24,537 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:24:24,537 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:24:24,538 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:24:25,350 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:24:25,350 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:24:25,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:26,767 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:24:27,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:24:27,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:24:27,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:24:31,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:36,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:41,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:42,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:24:42,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:24:42,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:24:47,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:52,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:24:56,768 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:24:57,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:24:57,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:24:57,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:24:57,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:02,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:07,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:12,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:25:12,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:25:12,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:25:12,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:17,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:22,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:26,769 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:25:27,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:25:27,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:25:27,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:25:28,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:33,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:38,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:38,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:25:38,368 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:25:38,368 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:25:38,369 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:25:38,393 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:25:39,394 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:25:42,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:25:42,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:25:42,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:25:44,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:49,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:54,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:25:56,769 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:25:57,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:25:57,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:25:57,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:25:59,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:04,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:09,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:12,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:26:12,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:26:12,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:26:15,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:20,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:25,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:26,770 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:26:27,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:26:27,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:26:27,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:26:30,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:35,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:40,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:42,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:26:42,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:26:42,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:26:45,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:50,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:52,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:26:52,120 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:26:52,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:26:52,121 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:26:52,436 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:26:53,437 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:26:56,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:26:56,771 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:26:57,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:26:57,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:26:57,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:27:01,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:06,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:11,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:12,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:27:12,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:27:12,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:27:17,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:22,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:26,773 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:27:27,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:27:27,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:27:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:27:27,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:32,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:37,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:42,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:27:42,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:27:42,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:27:43,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:48,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:53,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:27:56,773 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:27:57,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:27:57,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:27:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:27:59,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:04,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:05,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:28:05,686 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:28:05,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:28:05,687 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:28:06,479 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:28:07,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:28:09,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:12,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:28:12,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:28:12,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:28:15,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:20,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:25,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:26,774 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:28:27,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:28:27,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:28:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:28:30,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:35,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:40,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:42,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:28:42,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:28:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:28:46,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:51,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:56,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:28:56,776 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:28:57,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:28:57,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:28:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:29:01,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:06,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:11,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:12,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:29:12,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:29:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:29:17,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:19,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:29:19,411 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:29:19,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:29:19,413 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:29:19,522 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:29:21,523 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:29:22,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:26,776 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:29:27,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:29:27,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:29:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:29:28,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:33,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:38,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:42,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:29:42,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:29:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:29:44,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:49,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:54,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:29:56,778 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:29:57,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:29:57,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:29:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:29:59,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:04,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:09,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:12,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:30:12,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:30:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:30:14,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:19,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:24,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:26,778 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:30:27,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:30:27,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:30:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:30:30,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:33,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:30:33,260 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:30:33,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:30:33,262 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:30:33,565 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:30:33,565 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:30:35,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:40,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:42,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:30:42,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:30:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:30:45,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:50,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:55,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:30:56,779 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:30:57,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:30:57,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:30:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:31:01,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:06,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:11,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:12,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:31:12,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:31:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:31:16,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:21,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:26,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:26,780 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:31:27,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:31:27,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:31:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:31:32,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:37,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:42,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:31:42,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:31:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:31:42,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:46,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:31:46,990 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:31:46,990 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:31:46,992 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:31:47,607 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:31:47,607 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:31:48,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:53,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:31:56,781 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:31:57,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:31:57,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:31:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:31:58,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:03,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:08,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:12,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:32:12,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:32:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:32:13,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:18,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:23,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:26,782 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:32:27,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:32:27,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:32:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:32:29,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:34,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:39,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:42,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:32:42,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:32:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:32:44,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:49,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:54,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:32:56,783 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:32:57,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:32:57,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:32:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:33:00,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:00,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:33:00,861 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:33:00,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:33:00,862 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:33:01,651 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:33:01,651 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:33:05,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:10,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:12,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:33:12,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:33:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:33:16,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:21,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:26,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:26,784 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:33:27,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:33:27,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:33:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:33:32,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:37,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:42,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:33:42,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:33:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:33:43,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:48,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:53,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:33:56,784 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:33:57,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:33:57,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:33:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:33:58,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:03,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:08,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:12,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:34:12,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:34:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:34:14,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:14,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:34:14,610 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:34:14,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:34:14,612 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:34:14,695 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:34:15,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:34:15,696 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:34:17,698 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:34:19,699 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:34:19,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:24,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:26,785 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:34:27,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:34:27,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:34:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:34:30,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:35,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:40,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:42,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:34:42,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:34:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:34:46,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:51,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:56,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:34:56,787 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:34:57,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:34:57,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:34:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:35:01,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:06,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:11,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:12,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:35:12,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:35:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:35:17,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:22,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:26,787 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:35:27,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:35:27,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:35:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:35:27,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:32,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:32,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:35:32,544 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:35:32,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:35:32,546 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:35:32,741 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:35:34,743 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:35:37,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:42,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:35:42,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:35:42,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:35:43,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:48,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:53,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:35:56,788 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:35:57,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:35:57,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:35:57,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:35:59,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:04,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:09,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:12,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:36:12,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:36:12,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:36:14,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:19,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:24,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:26,790 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:36:27,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:36:27,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:36:27,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:36:29,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:34,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:39,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:42,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:36:42,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:36:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:36:45,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:46,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:36:46,511 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:36:46,511 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:36:46,513 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:36:46,784 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:36:48,785 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:36:50,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:55,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:36:56,790 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:36:57,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:36:57,032 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:36:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:37:01,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:06,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:11,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:12,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:37:12,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:37:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:37:16,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:21,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:26,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:26,791 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:37:27,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:37:27,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:37:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:37:32,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:37,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:42,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:37:42,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:37:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:37:42,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:47,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:52,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:37:56,792 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:37:57,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:37:57,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:37:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:37:58,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:00,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:38:00,330 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:38:00,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:38:00,331 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:38:00,829 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:38:02,831 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:38:03,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:08,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:12,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:38:12,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:38:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:38:14,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:19,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:24,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:26,793 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:38:27,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:38:27,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:38:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:38:30,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:35,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:40,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:42,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:38:42,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:38:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:38:46,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:51,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:56,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:38:56,794 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:38:57,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:38:57,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:38:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:39:01,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:06,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:11,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:12,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:39:12,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:39:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:39:14,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:39:14,030 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:39:14,030 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:39:14,032 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:39:14,873 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:39:16,875 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:39:17,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:22,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:26,795 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:39:27,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:39:27,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:39:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:39:27,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:32,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:37,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:42,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:39:42,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:39:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:39:42,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:47,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:52,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:39:56,796 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:39:57,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:39:57,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:39:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:39:58,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:03,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:08,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:12,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:40:12,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:40:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:40:13,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:18,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:23,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:26,797 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:40:27,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:40:27,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:40:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:40:27,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:40:27,872 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:40:27,872 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:40:27,874 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:40:27,916 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:40:28,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:28,917 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:40:33,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:38,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:42,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:40:42,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:40:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:40:44,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:49,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:54,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:40:56,797 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:40:57,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:40:57,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:40:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:40:59,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:04,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:09,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:12,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:41:12,034 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:41:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:41:15,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:20,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:25,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:26,798 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:41:27,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:41:27,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:41:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:41:31,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:36,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:41,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:41,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:41:41,772 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:41:41,772 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:41:41,774 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:41:41,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:41:42,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:41:42,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:41:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:41:42,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:41:47,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:52,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:41:56,799 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:41:57,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:41:57,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:41:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:41:58,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:03,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:08,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:12,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:42:12,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:42:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:42:14,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:19,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:24,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:26,800 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:42:27,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:42:27,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:42:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:42:29,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:34,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:39,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:42,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:42:42,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:42:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:42:44,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:49,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:54,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:42:55,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:42:55,522 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:42:55,522 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:42:55,523 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:42:56,005 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:42:56,802 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:42:57,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:42:57,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:42:57,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:42:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:43:00,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:05,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:10,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:12,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:43:12,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:43:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:43:16,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:21,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:26,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:26,802 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:43:27,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:43:27,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:43:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:43:31,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:36,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:41,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:42,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:43:42,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:43:42,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:43:47,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:52,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:43:56,803 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:43:57,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:43:57,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:43:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:43:58,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:03,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:08,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:09,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:44:09,319 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:44:09,320 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:44:09,321 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:44:10,049 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:44:11,050 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:44:12,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:44:12,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:44:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:44:13,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:18,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:23,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:26,804 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:44:27,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:44:27,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:44:27,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:44:28,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:33,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:38,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:42,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:44:42,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:44:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:44:43,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:48,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:53,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:44:56,805 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:44:57,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:44:57,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:44:57,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:44:59,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:04,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:09,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:12,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:45:12,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:45:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:45:15,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:20,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:23,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:45:23,243 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:45:23,243 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:45:23,244 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:45:24,093 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:45:25,094 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:45:25,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:26,806 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:45:27,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:45:27,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:45:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:45:30,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:35,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:40,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:42,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:45:42,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:45:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:45:46,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:51,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:56,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:45:56,807 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:45:57,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:45:57,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:45:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:46:01,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:06,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:11,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:12,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:46:12,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:46:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:46:16,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:21,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:26,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:26,808 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:46:27,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:46:27,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:46:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:46:31,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:36,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:36,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:46:36,918 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:46:36,918 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:46:36,919 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:46:37,138 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:46:37,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:46:39,139 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:46:41,141 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:46:41,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:42,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:46:42,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:46:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:46:43,142 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:46:47,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:52,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:46:56,809 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:46:57,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:46:57,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:46:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:46:57,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:02,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:07,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:12,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:47:12,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:47:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:47:13,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:18,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:23,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:26,810 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:47:27,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:47:27,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:47:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:47:28,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:33,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:38,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:42,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:47:42,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:47:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:47:44,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:49,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:54,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:47:54,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:47:54,829 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:47:54,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:47:54,831 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:47:55,185 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:47:56,810 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:47:57,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:47:57,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:47:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:47:57,186 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:48:00,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:05,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:10,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:12,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:48:12,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:48:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:48:15,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:20,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:25,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:26,811 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:48:27,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:48:27,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:48:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:48:31,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:36,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:41,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:42,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:48:42,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:48:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:48:46,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:51,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:56,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:48:56,812 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:48:57,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:48:57,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:48:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:49:01,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:06,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:08,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:49:08,483 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:49:08,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:49:08,484 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:49:09,229 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:49:11,230 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:49:11,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:12,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:49:12,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:49:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:49:17,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:22,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:26,813 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:49:27,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:49:27,038 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:49:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:49:27,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:32,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:37,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:42,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:49:42,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:49:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:49:43,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:48,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:53,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:49:56,814 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:49:57,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:49:57,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:49:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:49:58,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:03,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:08,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:12,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:50:12,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:50:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:50:14,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:19,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:22,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:50:22,296 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:50:22,296 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:50:22,298 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:50:23,272 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:50:23,273 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:50:24,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:26,815 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:50:27,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:50:27,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:50:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:50:30,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:35,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:40,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:42,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:50:42,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:50:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:50:45,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:50,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:55,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:50:56,816 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:50:57,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:50:57,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:50:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:51:01,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:06,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:11,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:12,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:51:12,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:51:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:51:17,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:22,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:26,817 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:51:27,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:51:27,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:51:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:51:27,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:32,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:35,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:51:35,967 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:51:35,967 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:51:35,969 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:51:36,317 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:51:37,318 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:51:38,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:42,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:51:42,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:51:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:51:43,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:48,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:53,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:51:56,818 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:51:57,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:51:57,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:51:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:51:59,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:04,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:09,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:12,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:52:12,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:52:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:52:14,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:19,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:24,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:26,819 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:52:27,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:52:27,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:52:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:52:30,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:35,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:40,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:42,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:52:42,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:52:42,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:52:46,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:49,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:52:49,724 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:52:49,724 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:52:49,726 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:52:50,362 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:52:51,363 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:52:51,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:56,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:52:56,821 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:52:57,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:52:57,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:52:57,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:53:02,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:07,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:12,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:53:12,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:53:12,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:53:12,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:17,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:22,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:26,822 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:53:27,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:53:27,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:53:27,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:53:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:33,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:38,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:42,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:53:42,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:53:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:53:44,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:49,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:54,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:53:56,823 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:53:57,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:53:57,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:53:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:53:59,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:03,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:54:03,615 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:54:03,615 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:54:03,616 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:54:04,407 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:54:04,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:05,408 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:54:09,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:12,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:54:12,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:54:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:54:15,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:20,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:25,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:26,824 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:54:27,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:54:27,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:54:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:54:31,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:36,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:41,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:42,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:54:42,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:54:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:54:47,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:52,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:54:56,825 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:54:57,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:54:57,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:54:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:54:57,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:02,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:07,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:12,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:55:12,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:55:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:55:12,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:17,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:17,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:55:17,592 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:55:17,592 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:55:17,594 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:55:18,453 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:55:19,453 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:55:22,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:26,826 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:55:27,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:55:27,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:55:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:55:28,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:33,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:38,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:42,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:55:42,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:55:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:55:43,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:48,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:53,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:55:56,827 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:55:57,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:55:57,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:55:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:55:59,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:04,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:09,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:12,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:56:12,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:56:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:56:14,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:19,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:24,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:26,828 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:56:27,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:56:27,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:56:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:56:30,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:31,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:56:31,294 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:56:31,294 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:56:31,295 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:56:31,496 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:56:33,497 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:56:35,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:40,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:42,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:56:42,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:56:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:56:46,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:51,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:56,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:56:56,829 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:56:57,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:56:57,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:56:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:57:02,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:07,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:12,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:57:12,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:57:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:57:13,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:18,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:23,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:26,829 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:57:27,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:57:27,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:57:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:57:28,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:33,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:38,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:42,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:57:42,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:57:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:57:44,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:45,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:57:45,171 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:57:45,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:57:45,173 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:57:45,541 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:57:47,542 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:57:50,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:55,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:57:56,830 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:57:57,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:57:57,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:57:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:58:01,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:06,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:11,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:12,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:58:12,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:58:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:58:16,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:21,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:26,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:26,832 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:58:27,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:58:27,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:58:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:58:32,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:37,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:42,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:58:42,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:58:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:58:43,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:48,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:53,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:56,833 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:58:57,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:58:57,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:58:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:58:58,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:58:58,904 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 11:58:58,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 11:58:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:58:58,906 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 11:58:59,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 11:58:59,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 11:58:59,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:59:01,595 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:59:03,596 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 11:59:03,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:08,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:12,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:59:12,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:59:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:59:14,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:19,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:24,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:26,833 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:59:27,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:59:27,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:59:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:59:29,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:34,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:39,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:42,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:59:42,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:59:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 11:59:45,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:50,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:55,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 11:59:56,834 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 11:59:57,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 11:59:57,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 11:59:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:00:01,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:06,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:11,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:12,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:00:12,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:00:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:00:16,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:00:16,635 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:00:16,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:00:16,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:16,637 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:00:16,641 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:00:17,642 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:00:21,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:26,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:26,835 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:00:27,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:00:27,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:00:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:00:32,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:37,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:42,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:00:42,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:00:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:00:43,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:48,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:53,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:00:56,837 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:00:57,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:00:57,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:00:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:00:58,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:03,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:08,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:12,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:01:12,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:01:12,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:01:14,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:19,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:24,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:26,837 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:01:27,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:01:27,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:01:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:01:30,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:30,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:01:30,639 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:01:30,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:01:30,641 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:01:30,687 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:01:31,687 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:01:35,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:40,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:42,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:01:42,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:01:42,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:01:46,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:51,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:56,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:01:56,838 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:01:57,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:01:57,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:01:57,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:02:01,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:06,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:11,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:12,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:02:12,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:02:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:02:16,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:21,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:26,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:26,839 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:02:27,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:02:27,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:02:27,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:02:32,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:37,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:42,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:02:42,044 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:02:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:02:43,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:44,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:02:44,449 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:02:44,449 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:02:44,450 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:02:44,740 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:02:45,741 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:02:48,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:53,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:02:56,840 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:02:57,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:02:57,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:02:57,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:02:59,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:04,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:09,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:12,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:03:12,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:03:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:03:15,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:20,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:25,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:26,841 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:03:27,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:03:27,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:03:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:03:30,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:35,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:40,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:42,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:03:42,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:03:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:03:45,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:50,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:55,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:03:56,842 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:03:57,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:03:57,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:03:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:03:58,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:03:58,208 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:03:58,209 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:03:58,210 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:03:58,791 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:03:59,791 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:04:01,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:06,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:11,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:12,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:04:12,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:04:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:04:17,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:22,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:26,843 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:04:27,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:04:27,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:04:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:04:27,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:32,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:37,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:42,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:04:42,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:04:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:04:43,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:48,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:53,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:04:56,844 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:04:57,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:04:57,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:04:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:04:58,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:03,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:08,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:11,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:05:11,951 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:05:11,952 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:05:11,953 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:05:12,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:05:12,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:05:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:05:12,838 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:05:13,839 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:05:14,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:19,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:24,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:26,845 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:05:27,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:05:27,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:05:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:05:30,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:35,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:40,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:42,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:05:42,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:05:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:05:45,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:50,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:55,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:05:56,847 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:05:57,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:05:57,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:05:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:06:01,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:06,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:11,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:12,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:06:12,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:06:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:06:16,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:21,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:25,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:06:25,793 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:06:25,794 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:06:25,795 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:06:25,884 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:06:26,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:26,847 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:06:27,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:06:27,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:06:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:06:27,885 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:06:32,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:37,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:42,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:06:42,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:06:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:06:43,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:48,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:53,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:06:56,848 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:06:57,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:06:57,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:06:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:06:59,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:04,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:09,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:12,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:07:12,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:07:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:07:14,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:19,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:24,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:26,849 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:07:27,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:07:27,047 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:07:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:07:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:35,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:39,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:07:39,592 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:07:39,592 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:07:39,593 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:07:39,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:07:39,930 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:07:40,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:42,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:07:42,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:07:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:07:46,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:51,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:56,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:07:56,850 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:07:57,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:07:57,047 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:07:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:08:02,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:07,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:12,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:08:12,047 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:08:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:08:12,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:17,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:22,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:26,851 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:08:27,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:08:27,047 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:08:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:08:28,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:33,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:38,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:42,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:08:42,047 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:08:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:08:43,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:48,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:53,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:08:53,261 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:08:53,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:08:53,263 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:08:53,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:08:53,975 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:08:53,975 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:08:56,852 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:08:57,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:08:57,047 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:08:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:08:59,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:04,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:09,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:12,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:09:12,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:09:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:09:15,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:20,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:25,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:26,853 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:09:27,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:09:27,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:09:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:09:30,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:35,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:40,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:42,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:09:42,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:09:42,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:09:46,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:51,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:56,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:09:56,854 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:09:57,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:09:57,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:09:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:10:01,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:06,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:07,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:10:07,098 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:10:07,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:10:07,100 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:10:08,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:10:08,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:10:12,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:12,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:10:12,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:10:12,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:10:17,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:22,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:26,855 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:10:27,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:10:27,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:10:27,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:10:27,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:32,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:37,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:42,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:10:42,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:10:42,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:10:43,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:48,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:53,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:10:56,856 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:10:57,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:10:57,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:10:57,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:10:58,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:03,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:08,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:12,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:11:12,049 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:11:12,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:11:14,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:19,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:20,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:11:20,817 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:11:20,817 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:11:20,818 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:11:21,065 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:11:21,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:11:22,066 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:11:24,067 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:11:24,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:26,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:11:26,856 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:11:27,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:11:27,049 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:11:27,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:11:30,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:35,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:40,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:42,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:11:42,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:11:42,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:11:45,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:50,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:55,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:11:56,857 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:11:57,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:11:57,097 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:11:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:12:00,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:05,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:10,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:12,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:12:12,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:12:12,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:12:16,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:21,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:26,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:26,858 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:12:27,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:12:27,097 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:12:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:12:32,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:37,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:38,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:12:38,882 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:12:38,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:12:38,884 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:12:39,114 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:12:41,115 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:12:42,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:12:42,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:12:42,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:12:42,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:47,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:52,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:12:56,861 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:12:57,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:12:57,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:12:57,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:12:57,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:02,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:07,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:12,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:13:12,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:13:12,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:13:13,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:18,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:23,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:26,863 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:13:27,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:13:27,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:13:27,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:13:28,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:33,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:38,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:42,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:13:42,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:13:42,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:13:43,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:48,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:52,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:13:52,747 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:13:52,747 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:13:52,756 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:13:53,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:13:53,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:13:55,159 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:13:56,864 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:13:57,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:13:57,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:13:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:13:59,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:04,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:09,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:12,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:14:12,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:14:12,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:14:14,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:19,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:24,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:26,865 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:14:27,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:14:27,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:14:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:14:29,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:34,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:39,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:42,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:14:42,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:14:42,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:14:44,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:49,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:54,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:14:56,866 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:14:57,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:14:57,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:14:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:14:59,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:04,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:06,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:15:06,502 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:15:06,502 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:15:06,503 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:15:07,203 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:15:09,204 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:15:09,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:12,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:15:12,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:15:12,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:15:15,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:20,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:25,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:26,867 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:15:27,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:15:27,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:15:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:15:31,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:36,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:41,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:42,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:15:42,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:15:42,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:15:47,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:52,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:15:56,868 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:15:57,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:15:57,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:15:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:15:57,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:02,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:07,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:12,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:16:12,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:16:12,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:16:12,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:17,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:20,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:16:20,226 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:16:20,226 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:16:20,227 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:16:20,249 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:16:23,250 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:16:23,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:26,869 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:16:27,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:16:27,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:16:27,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:16:28,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:33,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:38,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:42,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:16:42,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:16:42,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:16:44,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:49,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:54,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:16:56,870 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:16:57,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:16:57,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:16:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:17:00,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:05,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:10,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:12,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:17:12,100 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:17:12,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:17:16,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:21,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:26,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:26,871 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:17:27,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:17:27,100 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:17:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:17:32,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:34,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:17:34,157 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:17:34,157 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:17:34,158 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:17:34,294 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:17:35,294 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:17:38,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:42,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:17:42,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:17:42,101 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:17:43,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:48,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:53,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:17:56,872 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:17:57,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:17:57,100 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:17:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:17:58,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:03,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:08,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:12,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:18:12,100 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:18:12,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:18:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:18,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:23,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:26,873 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:18:27,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:18:27,100 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:18:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:18:28,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:33,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:38,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:42,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:18:42,101 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:18:42,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:18:43,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:48,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:18:48,165 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:18:48,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:18:48,166 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:18:48,338 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:18:49,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:49,338 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:18:54,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:18:56,874 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:18:57,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:18:57,100 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:18:57,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:18:59,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:04,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:09,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:12,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:19:12,101 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:19:12,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:19:15,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:20,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:25,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:26,875 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:19:27,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:19:27,101 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:19:27,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:19:31,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:36,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:41,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:42,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:19:42,101 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:19:42,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:19:46,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:51,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:56,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:19:56,876 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:19:57,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:19:57,134 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:19:57,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:20:01,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:02,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:20:02,132 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:20:02,132 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:20:02,134 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:20:02,383 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:20:03,383 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:20:07,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:12,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:20:12,134 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:20:12,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:20:12,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:17,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:22,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:26,877 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:20:27,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:20:27,134 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:20:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:20:28,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:33,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:38,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:42,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:20:42,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:20:42,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:20:43,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:48,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:53,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:20:56,878 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:20:57,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:20:57,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:20:57,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:20:58,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:03,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:08,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:12,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:21:12,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:21:12,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:21:14,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:15,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:21:15,898 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:21:15,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:21:15,899 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:21:16,428 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:21:17,428 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:21:19,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:24,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:26,878 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:21:27,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:21:27,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:21:27,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:21:30,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:35,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:40,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:42,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:21:42,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:21:42,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:21:45,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:50,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:55,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:21:56,879 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:21:57,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:21:57,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:21:57,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:22:00,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:05,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:10,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:12,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:22:12,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:22:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:22:15,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:20,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:25,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:26,880 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:22:27,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:22:27,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:22:27,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:22:29,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:22:29,653 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:22:29,653 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:22:29,654 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:22:30,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:22:30,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:31,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:22:35,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:40,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:42,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:22:42,135 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:22:42,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:22:46,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:51,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:56,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:22:56,881 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:22:57,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:22:57,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:22:57,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:23:02,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:07,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:12,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:23:12,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:23:12,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:23:12,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:17,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:22,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:26,882 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:23:27,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:23:27,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:23:27,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:23:28,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:33,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:38,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:42,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:23:42,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:23:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:23:43,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:43,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:23:43,639 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:23:43,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:23:43,641 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:23:44,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:23:44,524 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:23:45,525 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:23:47,526 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:23:48,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:49,528 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:23:53,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:23:56,883 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:23:57,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:23:57,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:23:57,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:23:59,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:04,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:09,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:12,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:24:12,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:24:12,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:24:14,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:19,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:24,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:26,884 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:24:27,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:24:27,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:24:27,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:24:29,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:34,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:39,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:42,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:24:42,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:24:42,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:24:45,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:50,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:55,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:24:56,885 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:24:57,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:24:57,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:24:57,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:25:01,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:01,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:25:01,485 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:25:01,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:25:01,486 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:25:01,568 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:25:03,569 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:25:06,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:11,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:12,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:25:12,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:25:12,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:25:17,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:22,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:26,886 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:25:27,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:25:27,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:25:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:25:27,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:32,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:37,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:42,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:25:42,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:25:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:25:42,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:47,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:52,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:25:56,887 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:25:57,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:25:57,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:25:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:25:58,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:03,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:08,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:12,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:26:12,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:26:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:26:14,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:15,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:26:15,383 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:26:15,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:26:15,385 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:26:15,612 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:26:17,613 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:26:19,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:24,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:26,888 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:26:27,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:26:27,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:26:27,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:26:30,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:35,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:40,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:42,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:26:42,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:26:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:26:46,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:51,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:56,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:26:56,889 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:26:57,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:26:57,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:26:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:27:01,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:06,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:11,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:12,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:27:12,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:27:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:27:17,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:22,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:26,890 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:27:27,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:27:27,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:27:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:27:27,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:29,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:27:29,345 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:27:29,345 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:27:29,347 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:27:29,656 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:27:31,658 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:27:33,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:38,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:42,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:27:42,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:27:42,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:27:43,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:48,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:53,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:27:56,890 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:27:57,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:27:57,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:27:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:27:59,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:04,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:09,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:12,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:28:12,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:28:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:28:14,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:19,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:24,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:26,893 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:28:27,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:28:27,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:28:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:28:30,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:35,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:40,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:42,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:28:42,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:28:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:28:43,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:28:43,048 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:28:43,048 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:28:43,049 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:28:43,701 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:28:45,703 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:28:46,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:51,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:56,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:28:56,892 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:28:57,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:28:57,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:28:57,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:29:01,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:06,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:11,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:12,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:29:12,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:29:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:29:17,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:22,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:26,894 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:29:27,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:29:27,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:29:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:29:27,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:32,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:37,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:42,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:29:42,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:29:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:29:43,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:48,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:53,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:29:56,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:29:56,691 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:29:56,692 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:29:56,693 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:29:56,745 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:29:56,894 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:29:57,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:29:57,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:29:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:29:57,746 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:29:58,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:03,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:08,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:12,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:30:12,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:30:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:30:14,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:19,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:24,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:26,895 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:30:27,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:30:27,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:30:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:30:29,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:34,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:39,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:42,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:30:42,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:30:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:30:44,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:49,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:54,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:30:56,896 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:30:57,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:30:57,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:30:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:31:00,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:05,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:10,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:10,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:31:10,578 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:31:10,578 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:31:10,580 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:31:10,789 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:31:11,790 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:31:12,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:31:12,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:31:12,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:31:16,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:21,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:26,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:26,897 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:31:27,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:31:27,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:31:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:31:32,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:37,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:42,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:31:42,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:31:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:31:42,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:47,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:52,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:31:56,898 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:31:57,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:31:57,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:31:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:31:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:02,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:07,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:12,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:32:12,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:32:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:32:13,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:18,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:23,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:24,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:32:24,385 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:32:24,386 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:32:24,387 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:32:24,833 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:32:25,834 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:32:26,898 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:32:27,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:32:27,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:32:27,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:32:28,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:33,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:38,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:42,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:32:42,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:32:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:32:43,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:48,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:53,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:32:56,899 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:32:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:32:57,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:32:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:32:59,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:04,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:09,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:12,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:33:12,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:33:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:33:14,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:19,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:24,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:26,900 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:33:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:33:27,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:33:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:33:30,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:35,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:38,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:33:38,057 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:33:38,057 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:33:38,059 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:33:38,878 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:33:39,878 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:33:41,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:42,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:33:42,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:33:42,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:33:46,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:51,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:56,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:33:56,901 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:33:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:33:57,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:33:57,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:34:02,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:07,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:12,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:34:12,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:34:12,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:34:12,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:17,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:22,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:26,902 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:34:27,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:34:27,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:34:27,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:34:28,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:33,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:38,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:42,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:34:42,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:34:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:34:43,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:48,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:51,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:34:51,866 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:34:51,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:34:51,868 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:34:51,921 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:34:53,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:34:53,922 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:34:56,903 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:34:57,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:34:57,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:34:57,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:34:59,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:04,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:09,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:12,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:35:12,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:35:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:35:15,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:20,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:25,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:26,904 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:35:27,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:35:27,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:35:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:35:30,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:35,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:40,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:42,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:35:42,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:35:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:35:45,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:50,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:55,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:35:56,905 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:35:57,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:35:57,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:35:57,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:36:01,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:05,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:36:05,906 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:36:05,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:36:05,908 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:36:05,965 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:36:06,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:06,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:36:07,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:36:09,968 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:36:10,969 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:36:11,970 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:36:12,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:36:12,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:36:12,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:36:12,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:17,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:22,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:26,906 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:36:27,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:36:27,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:36:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:36:27,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:32,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:37,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:42,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:36:42,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:36:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:36:43,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:48,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:53,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:36:56,907 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:36:57,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:36:57,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:36:57,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:36:58,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:03,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:08,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:12,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:37:12,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:37:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:37:14,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:19,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:23,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:37:23,964 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:37:23,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:37:23,965 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:37:24,015 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:37:25,005 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:26,016 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:37:26,908 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:37:27,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:37:27,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:37:27,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:37:30,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:35,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:40,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:42,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:37:42,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:37:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:37:45,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:50,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:55,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:37:56,909 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:37:57,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:37:57,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:37:57,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:38:01,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:06,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:11,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:12,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:38:12,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:38:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:38:16,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:21,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:26,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:26,910 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:38:27,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:38:27,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:38:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:38:32,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:37,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:37,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:38:37,638 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:38:37,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:38:37,639 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:38:38,061 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:38:38,061 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:38:42,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:38:42,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:38:42,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:38:42,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:47,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:52,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:38:56,911 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:38:57,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:38:57,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:38:57,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:38:58,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:03,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:08,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:12,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:39:12,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:39:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:39:13,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:18,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:23,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:26,911 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:39:27,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:39:27,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:39:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:39:29,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:34,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:39,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:42,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:39:42,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:39:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:39:45,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:50,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:51,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:39:51,498 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:39:51,498 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:39:51,499 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:39:52,107 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:39:52,107 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:39:55,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:39:56,912 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:39:57,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:39:57,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:39:57,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:40:01,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:06,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:11,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:12,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:40:12,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:40:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:40:17,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:22,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:26,913 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:40:27,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:40:27,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:40:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:40:27,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:32,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:37,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:42,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:40:42,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:40:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:40:43,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:48,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:53,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:40:56,914 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:40:57,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:40:57,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:40:57,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:40:59,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:04,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:05,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:41:05,169 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:41:05,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:41:05,170 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:41:06,151 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:41:06,152 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:41:10,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:12,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:41:12,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:41:12,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:41:15,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:20,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:25,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:26,915 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:41:27,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:41:27,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:41:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:41:31,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:36,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:41,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:42,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:41:42,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:41:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:41:46,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:51,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:56,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:41:56,916 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:41:57,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:41:57,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:41:57,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:42:01,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:06,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:11,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:12,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:42:12,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:42:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:42:16,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:18,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:42:18,887 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:42:18,887 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:42:18,888 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:42:19,197 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:42:20,198 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:42:21,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:26,917 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:42:27,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:42:27,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:42:27,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:42:27,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:32,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:37,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:42,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:42:42,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:42:42,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:42:42,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:47,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:52,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:42:56,918 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:42:57,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:42:57,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:42:57,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:42:58,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:03,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:08,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:12,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:43:12,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:43:12,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:43:13,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:18,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:23,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:26,919 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:43:27,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:43:27,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:43:27,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:43:28,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:32,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:43:32,729 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:43:32,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:43:32,730 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:43:33,241 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:43:33,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:34,242 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:43:38,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:42,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:43:42,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:43:42,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:43:44,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:49,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:54,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:43:56,920 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:43:57,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:43:57,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:43:57,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:44:00,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:05,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:10,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:12,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:44:12,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:44:12,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:44:16,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:21,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:26,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:26,921 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:44:27,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:44:27,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:44:27,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:44:31,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:36,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:41,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:42,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:44:42,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:44:42,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:44:46,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:44:46,469 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:44:46,469 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:44:46,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:46,471 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:44:47,287 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:44:48,287 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:44:51,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:56,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:44:56,921 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:44:57,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:44:57,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:44:57,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:45:02,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:07,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:12,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:45:12,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:45:12,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:45:12,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:17,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:22,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:26,922 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:45:27,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:45:27,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:45:27,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:45:27,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:32,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:37,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:42,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:45:42,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:45:42,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:45:43,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:48,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:53,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:45:56,923 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:45:57,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:45:57,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:45:57,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:45:58,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:00,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:46:00,368 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:46:00,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:46:00,370 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:46:01,331 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:46:02,331 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:46:03,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:08,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:12,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:46:12,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:46:12,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:46:14,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:19,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:24,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:26,924 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:46:27,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:46:27,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:46:27,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:46:29,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:34,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:39,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:42,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:46:42,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:46:42,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:46:44,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:49,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:54,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:46:56,925 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:46:57,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:46:57,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:46:57,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:46:59,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:04,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:09,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:12,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:47:12,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:47:12,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:47:14,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:47:14,231 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:47:14,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:47:14,232 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:47:14,376 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:47:15,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:16,378 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:47:20,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:25,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:26,926 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:47:27,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:47:27,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:47:27,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:47:30,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:35,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:40,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:42,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:47:42,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:47:42,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:47:46,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:51,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:56,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:47:56,927 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:47:57,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:47:57,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:47:57,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:48:01,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:06,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:11,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:12,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:48:12,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:48:12,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:48:17,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:22,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:26,928 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:48:27,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:48:27,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:48:27,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:48:27,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:28,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:48:28,061 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:48:28,061 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:48:28,063 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:48:28,423 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:48:28,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:48:29,424 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:48:31,426 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:48:33,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:33,427 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:48:38,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:42,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:48:42,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:48:42,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:48:43,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:48,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:53,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:48:56,929 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:48:57,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:48:57,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:48:57,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:48:58,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:03,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:08,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:12,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:49:12,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:49:12,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:49:13,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:18,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:23,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:26,930 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:49:27,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:49:27,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:49:27,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:49:28,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:33,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:38,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:42,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:49:42,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:49:42,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:49:43,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:46,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:49:46,143 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:49:46,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:49:46,145 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:49:46,472 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:49:47,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:49:49,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:54,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:49:56,930 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:49:57,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:49:57,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:49:57,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:49:59,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:04,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:09,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:12,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:50:12,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:50:12,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:50:14,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:19,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:24,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:26,931 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:50:27,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:50:27,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:50:27,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:50:29,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:34,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:39,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:42,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:50:42,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:50:42,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:50:45,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:50,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:55,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:50:56,932 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:50:57,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:50:57,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:50:57,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:50:59,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:50:59,916 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:50:59,916 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:50:59,917 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:51:00,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:51:00,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:01,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:51:05,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:10,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:12,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:51:12,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:51:12,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:51:16,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:21,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:26,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:26,933 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:51:27,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:51:27,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:51:27,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:51:31,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:36,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:41,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:42,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:51:42,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:51:42,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:51:46,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:51,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:56,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:51:56,934 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:51:57,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:51:57,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:51:57,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:52:02,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:07,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:12,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:52:12,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:52:12,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:52:12,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:13,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:52:13,741 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:52:13,741 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:52:13,742 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:52:14,563 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:52:15,563 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:52:17,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:22,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:26,935 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:52:27,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:52:27,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:52:27,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:52:28,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:33,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:38,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:42,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:52:42,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:52:42,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:52:43,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:48,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:53,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:52:56,936 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:52:57,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:52:57,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:52:57,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:52:59,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:04,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:09,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:12,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:53:12,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:53:12,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:53:15,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:20,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:25,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:26,937 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:53:27,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:53:27,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:53:27,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:53:27,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:53:27,691 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:53:27,692 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:53:27,693 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:53:28,609 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:53:29,610 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:53:30,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:35,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:40,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:42,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:53:42,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:53:42,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:53:46,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:51,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:56,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:53:56,938 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:53:57,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:53:57,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:53:57,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:54:02,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:07,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:12,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:54:12,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:54:12,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:54:12,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:17,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:22,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:26,939 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:54:27,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:54:27,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:54:27,203 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:54:27,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:32,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:37,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:41,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:54:41,550 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:54:41,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:54:41,552 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:54:41,655 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:54:42,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:54:42,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:54:42,203 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:54:42,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:43,656 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:54:47,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:52,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:54:56,940 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:54:57,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:54:57,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:54:57,203 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:54:58,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:03,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:08,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:12,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:55:12,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:55:12,203 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:55:13,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:18,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:23,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:26,941 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:55:27,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:55:27,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:55:27,203 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:55:28,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:33,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:38,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:42,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:55:42,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:55:42,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:55:43,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:48,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:53,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:55:55,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:55:55,387 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:55:55,387 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:55:55,388 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:55:55,698 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:55:56,941 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:55:57,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:55:57,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:55:57,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:55:57,700 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:55:59,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:04,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:09,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:12,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:56:12,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:56:12,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:56:15,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:20,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:25,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:26,942 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:56:27,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:56:27,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:56:27,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:56:30,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:35,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:40,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:42,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:56:42,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:56:42,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:56:46,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:51,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:56,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:56:56,943 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:56:57,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:56:57,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:56:57,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:57:02,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:07,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:09,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:57:09,284 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:57:09,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:57:09,286 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:57:09,744 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:57:11,745 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:57:12,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:57:12,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:57:12,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:57:13,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:18,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:23,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:26,944 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:57:27,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:57:27,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:57:27,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:57:28,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:33,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:38,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:42,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:57:42,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:57:42,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:57:44,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:49,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:54,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:57:56,945 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:57:57,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:57:57,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:57:57,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:57:59,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:04,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:09,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:12,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:58:12,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:58:12,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:58:14,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:19,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:22,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:58:22,920 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:58:22,920 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:58:22,921 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:58:23,788 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:58:24,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:25,790 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:58:26,946 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:58:27,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:58:27,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:58:27,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:58:30,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:35,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:40,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:42,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:58:42,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:58:42,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:58:46,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:51,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:56,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:58:56,947 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:58:57,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:58:57,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:58:57,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:59:01,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:06,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:11,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:12,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:59:12,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:59:12,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:59:16,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:21,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:26,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:26,949 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:59:27,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:59:27,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:59:27,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:59:31,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:36,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:36,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 12:59:36,897 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 12:59:36,897 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 12:59:36,899 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 12:59:37,834 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 12:59:37,834 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 12:59:41,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:42,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:59:42,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:59:42,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:59:47,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:52,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 12:59:56,949 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 12:59:57,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 12:59:57,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 12:59:57,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 12:59:57,507 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:02,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:07,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:12,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:00:12,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:00:12,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:00:12,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:17,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:22,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:26,950 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:00:27,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:00:27,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:00:27,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:00:27,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:32,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:37,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:42,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:00:42,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:00:42,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:00:43,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:48,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:50,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:00:50,907 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:00:50,908 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:00:50,909 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:00:51,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:00:51,879 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:00:51,879 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:00:53,880 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:00:54,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:00:55,881 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:00:56,950 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:00:57,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:00:57,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:00:57,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:00:59,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:04,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:09,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:12,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:01:12,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:01:12,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:01:14,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:19,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:24,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:26,951 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:01:27,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:01:27,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:01:27,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:01:29,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:34,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:39,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:42,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:01:42,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:01:42,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:01:45,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:50,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:55,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:01:56,952 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:01:57,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:01:57,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:01:57,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:02:00,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:05,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:08,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:02:08,951 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:02:08,952 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:02:08,953 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:02:09,927 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:02:09,928 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:02:10,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:12,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:02:12,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:02:12,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:02:16,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:21,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:26,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:26,953 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:02:27,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:02:27,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:02:27,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:02:32,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:37,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:42,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:02:42,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:02:42,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:02:42,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:47,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:52,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:02:56,954 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:02:57,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:02:57,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:02:57,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:02:57,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:02,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:07,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:12,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:03:12,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:03:12,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:03:13,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:18,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:22,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:03:22,938 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:03:22,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:03:22,940 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:03:22,973 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:03:23,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:23,974 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:03:26,955 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:03:27,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:03:27,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:03:27,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:03:29,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:34,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:39,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:42,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:03:42,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:03:42,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:03:45,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:50,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:55,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:03:56,956 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:03:57,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:03:57,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:03:57,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:04:01,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:06,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:11,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:12,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:04:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:04:12,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:04:17,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:22,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:26,957 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:04:27,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:04:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:04:27,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:04:27,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:32,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:36,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:04:36,717 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:04:36,718 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:04:36,719 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:04:37,021 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:04:37,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:38,022 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:04:42,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:04:42,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:04:42,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:04:43,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:48,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:53,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:04:56,958 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:04:57,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:04:57,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:04:57,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:04:58,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:03,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:08,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:12,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:05:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:05:12,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:05:14,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:19,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:24,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:26,959 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:05:27,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:05:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:05:27,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:05:29,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:34,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:39,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:42,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:05:42,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:05:42,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:05:45,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:50,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:50,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:05:50,506 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:05:50,506 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:05:50,507 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:05:51,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:05:52,069 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:05:55,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:05:56,960 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:05:57,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:05:57,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:05:57,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:06:00,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:05,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:10,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:12,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:06:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:06:12,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:06:16,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:21,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:26,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:26,962 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:06:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:06:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:06:27,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:06:31,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:36,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:41,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:42,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:06:42,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:06:42,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:06:46,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:51,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:56,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:06:56,962 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:06:57,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:06:57,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:06:57,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:07:02,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:04,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:07:04,369 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:07:04,369 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:07:04,371 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:07:05,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:07:06,114 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:07:08,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:07:12,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:07:12,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:07:13,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:18,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:23,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:26,963 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:07:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:07:27,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:07:27,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:07:28,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:33,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:38,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:42,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:07:42,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:07:42,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:07:44,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:49,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:54,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:07:56,964 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:07:57,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:07:57,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:07:57,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:07:59,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:04,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:09,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:08:12,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:08:12,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:08:14,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:18,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:08:18,138 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:08:18,139 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:08:18,140 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:08:18,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:08:20,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:20,160 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:08:25,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:26,964 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:08:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:08:27,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:08:27,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:08:30,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:35,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:40,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:42,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:08:42,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:08:42,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:08:46,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:51,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:56,504 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:08:56,965 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:08:57,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:08:57,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:08:57,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:09:01,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:06,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:11,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:09:12,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:09:12,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:09:17,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:22,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:26,966 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:09:27,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:09:27,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:09:27,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:09:27,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:31,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:09:31,952 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:09:31,952 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:09:31,954 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:09:32,205 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:09:32,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:34,206 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:09:37,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:42,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:09:42,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:09:42,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:09:43,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:48,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:53,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:09:56,967 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:09:57,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:09:57,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:09:57,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:09:59,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:04,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:09,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:12,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:10:12,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:10:12,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:10:15,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:20,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:25,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:26,969 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:10:27,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:10:27,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:10:27,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:10:30,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:35,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:40,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:42,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:10:42,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:10:42,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:10:45,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:10:45,818 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:10:45,818 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:10:45,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:45,820 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:10:46,249 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:10:48,250 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:10:50,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:55,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:10:56,969 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:10:57,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:10:57,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:10:57,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:11:01,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:06,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:11,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:12,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:11:12,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:11:12,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:11:17,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:22,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:26,970 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:11:27,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:11:27,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:11:27,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:11:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:32,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:37,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:42,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:11:42,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:11:42,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:11:43,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:48,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:53,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:56,971 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:11:57,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:11:57,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:11:57,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:11:59,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:11:59,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:11:59,783 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:11:59,783 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:11:59,784 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:12:00,293 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:12:00,293 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:12:04,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:09,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:12,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:12:12,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:12:12,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:12:15,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:20,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:25,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:26,972 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:12:27,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:12:27,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:12:27,315 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:12:31,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:36,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:41,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:42,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:12:42,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:12:42,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:12:46,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:51,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:56,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:12:56,973 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:12:57,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:12:57,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:12:57,325 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:13:01,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:06,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:11,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:12,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:13:12,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:13:12,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:13:13,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:13:13,663 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:13:13,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:13:13,664 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:13:14,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:13:14,337 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:13:14,338 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:13:16,339 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:13:16,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:18,340 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:13:22,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:26,974 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:13:27,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:13:27,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:13:27,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:13:28,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:33,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:38,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:42,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:13:42,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:13:42,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:13:44,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:49,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:54,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:13:56,976 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:13:57,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:13:57,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:13:57,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:13:59,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:04,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:09,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:12,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:14:12,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:14:12,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:14:15,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:20,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:25,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:26,988 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:14:27,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:14:27,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:14:27,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:14:30,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:31,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:14:31,729 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:14:31,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:14:31,730 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:14:32,385 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:14:32,385 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:14:35,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:40,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:42,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:14:42,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:14:42,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:14:46,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:51,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:56,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:14:56,977 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:14:57,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:14:57,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:14:57,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:15:02,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:07,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:12,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:15:12,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:15:12,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:15:13,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:18,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:23,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:26,978 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:15:27,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:15:27,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:15:27,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:15:29,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:34,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:39,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:42,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:15:42,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:15:42,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:15:45,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:45,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:15:45,680 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:15:45,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:15:45,682 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:15:46,429 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:15:46,429 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:15:50,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:55,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:15:56,979 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:15:57,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:15:57,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:15:57,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:16:01,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:06,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:11,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:12,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:16:12,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:16:12,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:16:16,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:21,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:26,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:26,979 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:16:27,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:16:27,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:16:27,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:16:32,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:37,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:42,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:16:42,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:16:42,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:16:42,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:47,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:52,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:56,981 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:16:57,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:16:57,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:16:57,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:16:58,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:16:59,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:16:59,448 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:16:59,448 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:16:59,449 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:16:59,471 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:17:00,471 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:17:04,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:09,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:12,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:17:12,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:17:12,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:17:14,507 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:19,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:24,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:26,982 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:17:27,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:17:27,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:17:27,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:17:29,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:34,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:39,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:42,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:17:42,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:17:42,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:17:44,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:49,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:54,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:17:56,983 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:17:57,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:17:57,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:17:57,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:18:00,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:05,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:10,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:12,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:18:12,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:18:12,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:18:13,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:18:13,308 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:18:13,309 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:18:13,310 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:18:13,521 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:18:14,521 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:18:16,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:21,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:26,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:26,983 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:18:27,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:18:27,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:18:27,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:18:31,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:36,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:41,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:42,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:18:42,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:18:42,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:18:46,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:51,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:56,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:18:56,984 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:18:57,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:18:57,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:18:57,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:19:02,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:07,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:12,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:19:12,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:19:12,329 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:19:13,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:18,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:23,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:26,985 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:19:26,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:19:27,001 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:19:27,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:19:27,004 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:19:27,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:19:27,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:19:27,373 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:19:27,564 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:19:28,564 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:19:28,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:33,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:38,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:42,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:19:42,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:19:42,373 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:19:43,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:48,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:53,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:19:56,986 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:19:57,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:19:57,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:19:57,373 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:19:58,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:03,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:08,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:12,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:20:12,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:20:12,373 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:20:14,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:19,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:24,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:26,987 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:20:27,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:20:27,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:20:27,373 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:20:30,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:35,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:40,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:40,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:20:40,738 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:20:40,738 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:20:40,739 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:20:41,608 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:20:42,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:20:42,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:20:42,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:20:42,609 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:20:45,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:50,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:55,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:20:56,988 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:20:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:20:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:20:57,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:21:01,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:06,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:11,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:12,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:21:12,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:21:12,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:21:17,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:22,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:26,989 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:21:27,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:21:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:21:27,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:21:27,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:32,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:37,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:42,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:21:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:21:42,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:21:42,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:47,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:52,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:21:54,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:21:54,642 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:21:54,642 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:21:54,644 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:21:54,650 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:21:56,652 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:21:56,990 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:21:57,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:21:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:21:57,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:21:58,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:03,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:08,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:12,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:22:12,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:22:12,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:22:14,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:19,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:24,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:26,991 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:22:27,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:22:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:22:27,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:22:29,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:34,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:39,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:42,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:22:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:22:42,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:22:45,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:50,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:55,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:22:56,992 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:22:57,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:22:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:22:57,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:23:01,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:06,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:08,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:23:08,471 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:23:08,471 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:23:08,472 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:23:08,694 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:23:08,694 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:23:12,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:23:12,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:23:12,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:23:12,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:17,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:22,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:26,993 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:23:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:23:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:23:27,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:23:28,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:33,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:38,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:23:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:23:42,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:23:43,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:48,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:53,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:23:56,994 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:23:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:23:57,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:23:57,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:23:59,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:04,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:09,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:12,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:24:12,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:24:12,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:24:14,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:19,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:22,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:24:22,483 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:24:22,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:24:22,485 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:24:22,737 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:24:23,738 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:24:25,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:26,995 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:24:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:24:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:24:27,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:24:30,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:35,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:40,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:24:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:24:42,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:24:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:50,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:55,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:24:56,996 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:24:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:24:57,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:24:57,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:25:00,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:05,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:10,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:12,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:25:12,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:25:12,376 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:25:16,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:21,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:26,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:26,997 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:25:27,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:25:27,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:25:27,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:25:31,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:36,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:25:36,520 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:25:36,521 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:25:36,522 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:25:36,781 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:25:37,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:37,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:25:37,782 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:25:39,783 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:25:41,785 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:25:42,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:25:42,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:25:42,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:25:42,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:47,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:52,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:25:56,998 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:25:57,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:25:57,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:25:57,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:25:58,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:03,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:08,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:12,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:26:12,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:26:12,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:26:14,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:19,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:24,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:26,998 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:26:27,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:26:27,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:26:27,375 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:26:29,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:34,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:39,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:42,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:26:42,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:26:42,376 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:26:45,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:50,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:54,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:26:54,494 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:26:54,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:26:54,496 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:26:54,828 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:26:55,828 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:26:56,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:26:56,999 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:26:57,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:26:57,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:26:57,376 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:27:01,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:06,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:11,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:12,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:27:12,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:27:12,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:27:17,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:22,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:27,000 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:27:27,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:27:27,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:27:27,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:27:28,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:33,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:38,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:42,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:27:42,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:27:42,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:27:43,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:48,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:53,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:27:57,001 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:27:57,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:27:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:27:57,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:27:58,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:03,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:08,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:28:08,307 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:28:08,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:28:08,309 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:28:08,874 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:28:09,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:09,875 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:28:12,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:28:12,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:28:12,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:28:14,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:19,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:24,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:27,002 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:28:27,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:28:27,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:28:27,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:28:30,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:35,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:40,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:42,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:28:42,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:28:42,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:28:45,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:50,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:55,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:28:57,003 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:28:57,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:28:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:28:57,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:29:00,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:05,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:10,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:12,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:29:12,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:29:12,400 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:29:16,592 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:21,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:22,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:29:22,014 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:29:22,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:29:22,015 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:29:22,918 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:29:23,919 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:29:27,004 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:29:27,005 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:27,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:29:27,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:29:27,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:29:32,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:37,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:42,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:29:42,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:29:42,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:29:43,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:48,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:53,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:29:57,005 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:29:57,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:29:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:29:57,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:29:58,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:03,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:08,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:12,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:30:12,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:30:12,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:30:14,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:19,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:24,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:27,006 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:30:27,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:30:27,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:30:27,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:30:30,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:35,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:35,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:30:35,756 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:30:35,756 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:30:35,758 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:30:35,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:30:37,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:30:40,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:42,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:30:42,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:30:42,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:30:46,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:51,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:56,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:30:57,007 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:30:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:30:57,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:30:57,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:31:01,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:06,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:11,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:12,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:31:12,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:31:12,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:31:17,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:22,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:27,008 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:31:27,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:31:27,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:31:27,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:31:27,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:32,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:37,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:42,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:31:42,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:31:42,402 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:31:43,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:48,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:49,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:31:49,559 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:31:49,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:31:49,560 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:31:50,004 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:31:52,005 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:31:53,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:31:57,008 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:31:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:31:57,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:31:57,402 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:31:59,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:04,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:09,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:12,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:32:12,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:32:12,402 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:32:14,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:19,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:24,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:27,009 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:32:27,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:32:27,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:32:27,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:32:29,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:34,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:39,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:42,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:32:42,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:32:42,424 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:32:45,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:50,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:55,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:32:57,010 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:32:57,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:32:57,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:32:57,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:33:01,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:03,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:33:03,367 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:33:03,367 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:33:03,368 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:33:04,050 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:33:06,052 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:33:07,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:12,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:12,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:33:12,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:33:12,424 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:33:17,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:22,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:27,011 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:33:27,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:33:27,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:33:27,424 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:33:28,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:33,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:38,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:42,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:33:42,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:33:42,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:33:43,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:48,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:53,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:33:57,012 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:33:57,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:33:57,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:33:57,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:33:58,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:03,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:08,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:12,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:34:12,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:34:12,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:34:13,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:17,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:34:17,166 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:34:17,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:34:17,167 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:34:18,092 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:34:18,092 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:34:19,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:24,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:27,013 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:34:27,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:34:27,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:34:27,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:34:29,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:34,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:39,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:42,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:34:42,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:34:42,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:34:44,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:49,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:54,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:34:57,014 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:34:57,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:34:57,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:34:57,425 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:35:00,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:05,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:10,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:12,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:35:12,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:35:12,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:35:15,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:20,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:25,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:27,015 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:35:27,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:35:27,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:35:27,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:35:31,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:35:31,036 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:35:31,037 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:35:31,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:31,038 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:35:31,136 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:35:32,136 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:35:36,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:41,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:42,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:35:42,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:35:42,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:35:46,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:51,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:56,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:35:57,016 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:35:57,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:35:57,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:35:57,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:36:02,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:07,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:12,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:36:12,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:36:12,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:36:13,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:18,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:23,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:27,017 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:36:27,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:36:27,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:36:27,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:36:28,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:33,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:38,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:42,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:36:42,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:36:42,426 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:36:43,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:44,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:36:44,967 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:36:44,968 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:36:44,969 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:36:45,179 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:36:46,180 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:36:48,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:53,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:36:57,017 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:36:57,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:36:57,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:36:57,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:36:59,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:04,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:09,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:12,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:37:12,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:37:12,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:37:15,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:20,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:25,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:27,020 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:37:27,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:37:27,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:37:27,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:37:30,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:35,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:40,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:42,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:37:42,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:37:42,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:37:45,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:50,696 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:55,696 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:37:57,019 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:37:57,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:37:57,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:37:57,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:37:58,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:37:58,693 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:37:58,693 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:37:58,696 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:37:59,223 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:37:59,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:38:00,224 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:38:01,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:02,225 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:38:03,225 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:38:04,226 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:38:06,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:11,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:12,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:38:12,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:38:12,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:38:17,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:22,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:27,020 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:38:27,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:38:27,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:38:27,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:38:27,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:32,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:37,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:42,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:38:42,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:38:42,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:38:43,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:48,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:53,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:38:57,022 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:38:57,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:38:57,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:38:57,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:38:59,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:04,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:09,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:12,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:39:12,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:39:12,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:39:14,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:16,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:39:16,748 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:39:16,749 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:39:16,750 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:39:17,270 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:39:18,270 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:39:19,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:24,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:27,022 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:39:27,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:39:27,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:39:27,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:39:30,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:35,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:40,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:42,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:39:42,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:39:42,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:39:45,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:50,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:55,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:39:57,023 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:39:57,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:39:57,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:39:57,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:40:01,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:06,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:11,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:12,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:40:12,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:40:12,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:40:17,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:22,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:27,024 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:40:27,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:40:27,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:40:27,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:40:27,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:30,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:40:30,584 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:40:30,584 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:40:30,586 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:40:31,315 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:40:32,315 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:40:33,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:38,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:42,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:40:42,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:40:42,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:40:43,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:48,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:53,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:40:57,025 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:40:57,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:40:57,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:40:57,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:40:59,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:04,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:09,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:12,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:41:12,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:41:12,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:41:14,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:19,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:24,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:27,026 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:41:27,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:41:27,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:41:27,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:41:29,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:34,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:39,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:42,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:41:42,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:41:42,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:41:44,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:41:44,482 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:41:44,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:41:44,484 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:41:45,360 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:41:45,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:46,361 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:41:50,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:55,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:41:57,026 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:41:57,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:41:57,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:41:57,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:42:00,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:05,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:10,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:12,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:42:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:42:12,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:42:16,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:21,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:26,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:27,027 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:42:27,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:42:27,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:42:27,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:42:32,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:37,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:42,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:42:42,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:42:42,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:42:43,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:48,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:53,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:42:57,028 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:42:57,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:42:57,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:42:57,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:42:58,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:42:58,485 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:42:58,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:42:58,486 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:42:59,405 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:42:59,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:00,406 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:43:04,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:09,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:12,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:43:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:43:12,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:43:14,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:19,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:24,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:27,029 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:43:27,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:43:27,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:43:27,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:43:29,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:34,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:39,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:42,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:43:42,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:43:42,430 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:43:45,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:50,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:55,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:43:57,030 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:43:57,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:43:57,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:43:57,430 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:44:00,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:05,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:10,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:12,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:44:12,266 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:44:12,267 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:44:12,268 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:44:12,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:44:12,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:44:12,430 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:44:12,450 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:44:14,451 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:44:16,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:21,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:26,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:27,032 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:44:27,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:44:27,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:44:27,430 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:44:31,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:36,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:41,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:42,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:44:42,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:44:42,430 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:44:47,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:52,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:44:57,032 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:44:57,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:44:57,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:44:57,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:44:58,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:03,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:08,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:45:12,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:45:12,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:45:13,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:18,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:23,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:26,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:45:26,215 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:45:26,216 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:45:26,218 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:45:26,501 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:45:27,033 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:45:27,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:45:27,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:45:27,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:45:28,502 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:45:29,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:34,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:39,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:42,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:45:42,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:45:42,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:45:44,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:49,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:54,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:45:57,034 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:45:57,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:45:57,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:45:57,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:45:59,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:04,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:09,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:46:12,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:46:12,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:46:14,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:19,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:24,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:27,035 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:46:27,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:46:27,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:46:27,431 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:46:29,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:34,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:39,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:40,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:46:40,406 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:46:40,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:46:40,408 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:46:40,551 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:46:42,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:46:42,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:46:42,432 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:46:42,552 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:46:45,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:50,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:55,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:46:57,036 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:46:57,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:46:57,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:46:57,432 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:47:00,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:05,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:10,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:47:12,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:47:12,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:47:15,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:20,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:25,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:27,037 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:47:27,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:47:27,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:47:27,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:47:31,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:36,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:41,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:42,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:47:42,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:47:42,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:47:46,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:51,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:54,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:47:54,468 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:47:54,469 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:47:54,470 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:47:54,601 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:47:56,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:47:57,038 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:47:57,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:47:57,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:47:57,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:47:57,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:48:02,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:07,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:12,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:48:12,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:48:12,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:48:13,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:18,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:23,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:27,039 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:48:27,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:48:27,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:48:27,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:48:29,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:34,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:39,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:42,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:48:42,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:48:42,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:48:45,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:50,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:55,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:48:57,040 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:48:57,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:48:57,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:48:57,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:49:00,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:05,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:08,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:49:08,640 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:49:08,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:49:08,642 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:49:08,651 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:49:10,652 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:49:11,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:12,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:49:12,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:49:12,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:49:16,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:21,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:26,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:27,041 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:49:27,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:49:27,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:49:27,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:49:32,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:37,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:42,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:49:42,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:49:42,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:49:42,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:47,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:52,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:49:57,042 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:49:57,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:49:57,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:49:57,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:49:58,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:03,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:08,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:12,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:50:12,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:50:12,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:50:13,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:18,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:22,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:50:22,438 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:50:22,439 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:50:22,440 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:50:22,699 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:50:23,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:50:24,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:24,700 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:50:25,701 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:50:26,702 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:50:27,042 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:50:27,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:50:27,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:50:27,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:50:28,703 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:50:29,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:34,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:39,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:42,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:50:42,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:50:42,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:50:45,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:50,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:55,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:50:57,043 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:50:57,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:50:57,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:50:57,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:51:00,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:05,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:10,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:12,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:51:12,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:51:12,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:51:16,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:21,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:26,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:27,044 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:51:27,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:51:27,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:51:27,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:51:32,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:37,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:40,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:51:40,891 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:51:40,891 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:51:40,893 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:51:41,748 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:51:42,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:51:42,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:51:42,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:51:42,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:42,749 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:51:47,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:52,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:51:57,045 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:51:57,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:51:57,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:51:57,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:51:58,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:03,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:08,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:12,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:52:12,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:52:12,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:52:13,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:18,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:23,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:27,046 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:52:27,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:52:27,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:52:27,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:52:28,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:33,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:38,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:42,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:52:42,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:52:42,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:52:44,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:49,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:54,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:52:54,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:52:54,745 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:52:54,746 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:52:54,747 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:52:54,792 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:52:56,793 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:52:57,047 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:52:57,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:52:57,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:52:57,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:53:00,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:05,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:10,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:12,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:53:12,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:53:12,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:53:15,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:20,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:25,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:27,048 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:53:27,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:53:27,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:53:27,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:53:31,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:36,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:41,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:42,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:53:42,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:53:42,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:53:47,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:52,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:53:57,049 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:53:57,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:53:57,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:53:57,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:53:57,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:02,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:07,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:08,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:54:08,637 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:54:08,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:54:08,639 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:54:08,838 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:54:10,840 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:54:12,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:54:12,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:54:12,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:54:12,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:17,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:22,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:27,050 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:54:27,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:54:27,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:54:27,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:54:28,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:33,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:38,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:42,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:54:42,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:54:42,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:54:44,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:49,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:54,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:54:57,051 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:54:57,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:54:57,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:54:57,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:54:59,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:04,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:09,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:12,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:55:12,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:55:12,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:55:14,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:19,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:22,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:55:22,671 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:55:22,672 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:55:22,673 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:55:22,883 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:55:24,884 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:55:25,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:27,052 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:55:27,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:55:27,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:55:27,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:55:30,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:35,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:40,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:42,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:55:42,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:55:42,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:55:46,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:51,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:56,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:55:57,053 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:55:57,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:55:57,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:55:57,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:56:02,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:07,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:12,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:56:12,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:56:12,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:56:12,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:17,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:22,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:27,054 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:56:27,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:56:27,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:56:27,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:56:28,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:33,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:36,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:56:36,486 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:56:36,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:56:36,488 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:56:36,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:56:36,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:56:39,489 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:42,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:56:42,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:56:42,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:56:44,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:49,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:54,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:56:57,055 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:56:57,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:56:57,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:56:57,454 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:57:00,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:05,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:10,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:12,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:57:12,453 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:57:12,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:57:15,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:20,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:25,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:27,055 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:57:27,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:57:27,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:57:27,454 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:57:31,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:36,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:41,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:42,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:57:42,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:57:42,454 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:57:47,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:50,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:57:50,597 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:57:50,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:57:50,599 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:57:50,977 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:57:50,978 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:57:52,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:57:57,057 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:57:57,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:57:57,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:57:57,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:57:57,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:02,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:07,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:12,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:58:12,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:58:12,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:58:12,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:17,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:22,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:27,057 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:58:27,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:58:27,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:58:27,454 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:58:28,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:33,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:38,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:42,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:58:42,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:58:42,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:58:44,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:49,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:54,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:58:57,058 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:58:57,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:58:57,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:58:57,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:59:00,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:04,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 13:59:04,485 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 13:59:04,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 13:59:04,487 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 13:59:05,024 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 13:59:05,024 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 13:59:06,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:11,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:12,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:59:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:59:12,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:59:16,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:21,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:26,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:27,059 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:59:27,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:59:27,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:59:27,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:59:32,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:37,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:42,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:59:42,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:59:42,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:59:42,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:47,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:52,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 13:59:57,061 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 13:59:57,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 13:59:57,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 13:59:57,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 13:59:58,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:03,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:08,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:00:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:00:12,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:00:13,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:18,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:00:18,514 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:00:18,514 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:00:18,515 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:00:19,069 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:00:19,069 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:00:19,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:24,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:27,061 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:00:27,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:00:27,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:00:27,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:00:29,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:34,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:39,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:42,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:00:42,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:00:42,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:00:44,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:49,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:54,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:00:57,063 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:00:57,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:00:57,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:00:57,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:00:59,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:04,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:09,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:01:12,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:01:12,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:01:15,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:20,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:25,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:27,063 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:01:27,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:01:27,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:01:27,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:01:31,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:32,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:01:32,440 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:01:32,441 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:01:32,442 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:01:33,120 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:01:33,120 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:01:37,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:42,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:42,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:01:42,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:01:42,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:01:47,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:52,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:01:57,064 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:01:57,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:01:57,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:01:57,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:01:58,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:03,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:08,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:02:12,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:02:12,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:02:13,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:18,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:23,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:27,065 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:02:27,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:02:27,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:02:27,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:02:28,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:33,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:38,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:42,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:02:42,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:02:42,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:02:43,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:46,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:02:46,290 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:02:46,290 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:02:46,291 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:02:46,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:02:47,167 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:02:48,168 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:02:49,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:50,169 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:02:51,170 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:02:52,171 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:02:55,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:02:57,066 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:02:57,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:02:57,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:02:57,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:03:00,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:05,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:10,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:12,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:03:12,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:03:12,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:03:15,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:20,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:25,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:27,067 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:03:27,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:03:27,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:03:27,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:03:30,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:35,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:40,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:42,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:03:42,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:03:42,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:03:46,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:51,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:56,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:03:57,068 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:03:57,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:03:57,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:03:57,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:04:02,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:04,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:04:04,429 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:04:04,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:04:04,431 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:04:05,216 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:04:06,216 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:04:08,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:12,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:04:12,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:04:12,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:04:13,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:18,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:23,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:27,069 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:04:27,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:04:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:04:27,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:04:28,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:33,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:38,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:42,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:04:42,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:04:42,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:04:43,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:48,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:53,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:04:57,070 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:04:57,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:04:57,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:04:57,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:04:58,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:03,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:08,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:12,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:05:12,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:05:12,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:05:14,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:18,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:05:18,350 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:05:18,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:05:18,352 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:05:19,261 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:05:20,261 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:05:20,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:25,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:27,070 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:05:27,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:05:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:05:27,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:05:30,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:35,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:40,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:42,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:05:42,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:05:42,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:05:46,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:51,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:56,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:05:57,071 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:05:57,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:05:57,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:05:57,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:06:02,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:07,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:12,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:06:12,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:06:12,458 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:06:12,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:17,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:22,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:27,072 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:06:27,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:06:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:06:27,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:06:28,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:32,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:06:32,049 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:06:32,049 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:06:32,051 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:06:32,305 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:06:34,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:34,306 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:06:39,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:42,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:06:42,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:06:42,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:06:44,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:49,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:54,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:06:57,073 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:06:57,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:06:57,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:06:57,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:06:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:04,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:09,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:12,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:07:12,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:07:12,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:07:15,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:20,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:25,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:27,075 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:07:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:07:27,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:07:27,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:07:30,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:35,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:40,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:42,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:07:42,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:07:42,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:07:45,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:07:45,817 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:07:45,817 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:07:45,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:45,818 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:07:46,349 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:07:48,350 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:07:50,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:55,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:07:57,075 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:07:57,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:07:57,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:07:57,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:08:01,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:06,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:11,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:12,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:08:12,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:08:12,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:08:17,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:22,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:27,077 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:08:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:08:27,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:08:27,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:08:28,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:33,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:38,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:42,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:08:42,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:08:42,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:08:43,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:48,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:53,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:57,077 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:08:57,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:08:57,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:08:57,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:08:58,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:08:59,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:08:59,668 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:08:59,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:08:59,670 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:09:00,396 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:09:02,397 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:09:04,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:09,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:12,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:09:12,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:09:12,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:09:15,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:20,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:25,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:27,078 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:09:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:09:27,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:09:27,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:09:30,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:35,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:40,699 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:42,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:09:42,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:09:42,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:09:45,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:50,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:55,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:09:57,080 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:09:57,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:09:57,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:09:57,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:10:00,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:05,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:10,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:12,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:10:12,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:10:12,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:10:13,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:10:13,489 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:10:13,489 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:10:13,490 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:10:14,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:10:14,442 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:10:16,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:21,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:26,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:27,080 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:10:27,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:10:27,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:10:27,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:10:31,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:36,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:41,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:42,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:10:42,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:10:42,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:10:46,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:51,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:56,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:10:57,081 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:10:57,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:10:57,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:10:57,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:11:02,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:07,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:12,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:11:12,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:11:12,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:11:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:17,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:22,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:27,082 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:11:27,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:11:27,285 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:11:27,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:11:27,286 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:11:27,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:11:27,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:11:27,461 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:11:27,486 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:11:28,487 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:11:28,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:33,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:38,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:42,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:11:42,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:11:42,461 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:11:43,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:48,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:53,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:11:57,083 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:11:57,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:11:57,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:11:57,461 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:11:59,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:04,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:09,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:12,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:12:12,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:12:12,461 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:12:15,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:20,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:25,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:27,084 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:12:27,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:12:27,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:12:27,461 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:12:30,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:35,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:40,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:41,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:12:41,264 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:12:41,265 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:12:41,266 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:12:41,534 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:12:42,535 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:12:42,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:12:42,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:12:42,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:12:45,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:50,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:55,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:12:57,085 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:12:57,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:12:57,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:12:57,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:13:01,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:06,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:11,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:12,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:13:12,542 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:13:12,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:13:16,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:21,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:26,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:27,086 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:13:27,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:13:27,542 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:13:27,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:13:32,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:37,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:42,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:13:42,542 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:13:42,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:13:42,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:47,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:52,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:13:55,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:13:55,271 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:13:55,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:13:55,272 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:13:55,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:13:56,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:13:57,087 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:13:57,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:13:57,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:13:57,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:13:57,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:02,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:07,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:12,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:14:12,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:14:12,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:14:13,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:18,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:23,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:27,088 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:14:27,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:14:27,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:14:27,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:14:29,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:34,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:39,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:42,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:14:42,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:14:42,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:14:44,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:49,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:54,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:14:57,089 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:14:57,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:14:57,543 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:14:57,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:15:00,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:05,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:09,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:15:09,090 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:15:09,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:15:09,092 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:15:09,627 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:15:09,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:15:10,636 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:15:10,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:12,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:15:12,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:15:12,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:15:12,639 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:15:13,640 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:15:16,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:21,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:26,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:27,089 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:15:27,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:15:27,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:15:27,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:15:31,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:36,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:41,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:42,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:15:42,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:15:42,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:15:46,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:51,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:56,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:15:57,090 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:15:57,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:15:57,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:15:57,550 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:16:02,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:07,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:12,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:16:12,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:16:12,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:16:13,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:18,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:23,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:26,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:16:26,862 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:16:26,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:16:26,864 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:16:27,091 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:16:27,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:16:27,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:16:27,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:16:27,685 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:16:28,686 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:16:29,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:34,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:39,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:42,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:16:42,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:16:42,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:16:44,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:49,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:54,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:16:57,092 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:16:57,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:16:57,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:16:57,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:16:59,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:04,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:09,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:12,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:17:12,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:17:12,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:17:15,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:20,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:25,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:27,093 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:17:27,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:17:27,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:17:27,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:17:30,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:35,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:40,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:17:40,487 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:17:40,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:17:40,489 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:17:40,729 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:17:41,489 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:42,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:17:42,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:17:42,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:17:42,730 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:17:46,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:51,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:56,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:17:57,094 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:17:57,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:17:57,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:17:57,551 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:18:01,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:06,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:11,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:12,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:18:12,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:18:12,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:18:16,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:21,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:26,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:27,095 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:18:27,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:18:27,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:18:27,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:18:32,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:37,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:42,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:18:42,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:18:42,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:18:42,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:47,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:52,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:18:54,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:18:54,325 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:18:54,326 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:18:54,327 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:18:54,793 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:18:56,794 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:18:57,096 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:18:57,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:18:57,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:18:57,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:18:58,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:03,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:08,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:12,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:19:12,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:19:12,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:19:13,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:18,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:23,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:27,096 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:19:27,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:19:27,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:19:27,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:19:28,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:33,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:38,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:42,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:19:42,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:19:42,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:19:44,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:49,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:54,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:19:57,097 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:19:57,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:19:57,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:19:57,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:20:00,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:05,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:07,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:20:07,950 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:20:07,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:20:07,952 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:20:08,839 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:20:08,839 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:20:10,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:12,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:20:12,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:20:12,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:20:16,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:21,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:27,098 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:20:27,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:20:27,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:20:27,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:20:31,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:36,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:41,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:42,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:20:42,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:20:42,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:20:47,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:52,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:20:57,099 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:20:57,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:20:57,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:20:57,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:20:58,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:03,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:08,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:12,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:21:12,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:21:12,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:21:14,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:19,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:21,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:21:21,697 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:21:21,697 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:21:21,698 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:21:21,883 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:21:22,884 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:21:25,699 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:27,100 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:21:27,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:21:27,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:21:27,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:21:30,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:35,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:40,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:42,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:21:42,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:21:42,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:21:46,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:51,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:56,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:21:57,101 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:21:57,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:21:57,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:21:57,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:22:01,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:06,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:11,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:12,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:22:12,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:22:12,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:22:17,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:22,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:27,102 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:22:27,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:22:27,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:22:27,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:22:28,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:33,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:35,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:22:35,360 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:22:35,360 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:22:35,361 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:22:35,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:22:36,930 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:22:39,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:42,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:22:42,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:22:42,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:22:44,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:49,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:54,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:22:57,103 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:22:57,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:22:57,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:22:57,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:22:59,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:04,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:09,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:12,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:23:12,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:23:12,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:23:14,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:19,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:24,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:27,104 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:23:27,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:23:27,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:23:27,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:23:30,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:35,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:40,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:42,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:23:42,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:23:42,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:23:46,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:49,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:23:49,286 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:23:49,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:23:49,287 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:23:49,975 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:23:50,976 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:23:52,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:23:57,105 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:23:57,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:23:57,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:23:57,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:23:57,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:02,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:07,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:12,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:24:12,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:24:12,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:24:12,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:17,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:22,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:27,106 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:24:27,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:24:27,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:24:27,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:24:28,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:33,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:38,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:42,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:24:42,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:24:42,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:24:43,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:48,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:53,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:24:57,106 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:24:57,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:24:57,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:24:57,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:24:59,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:03,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:25:03,074 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:25:03,074 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:25:03,076 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:25:04,021 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:25:05,021 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:25:05,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:10,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:12,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:25:12,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:25:12,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:25:15,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:20,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:25,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:27,107 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:25:27,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:25:27,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:25:27,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:25:31,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:36,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:41,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:42,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:25:42,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:25:42,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:25:46,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:51,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:56,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:25:57,108 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:25:57,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:25:57,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:25:57,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:26:01,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:06,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:11,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:12,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:26:12,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:26:12,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:26:16,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:26:16,913 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:26:16,913 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:26:16,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:16,915 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:26:17,066 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:26:19,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:26:21,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:26,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:27,109 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:26:27,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:26:27,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:26:27,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:26:32,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:37,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:42,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:26:42,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:26:42,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:26:42,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:47,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:52,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:26:57,110 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:26:57,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:26:57,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:26:57,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:26:57,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:02,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:07,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:12,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:27:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:27:12,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:27:13,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:18,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:23,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:27,111 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:27:27,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:27:27,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:27:27,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:27:29,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:30,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:27:30,681 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:27:30,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:27:30,682 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:27:31,112 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:27:31,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:27:33,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:27:34,114 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:27:34,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:37,116 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:27:39,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:42,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:27:42,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:27:42,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:27:45,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:50,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:55,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:27:57,112 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:27:57,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:27:57,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:27:57,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:28:00,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:05,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:10,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:28:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:28:12,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:28:16,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:21,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:26,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:27,113 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:28:27,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:28:27,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:28:27,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:28:31,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:36,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:41,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:42,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:28:42,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:28:42,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:28:47,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:48,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:28:48,688 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:28:48,689 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:28:48,690 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:28:49,174 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:28:49,174 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:28:53,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:28:57,116 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:28:57,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:28:57,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:28:57,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:28:58,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:03,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:08,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:29:12,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:29:12,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:29:14,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:19,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:24,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:27,115 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:29:27,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:29:27,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:29:27,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:29:30,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:35,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:40,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:42,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:29:42,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:29:42,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:29:46,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:51,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:56,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:29:57,118 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:29:57,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:29:57,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:29:57,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:30:01,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:02,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:30:02,516 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:30:02,516 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:30:02,518 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:30:03,221 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:30:03,221 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:30:07,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:12,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:30:12,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:30:12,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:30:17,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:22,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:27,118 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:30:27,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:30:27,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:30:27,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:30:27,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:32,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:37,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:42,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:30:42,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:30:42,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:30:42,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:47,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:52,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:30:57,119 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:30:57,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:30:57,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:30:57,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:30:58,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:03,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:08,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:12,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:31:12,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:31:12,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:31:14,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:16,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:31:16,252 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:31:16,252 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:31:16,254 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:31:16,269 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:31:17,269 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:31:20,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:25,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:27,120 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:31:27,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:31:27,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:31:27,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:31:30,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:35,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:40,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:42,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:31:42,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:31:42,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:31:45,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:50,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:55,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:31:57,121 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:31:57,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:31:57,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:31:57,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:32:00,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:05,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:10,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:12,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:32:12,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:32:12,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:32:15,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:20,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:25,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:27,122 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:32:27,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:32:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:32:27,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:32:30,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:32:30,123 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:32:30,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:32:30,124 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:32:30,314 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:32:31,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:31,315 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:32:36,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:41,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:42,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:32:42,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:32:42,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:32:46,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:51,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:56,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:32:57,123 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:32:57,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:32:57,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:32:57,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:33:02,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:07,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:12,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:33:12,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:33:12,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:33:13,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:18,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:23,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:27,124 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:33:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:33:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:33:27,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:33:28,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:33,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:38,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:42,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:33:42,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:33:42,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:33:43,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:44,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:33:44,039 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:33:44,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:33:44,040 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:33:44,359 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:33:45,360 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:33:49,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:54,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:33:57,124 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:33:57,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:33:57,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:33:57,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:33:59,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:04,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:09,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:12,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:34:12,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:34:12,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:34:14,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:19,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:24,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:27,125 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:34:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:34:27,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:34:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:34:30,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:35,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:40,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:42,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:34:42,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:34:42,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:34:46,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:51,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:56,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:34:57,126 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:34:57,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:34:57,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:34:57,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:34:57,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:34:57,823 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:34:57,823 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:34:57,825 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:34:58,404 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:34:59,405 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:35:01,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:06,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:11,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:12,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:35:12,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:35:12,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:35:17,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:22,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:27,127 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:35:27,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:35:27,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:35:27,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:35:27,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:32,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:37,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:42,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:35:42,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:35:42,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:35:43,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:48,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:53,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:35:57,128 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:35:57,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:35:57,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:35:57,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:35:58,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:03,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:08,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:11,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:36:11,608 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:36:11,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:36:11,609 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:36:12,450 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:36:12,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:36:12,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:36:12,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:36:13,450 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:36:14,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:19,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:24,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:27,129 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:36:27,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:36:27,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:36:27,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:36:29,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:34,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:39,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:42,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:36:42,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:36:42,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:36:45,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:50,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:55,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:36:57,130 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:36:57,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:36:57,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:36:57,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:37:00,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:05,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:10,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:12,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:37:12,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:37:12,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:37:16,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:21,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:25,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:37:25,452 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:37:25,452 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:37:25,454 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:37:25,496 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:37:27,131 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:37:27,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:27,497 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:37:27,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:37:27,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:37:27,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:37:32,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:37,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:42,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:37:42,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:37:42,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:37:42,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:47,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:52,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:37:57,133 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:37:57,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:37:57,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:37:57,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:37:58,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:03,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:08,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:12,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:38:12,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:38:12,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:38:13,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:18,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:23,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:27,133 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:38:27,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:38:27,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:38:27,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:38:28,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:33,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:38,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:39,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:38:39,280 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:38:39,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:38:39,282 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:38:39,542 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:38:39,542 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:38:42,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:38:42,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:38:42,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:38:44,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:49,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:54,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:38:57,134 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:38:57,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:38:57,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:38:57,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:38:59,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:04,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:09,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:12,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:39:12,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:39:12,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:39:15,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:20,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:25,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:27,136 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:39:27,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:39:27,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:39:27,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:39:30,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:35,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:40,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:42,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:39:42,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:39:42,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:39:46,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:51,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:53,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:39:53,310 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:39:53,310 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:39:53,312 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:39:53,606 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:39:53,606 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:39:53,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:39:55,607 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:39:57,136 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:39:57,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:39:57,608 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:39:57,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:39:57,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:39:57,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:40:02,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:07,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:12,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:40:12,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:40:12,629 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:40:13,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:18,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:23,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:27,137 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:40:27,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:40:27,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:40:27,629 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:40:29,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:34,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:39,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:42,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:40:42,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:40:42,629 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:40:44,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:49,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:54,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:40:57,138 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:40:57,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:40:57,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:40:57,629 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:40:59,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:04,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:09,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:11,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:41:11,201 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:41:11,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:41:11,203 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:41:11,655 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:41:11,655 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:41:12,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:41:12,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:41:12,629 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:41:15,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:20,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:25,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:27,139 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:41:27,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:41:27,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:41:27,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:41:30,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:35,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:40,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:42,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:41:42,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:41:42,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:41:45,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:50,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:55,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:41:57,140 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:41:57,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:41:57,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:41:57,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:42:01,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:06,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:11,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:12,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:42:12,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:42:12,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:42:16,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:21,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:24,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:42:24,972 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:42:24,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:42:24,974 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:42:25,702 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:42:25,702 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:42:27,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:27,141 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:42:27,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:42:27,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:42:27,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:42:32,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:37,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:42,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:42:42,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:42:42,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:42:42,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:47,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:52,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:42:57,142 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:42:57,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:42:57,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:42:57,631 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:42:58,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:03,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:08,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:12,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:43:12,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:43:12,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:43:14,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:19,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:24,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:27,143 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:43:27,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:43:27,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:43:27,631 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:43:29,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:34,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:38,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:43:38,815 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:43:38,815 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:43:38,816 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:43:39,748 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:43:39,749 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:43:40,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:42,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:43:42,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:43:42,631 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:43:45,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:50,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:55,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:43:57,144 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:43:57,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:43:57,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:43:57,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:44:00,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:05,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:10,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:12,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:44:12,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:44:12,631 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:44:16,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:21,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:26,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:27,145 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:44:27,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:44:27,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:44:27,631 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:44:31,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:36,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:41,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:42,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:44:42,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:44:42,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:44:46,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:51,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:52,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:44:52,473 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:44:52,473 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:44:52,475 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:44:52,794 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:44:53,794 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:44:57,146 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:44:57,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:44:57,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:44:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:44:57,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:45:02,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:07,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:12,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:45:12,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:45:12,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:45:13,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:18,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:23,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:27,148 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:45:27,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:45:27,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:45:27,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:45:28,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:33,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:38,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:42,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:45:42,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:45:42,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:45:43,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:48,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:53,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:45:57,147 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:45:57,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:45:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:45:57,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:45:59,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:04,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:06,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:46:06,236 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:46:06,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:46:06,239 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:46:06,840 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:46:08,841 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:46:10,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:12,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:46:12,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:46:12,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:46:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:20,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:25,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:27,149 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:46:27,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:46:27,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:46:27,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:46:31,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:36,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:41,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:42,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:46:42,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:46:42,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:46:46,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:51,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:56,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:46:57,150 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:46:57,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:46:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:46:57,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:47:02,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:07,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:12,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:47:12,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:47:12,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:47:12,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:17,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:19,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:47:19,966 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:47:19,966 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:47:19,968 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:47:20,885 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:47:22,886 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:47:22,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:27,150 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:47:27,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:47:27,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:47:27,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:47:28,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:33,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:38,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:42,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:47:42,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:47:42,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:47:43,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:48,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:53,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:47:57,151 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:47:57,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:47:57,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:47:57,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:47:59,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:04,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:09,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:12,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:48:12,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:48:12,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:48:14,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:19,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:24,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:27,152 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:48:27,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:48:27,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:48:27,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:48:29,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:33,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:48:33,561 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:48:33,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:48:33,562 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:48:33,933 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:48:34,933 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:48:35,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:40,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:42,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:48:42,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:48:42,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:48:45,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:50,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:55,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:48:57,153 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:48:57,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:48:57,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:48:57,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:49:01,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:06,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:11,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:12,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:49:12,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:49:12,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:49:16,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:21,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:26,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:27,154 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:49:27,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:49:27,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:49:27,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:49:31,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:36,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:41,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:42,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:49:42,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:49:42,633 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:49:46,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:47,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:49:47,255 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:49:47,256 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:49:47,257 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:49:47,978 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:49:48,979 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:49:52,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:57,155 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:49:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:49:57,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:49:57,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:49:57,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:50:02,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:07,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:12,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:50:12,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:50:12,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:50:12,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:17,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:22,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:27,156 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:50:27,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:50:27,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:50:27,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:50:28,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:33,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:38,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:42,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:50:42,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:50:42,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:50:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:48,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:53,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:50:57,157 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:50:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:50:57,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:50:57,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:50:59,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:01,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:51:01,122 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:51:01,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:51:01,125 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:51:02,023 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:51:03,024 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:51:05,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:10,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:12,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:51:12,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:51:12,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:51:15,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:20,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:25,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:27,158 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:51:27,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:51:27,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:51:27,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:51:31,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:36,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:41,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:42,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:51:42,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:51:42,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:51:46,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:51,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:56,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:51:57,159 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:51:57,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:51:57,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:51:57,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:52:01,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:06,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:11,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:12,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:52:12,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:52:12,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:52:14,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:52:14,834 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:52:14,835 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:52:14,836 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:52:15,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:52:15,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:52:17,069 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:52:17,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:18,070 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:52:19,071 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:52:21,073 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:52:22,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:27,161 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:52:27,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:52:27,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:52:27,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:52:28,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:33,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:38,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:42,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:52:42,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:52:42,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:52:43,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:48,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:53,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:52:57,161 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:52:57,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:52:57,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:52:57,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:52:59,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:04,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:09,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:12,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:53:12,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:53:12,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:53:14,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:19,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:24,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:27,162 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:53:27,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:53:27,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:53:27,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:53:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:32,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:53:32,735 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:53:32,735 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:53:32,736 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:53:33,116 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:53:35,118 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:53:36,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:41,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:42,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:53:42,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:53:42,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:53:46,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:51,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:56,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:53:57,163 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:53:57,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:53:57,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:53:57,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:54:02,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:07,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:12,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:54:12,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:54:12,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:54:12,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:17,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:22,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:27,163 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:54:27,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:54:27,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:54:27,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:54:27,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:32,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:37,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:42,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:54:42,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:54:42,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:54:42,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:46,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:54:46,599 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:54:46,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:54:46,601 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:54:47,165 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:54:48,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:49,166 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:54:53,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:54:57,164 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:54:57,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:54:57,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:54:57,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:54:58,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:03,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:08,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:12,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:55:12,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:55:12,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:55:14,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:19,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:24,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:27,165 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:55:27,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:55:27,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:55:27,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:55:30,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:35,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:40,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:42,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:55:42,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:55:42,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:55:45,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:50,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:55,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:55:57,166 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:55:57,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:55:57,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:55:57,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:56:00,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:56:00,505 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:56:00,506 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:56:00,507 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:56:01,209 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:56:01,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:03,210 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:56:06,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:11,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:12,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:56:12,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:56:12,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:56:16,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:21,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:26,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:27,167 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:56:27,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:56:27,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:56:27,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:56:31,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:36,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:41,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:42,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:56:42,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:56:42,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:56:46,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:51,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:56,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:56:57,171 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:56:57,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:56:57,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:56:57,636 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:57:02,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:07,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:12,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:57:12,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:57:12,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:57:12,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:14,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:57:14,542 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:57:14,542 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:57:14,543 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:57:15,253 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:57:17,254 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:57:18,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:23,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:27,172 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:57:27,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:57:27,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:57:27,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:57:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:33,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:38,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:42,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:57:42,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:57:42,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:57:44,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:49,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:54,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:57:57,173 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:57:57,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:57:57,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:57:57,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:58:00,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:05,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:10,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:12,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:58:12,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:58:12,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:58:15,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:20,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:25,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:27,174 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:58:27,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:58:27,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:58:27,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:58:28,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:58:28,441 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:58:28,442 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:58:28,443 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:58:29,296 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:58:29,296 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:58:31,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:36,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:41,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:42,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:58:42,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:58:42,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:58:46,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:51,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:56,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:58:57,175 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:58:57,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:58:57,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:58:57,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:59:01,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:06,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:11,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:12,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:59:12,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:59:12,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:59:16,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:21,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:26,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:27,175 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:59:27,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:59:27,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:59:27,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:59:32,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:37,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:42,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 14:59:42,498 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 14:59:42,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 14:59:42,501 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 14:59:42,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:59:42,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:59:42,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:59:42,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:43,340 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 14:59:43,340 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 14:59:47,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:52,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 14:59:57,176 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 14:59:57,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 14:59:57,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 14:59:57,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 14:59:57,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:02,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:07,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:12,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:00:12,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:00:12,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:00:13,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:18,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:23,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:27,180 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:00:27,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:00:27,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:00:27,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:00:28,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:33,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:38,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:42,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:00:42,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:00:42,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:00:44,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:49,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:54,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:00:56,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:00:56,525 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:00:56,525 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:00:56,526 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:00:57,181 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:00:57,385 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:00:57,385 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:00:57,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:00:57,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:00:57,638 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:01:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:05,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:10,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:12,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:01:12,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:01:12,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:01:15,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:20,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:25,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:27,182 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:01:27,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:01:27,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:01:27,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:01:31,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:36,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:41,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:42,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:01:42,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:01:42,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:01:46,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:51,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:56,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:01:57,183 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:01:57,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:01:57,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:01:57,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:02:02,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:07,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:10,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:02:10,251 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:02:10,251 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:02:10,252 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:02:10,430 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:02:11,431 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:02:12,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:02:12,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:02:12,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:02:13,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:18,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:23,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:27,184 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:02:27,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:02:27,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:02:27,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:02:28,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:33,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:38,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:42,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:02:42,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:02:42,639 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:02:44,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:49,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:54,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:02:57,185 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:02:57,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:02:57,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:02:57,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:03:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:05,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:10,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:12,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:03:12,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:03:12,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:03:16,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:21,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:24,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:03:24,586 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:03:24,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:03:24,588 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:03:25,479 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:03:25,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:03:27,186 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:03:27,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:27,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:03:27,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:03:27,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:03:32,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:37,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:42,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:03:42,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:03:42,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:03:43,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:48,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:53,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:03:57,187 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:03:57,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:03:57,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:03:57,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:03:58,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:03,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:08,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:12,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:04:12,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:04:12,640 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:04:14,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:19,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:24,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:27,188 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:04:27,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:04:27,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:04:27,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:04:29,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:34,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:38,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:04:38,717 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:04:38,718 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:04:38,719 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:04:39,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:04:39,524 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:04:39,525 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:04:40,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:41,526 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:04:42,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:04:42,692 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:04:42,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:04:43,527 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:04:45,528 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:04:46,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:51,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:56,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:04:57,189 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:04:57,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:04:57,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:04:57,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:05:01,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:06,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:11,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:12,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:05:12,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:05:12,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:05:17,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:22,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:27,190 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:05:27,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:05:27,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:05:27,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:05:28,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:33,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:38,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:42,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:05:42,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:05:42,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:05:43,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:48,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:53,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:05:57,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:05:57,139 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:05:57,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:05:57,141 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:05:57,190 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:05:57,571 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:05:57,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:05:57,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:05:57,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:05:59,572 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:05:59,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:04,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:09,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:12,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:06:12,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:06:12,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:06:14,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:19,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:24,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:27,191 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:06:27,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:06:27,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:06:27,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:06:29,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:34,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:39,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:42,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:06:42,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:06:42,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:06:45,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:50,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:55,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:06:57,192 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:06:57,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:06:57,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:06:57,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:07:00,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:05,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:10,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:07:10,929 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:07:10,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:07:10,931 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:07:11,614 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:07:11,614 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:07:11,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:12,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:07:12,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:07:12,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:07:17,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:22,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:27,193 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:07:27,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:07:27,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:07:27,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:07:28,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:33,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:38,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:42,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:07:42,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:07:42,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:07:43,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:48,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:53,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:07:57,195 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:07:57,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:07:57,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:07:57,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:07:58,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:03,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:08,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:12,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:08:12,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:08:12,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:08:14,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:19,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:24,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:25,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:08:25,174 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:08:25,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:08:25,176 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:08:25,658 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:08:25,658 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:08:27,195 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:08:27,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:08:27,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:08:27,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:08:29,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:34,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:39,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:42,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:08:42,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:08:42,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:08:44,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:49,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:54,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:08:57,197 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:08:57,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:08:57,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:08:57,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:08:59,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:04,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:09,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:12,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:09:12,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:09:12,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:09:15,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:20,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:25,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:27,197 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:09:27,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:09:27,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:09:27,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:09:30,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:35,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:39,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:09:39,067 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:09:39,068 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:09:39,070 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:09:39,701 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:09:39,701 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:09:41,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:42,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:09:42,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:09:42,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:09:46,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:51,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:56,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:09:57,198 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:09:57,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:09:57,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:09:57,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:10:02,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:07,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:12,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:10:12,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:10:12,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:10:12,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:17,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:22,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:27,199 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:10:27,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:10:27,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:10:27,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:10:28,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:33,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:38,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:42,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:10:42,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:10:42,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:10:43,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:48,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:52,975 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:10:52,977 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:10:52,978 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:10:52,979 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:10:53,745 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:10:53,745 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:10:53,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:10:57,200 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:10:57,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:10:57,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:10:57,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:10:59,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:04,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:09,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:12,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:11:12,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:11:12,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:11:15,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:20,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:25,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:27,201 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:11:27,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:11:27,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:11:27,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:11:30,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:35,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:40,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:42,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:11:42,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:11:42,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:11:45,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:50,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:55,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:11:57,202 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:11:57,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:11:57,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:11:57,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:12:00,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:05,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:07,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:12:07,019 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:12:07,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:12:07,021 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:12:07,789 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:12:07,789 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:12:11,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:12,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:12:12,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:12:12,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:12:16,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:21,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:26,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:27,203 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:12:27,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:12:27,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:12:27,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:12:31,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:36,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:41,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:42,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:12:42,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:12:42,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:12:47,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:52,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:12:57,203 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:12:57,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:12:57,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:12:57,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:12:57,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:02,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:07,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:12,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:13:12,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:13:12,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:13:12,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:17,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:21,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:13:21,112 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:13:21,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:13:21,114 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:13:21,831 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:13:21,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:13:23,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:27,204 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:13:27,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:13:27,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:13:27,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:13:28,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:33,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:38,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:42,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:13:42,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:13:42,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:13:43,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:48,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:53,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:13:57,205 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:13:57,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:13:57,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:13:57,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:13:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:03,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:08,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:12,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:14:12,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:14:12,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:14:13,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:18,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:23,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:27,206 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:14:27,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:14:27,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:14:27,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:14:29,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:34,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:35,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:14:35,094 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:14:35,095 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:14:35,096 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:14:35,875 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:14:35,875 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:14:40,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:42,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:14:42,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:14:42,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:14:45,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:50,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:55,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:14:57,207 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:14:57,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:14:57,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:14:57,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:15:00,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:05,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:10,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:12,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:15:12,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:15:12,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:15:15,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:20,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:25,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:27,208 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:15:27,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:15:27,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:15:27,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:15:30,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:35,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:40,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:42,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:15:42,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:15:42,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:15:46,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:49,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:15:49,142 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:15:49,143 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:15:49,144 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:15:49,918 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:15:49,918 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:15:52,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:57,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:15:57,210 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:15:57,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:15:57,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:15:57,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:16:02,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:07,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:12,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:16:12,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:16:12,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:16:12,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:17,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:22,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:27,210 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:16:27,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:16:27,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:16:27,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:16:28,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:33,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:38,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:42,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:16:42,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:16:42,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:16:44,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:49,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:54,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:16:57,211 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:16:57,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:16:57,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:16:57,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:16:59,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:02,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:17:02,867 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:17:02,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:17:02,869 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:17:02,960 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:17:03,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:17:03,960 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:17:05,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:05,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:17:07,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:17:11,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:12,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:17:12,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:17:12,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:17:16,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:21,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:26,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:27,213 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:17:27,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:17:27,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:17:27,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:17:31,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:36,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:41,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:42,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:17:42,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:17:42,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:17:47,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:52,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:17:57,215 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:17:57,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:17:57,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:17:57,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:17:58,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:03,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:08,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:12,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:18:12,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:18:12,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:18:13,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:18,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:20,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:18:20,859 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:18:20,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:18:20,861 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:18:21,004 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:18:22,005 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:18:24,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:27,215 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:18:27,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:18:27,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:18:27,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:18:29,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:34,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:39,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:42,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:18:42,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:18:42,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:18:45,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:50,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:55,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:18:57,216 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:18:57,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:18:57,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:18:57,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:19:00,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:05,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:10,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:12,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:19:12,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:19:12,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:19:16,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:21,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:26,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:27,217 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:19:27,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:19:27,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:19:27,683 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:19:31,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:34,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:19:34,883 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:19:34,884 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:19:34,885 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:19:35,048 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:19:36,049 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:19:36,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:41,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:42,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:19:42,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:19:42,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:19:46,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:51,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:56,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:19:57,218 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:19:57,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:19:57,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:19:57,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:20:02,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:07,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:12,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:20:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:20:12,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:20:12,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:17,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:22,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:27,219 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:20:27,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:20:27,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:20:27,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:20:28,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:33,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:38,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:42,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:20:42,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:20:42,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:20:43,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:48,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:20:48,777 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:20:48,777 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:20:48,779 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:20:49,092 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:20:49,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:51,094 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:20:54,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:20:57,220 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:20:57,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:20:57,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:20:57,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:20:59,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:04,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:09,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:12,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:21:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:21:12,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:21:14,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:19,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:24,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:27,221 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:21:27,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:21:27,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:21:27,684 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:21:29,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:34,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:39,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:42,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:21:42,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:21:42,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:21:44,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:49,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:54,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:21:57,222 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:21:57,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:21:57,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:21:57,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:21:59,975 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:02,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:22:02,787 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:22:02,787 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:22:02,788 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:22:03,135 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:22:05,136 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:22:05,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:10,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:12,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:22:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:22:12,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:22:15,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:20,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:25,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:27,223 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:22:27,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:22:27,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:22:27,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:22:30,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:35,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:40,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:42,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:22:42,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:22:42,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:22:45,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:50,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:55,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:22:57,224 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:22:57,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:22:57,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:22:57,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:23:01,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:06,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:11,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:23:12,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:23:12,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:23:16,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:23:16,906 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:23:16,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:23:16,908 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:23:17,178 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:23:17,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:19,179 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:23:22,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:27,225 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:23:27,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:23:27,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:23:27,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:23:27,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:32,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:37,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:42,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:23:42,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:23:42,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:23:42,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:47,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:52,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:23:57,225 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:23:57,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:23:57,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:23:57,685 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:23:58,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:03,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:08,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:24:12,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:24:12,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:24:14,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:19,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:24,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:27,226 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:24:27,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:24:27,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:24:27,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:24:30,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:24:30,604 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:24:30,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:24:30,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:30,606 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:24:31,222 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:24:33,223 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:24:35,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:40,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:42,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:24:42,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:24:42,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:24:45,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:50,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:55,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:24:57,227 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:24:57,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:24:57,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:24:57,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:25:00,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:05,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:10,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:12,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:25:12,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:25:12,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:25:16,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:21,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:26,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:27,228 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:25:27,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:25:27,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:25:27,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:25:31,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:36,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:41,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:42,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:25:42,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:25:42,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:25:44,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:25:44,584 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:25:44,584 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:25:44,586 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:25:45,265 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:25:47,266 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:25:47,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:52,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:25:57,229 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:25:57,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:25:57,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:25:57,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:25:57,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:02,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:07,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:12,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:26:12,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:26:12,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:26:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:18,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:23,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:27,230 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:26:27,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:26:27,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:26:27,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:26:29,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:34,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:39,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:42,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:26:42,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:26:42,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:26:44,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:49,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:54,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:26:57,231 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:26:57,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:26:57,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:26:57,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:26:58,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:26:58,632 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:26:58,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:26:58,634 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:26:59,309 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:27:00,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:01,310 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:27:05,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:10,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:12,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:27:12,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:27:12,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:27:15,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:20,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:25,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:27,232 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:27:27,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:27:27,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:27:27,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:27:30,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:35,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:40,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:42,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:27:42,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:27:42,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:27:46,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:51,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:56,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:27:57,233 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:27:57,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:27:57,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:27:57,688 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:28:01,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:06,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:11,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:12,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:28:12,540 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:28:12,540 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:28:12,542 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:28:12,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:28:12,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:28:12,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:28:13,353 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:28:15,354 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:28:17,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:22,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:27,233 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:28:27,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:28:27,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:28:27,688 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:28:28,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:33,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:38,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:42,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:28:42,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:28:42,688 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:28:44,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:49,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:54,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:28:57,234 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:28:57,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:28:57,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:28:57,688 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:28:59,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:04,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:09,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:12,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:29:12,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:29:12,688 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:29:14,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:19,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:24,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:26,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:29:26,782 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:29:26,782 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:29:26,783 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:29:27,235 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:29:27,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:29:27,398 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:29:27,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:29:27,733 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:29:27,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:29:29,399 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:29:30,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:30,400 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:29:33,402 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:29:35,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:40,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:42,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:29:42,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:29:42,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:29:45,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:50,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:55,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:29:57,236 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:29:57,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:29:57,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:29:57,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:30:01,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:06,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:11,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:12,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:30:12,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:30:12,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:30:16,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:21,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:26,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:27,237 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:30:27,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:30:27,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:30:27,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:30:32,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:37,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:42,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:30:42,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:30:42,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:30:43,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:30:45,072 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:30:45,072 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:30:45,073 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:30:45,443 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:30:47,444 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:30:49,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:54,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:30:57,238 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:30:57,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:30:57,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:30:57,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:30:59,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:04,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:09,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:12,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:31:12,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:31:12,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:31:14,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:19,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:24,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:27,239 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:31:27,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:31:27,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:31:27,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:31:29,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:34,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:39,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:42,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:31:42,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:31:42,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:31:45,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:50,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:55,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:31:57,240 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:31:57,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:31:57,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:31:57,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:31:58,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:31:58,772 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:31:58,772 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:31:58,774 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:31:59,487 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:32:01,488 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:32:01,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:06,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:11,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:12,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:32:12,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:32:12,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:32:16,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:21,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:26,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:27,241 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:32:27,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:32:27,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:32:27,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:32:31,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:36,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:41,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:42,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:32:42,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:32:42,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:32:46,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:51,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:56,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:32:57,242 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:32:57,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:32:57,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:32:57,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:33:02,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:07,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:12,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:33:12,609 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:33:12,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:33:12,611 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:33:12,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:33:12,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:33:12,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:33:12,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:13,531 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:33:13,531 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:33:17,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:22,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:27,243 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:33:27,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:33:27,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:33:27,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:33:28,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:33,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:38,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:42,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:33:42,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:33:42,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:33:44,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:49,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:54,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:33:57,244 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:33:57,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:33:57,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:33:57,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:33:59,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:04,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:09,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:12,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:34:12,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:34:12,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:34:14,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:19,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:24,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:26,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:34:26,578 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:34:26,579 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:34:26,580 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:34:27,245 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:34:27,576 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:34:27,577 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:34:27,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:34:27,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:34:27,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:34:29,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:34,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:39,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:42,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:34:42,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:34:42,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:34:45,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:50,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:55,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:34:57,246 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:34:57,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:34:57,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:34:57,711 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:35:01,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:06,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:11,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:12,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:35:12,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:35:12,711 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:35:16,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:21,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:26,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:27,247 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:35:27,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:35:27,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:35:27,711 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:35:32,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:37,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:40,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:35:40,526 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:35:40,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:35:40,528 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:35:40,622 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:35:41,623 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:35:42,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:35:42,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:35:42,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:35:42,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:47,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:52,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:35:57,248 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:35:57,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:35:57,728 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:35:57,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:35:58,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:03,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:08,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:12,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:36:12,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:36:12,728 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:36:13,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:18,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:23,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:27,249 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:36:27,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:36:27,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:36:27,728 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:36:29,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:34,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:39,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:42,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:36:42,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:36:42,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:36:44,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:49,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:54,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:36:54,438 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:36:54,439 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:36:54,441 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:36:54,667 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:36:55,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:36:55,668 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:36:57,250 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:36:57,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:36:57,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:36:57,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:37:00,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:05,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:10,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:12,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:37:12,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:37:12,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:37:16,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:21,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:26,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:27,250 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:37:27,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:37:27,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:37:27,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:37:31,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:36,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:41,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:42,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:37:42,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:37:42,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:37:47,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:52,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:37:57,251 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:37:57,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:37:57,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:37:57,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:37:57,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:02,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:07,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:08,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:38:08,388 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:38:08,388 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:38:08,390 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:38:08,713 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:38:09,714 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:38:12,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:38:12,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:38:12,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:38:13,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:18,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:23,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:27,252 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:38:27,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:38:27,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:38:27,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:38:28,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:33,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:38,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:42,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:38:42,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:38:42,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:38:44,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:49,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:54,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:38:57,254 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:38:57,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:38:57,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:38:57,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:38:59,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:04,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:09,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:12,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:39:12,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:39:12,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:39:15,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:20,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:22,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:39:22,165 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:39:22,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:39:22,167 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:39:22,759 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:39:23,759 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:39:25,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:27,254 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:39:27,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:39:27,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:39:27,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:39:30,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:35,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:40,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:42,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:39:42,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:39:42,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:39:45,975 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:50,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:55,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:39:57,256 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:39:57,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:39:57,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:39:57,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:40:00,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:05,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:10,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:12,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:40:12,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:40:12,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:40:16,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:21,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:26,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:27,256 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:40:27,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:40:27,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:40:27,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:40:31,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:36,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:40:36,013 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:40:36,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:40:36,016 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:40:36,804 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:40:37,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:37,805 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:40:42,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:42,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:40:42,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:40:42,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:40:47,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:52,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:40:57,257 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:40:57,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:40:57,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:40:57,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:40:57,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:02,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:07,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:12,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:41:12,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:41:12,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:41:13,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:18,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:23,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:27,257 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:41:27,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:41:27,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:41:27,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:41:28,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:33,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:38,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:42,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:41:42,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:41:42,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:41:44,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:49,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:50,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:41:50,064 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:41:50,064 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:41:50,065 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:41:50,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:41:50,850 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:41:51,851 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:41:53,852 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:41:55,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:41:55,853 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:41:57,258 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:41:57,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:41:57,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:41:57,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:42:00,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:05,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:10,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:12,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:42:12,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:42:12,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:42:16,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:21,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:26,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:27,259 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:42:27,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:42:27,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:42:27,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:42:31,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:36,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:41,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:42,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:42:42,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:42:42,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:42:47,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:52,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:42:57,260 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:42:57,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:42:57,731 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:42:57,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:42:57,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:02,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:07,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:08,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:43:08,144 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:43:08,144 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:43:08,146 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:43:08,902 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:43:09,903 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:43:12,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:43:12,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:43:12,828 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:43:13,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:18,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:23,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:27,261 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:43:27,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:43:27,827 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:43:27,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:43:29,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:34,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:39,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:42,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:43:42,827 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:43:42,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:43:45,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:50,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:55,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:43:57,262 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:43:57,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:43:57,827 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:43:57,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:44:00,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:05,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:10,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:12,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:44:12,827 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:44:12,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:44:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:20,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:22,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:44:22,057 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:44:22,057 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:44:22,058 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:44:22,948 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:44:23,948 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:44:25,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:27,263 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:44:27,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:44:27,827 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:44:27,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:44:31,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:36,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:41,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:42,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:44:42,827 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:44:42,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:44:46,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:51,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:56,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:44:57,264 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:44:57,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:44:57,828 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:44:57,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:45:02,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:07,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:12,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:12,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:45:12,828 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:45:12,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:45:18,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:23,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:27,267 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:45:27,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:45:27,828 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:45:27,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:45:28,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:33,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:36,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:45:36,060 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:45:36,060 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:45:36,064 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:45:37,025 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:45:38,025 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:45:38,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:42,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:45:42,828 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:45:42,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:45:44,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:49,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:54,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:45:57,269 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:45:57,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:45:57,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:45:57,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:46:00,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:05,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:10,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:12,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:46:12,828 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:46:12,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:46:16,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:21,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:26,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:27,269 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:46:27,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:46:27,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:46:27,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:46:32,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:37,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:42,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:42,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:46:42,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:46:42,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:46:47,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:50,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:46:50,180 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:46:50,181 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:46:50,182 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:46:51,070 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:46:52,071 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:46:52,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:57,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:46:57,269 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:46:57,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:46:57,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:46:57,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:47:03,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:08,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:12,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:47:12,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:47:12,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:47:14,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:19,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:24,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:27,271 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:47:27,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:47:27,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:47:27,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:47:29,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:34,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:39,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:42,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:47:42,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:47:42,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:47:45,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:50,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:55,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:47:57,271 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:47:57,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:47:57,829 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:47:57,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:48:00,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:04,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:48:04,059 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:48:04,060 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:48:04,061 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:48:04,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:48:05,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:06,115 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:48:10,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:12,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:48:12,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:48:12,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:48:16,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:21,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:26,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:27,272 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:48:27,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:48:27,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:48:27,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:48:31,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:36,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:41,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:42,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:48:42,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:48:42,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:48:46,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:51,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:56,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:48:57,273 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:48:57,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:48:57,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:48:57,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:49:02,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:07,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:12,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:12,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:49:12,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:49:12,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:49:17,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:49:17,903 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:49:17,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:49:17,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:17,904 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:49:18,160 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:49:20,162 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:49:22,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:27,274 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:49:27,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:49:27,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:49:27,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:49:28,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:33,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:38,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:42,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:49:42,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:49:42,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:49:43,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:48,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:53,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:49:57,275 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:49:57,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:49:57,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:49:57,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:49:59,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:04,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:09,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:12,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:50:12,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:50:12,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:50:15,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:20,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:25,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:27,276 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:50:27,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:50:27,880 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:50:27,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:50:30,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:31,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:50:31,765 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:50:31,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:50:31,767 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:50:32,205 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:50:32,205 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:50:35,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:40,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:42,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:50:42,880 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:50:42,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:50:46,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:51,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:56,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:50:57,277 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:50:57,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:50:57,880 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:50:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:51:01,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:06,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:11,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:12,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:51:12,880 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:51:12,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:51:17,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:22,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:27,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:27,278 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:51:27,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:51:27,880 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:51:27,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:51:32,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:37,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:42,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:42,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:51:42,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:51:42,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:51:45,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:51:45,691 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:51:45,691 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:51:45,693 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:51:46,249 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:51:46,249 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:51:47,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:52,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:57,279 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:51:57,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:51:57,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:51:57,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:51:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:52:03,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:08,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:12,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:52:12,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:52:12,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:52:14,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:19,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:24,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:27,280 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:52:27,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:52:27,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:52:27,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:52:29,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:34,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:39,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:42,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:52:42,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:52:42,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:52:45,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:50,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:55,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:52:57,281 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:52:57,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:52:57,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:52:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:52:59,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:52:59,581 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:52:59,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:52:59,582 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:53:00,293 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:53:00,293 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:53:00,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:05,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:10,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:12,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:53:12,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:53:12,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:53:16,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:21,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:26,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:27,282 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:53:27,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:53:27,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:53:27,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:53:31,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:36,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:41,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:42,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:53:42,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:53:42,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:53:46,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:51,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:56,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:53:57,283 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:53:57,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:53:57,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:53:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:54:02,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:07,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:12,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:12,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:54:12,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:54:12,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:54:13,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:54:13,472 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:54:13,472 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:54:13,473 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:54:14,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:54:14,352 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:54:14,352 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:54:16,354 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:54:17,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:18,355 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:54:22,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:27,283 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:54:27,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:27,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:54:27,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:54:27,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:54:33,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:38,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:42,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:54:42,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:54:42,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:54:44,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:49,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:54,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:54:57,284 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:54:57,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:54:57,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:54:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:54:59,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:04,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:09,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:12,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:55:12,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:55:12,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:55:15,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:20,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:25,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:27,285 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:55:27,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:55:27,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:55:27,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:55:31,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:31,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:55:31,490 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:55:31,490 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:55:31,491 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:55:32,400 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:55:32,400 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:55:36,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:41,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:42,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:55:42,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:55:42,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:55:47,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:52,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:57,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:55:57,286 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:55:57,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:55:57,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:55:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:56:02,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:07,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:12,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:12,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:56:12,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:56:12,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:56:18,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:23,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:27,287 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:56:27,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:56:27,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:56:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:56:28,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:33,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:38,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:42,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:56:42,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:56:42,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:56:44,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:45,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:56:45,339 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:56:45,339 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:56:45,344 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:56:45,444 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:56:46,445 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:56:49,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:54,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:56:57,290 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:56:57,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:56:57,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:56:57,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:57:00,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:05,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:10,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:12,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:57:12,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:57:12,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:57:16,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:21,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:26,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:27,291 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:57:27,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:57:27,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:57:27,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:57:32,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:37,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:42,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:42,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:57:42,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:57:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:57:47,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:52,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:57,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:57:57,292 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:57:57,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:57:57,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:57:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:57:59,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:57:59,256 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:57:59,256 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:57:59,257 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:57:59,489 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:58:00,489 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:58:02,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:07,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:12,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:12,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:58:12,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:58:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:58:18,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:23,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:27,293 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:58:27,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:58:27,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:58:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:58:28,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:33,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:38,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:42,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:58:42,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:58:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:58:44,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:49,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:54,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:58:57,294 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:58:57,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:58:57,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:58:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:59:00,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:05,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:10,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:12,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:59:12,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:59:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:59:13,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 15:59:13,192 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 15:59:13,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 15:59:13,193 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 15:59:13,535 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 15:59:14,536 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 15:59:15,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:20,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:25,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:27,295 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:59:27,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:59:27,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:59:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:59:31,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:36,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:41,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:42,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:59:42,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:59:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 15:59:47,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:52,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:57,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 15:59:57,296 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 15:59:57,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 15:59:57,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 15:59:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:00:02,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:07,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:12,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:12,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:00:12,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:00:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:00:18,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:23,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:27,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:00:27,081 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:00:27,081 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:00:27,082 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:00:27,296 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:00:27,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:00:27,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:00:27,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:00:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:00:28,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:28,582 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:00:33,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:38,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:42,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:00:42,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:00:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:00:44,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:49,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:54,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:00:57,297 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:00:57,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:00:57,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:00:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:00:59,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:04,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:09,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:12,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:01:12,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:01:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:01:15,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:20,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:25,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:27,298 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:01:27,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:01:27,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:01:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:01:30,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:35,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:40,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:41,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:01:41,076 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:01:41,076 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:01:41,077 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:01:41,634 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:01:42,635 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:01:42,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:01:42,938 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:01:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:01:46,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:51,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:56,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:01:57,299 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:01:57,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:01:57,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:01:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:02:01,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:06,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:11,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:12,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:02:12,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:02:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:02:17,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:22,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:27,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:27,300 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:02:27,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:02:27,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:02:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:02:33,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:38,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:42,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:02:42,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:02:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:02:43,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:48,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:53,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:02:55,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:02:55,040 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:02:55,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:02:55,042 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:02:55,681 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:02:57,301 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:02:57,683 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:02:57,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:02:57,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:02:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:02:59,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:04,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:09,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:12,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:03:12,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:03:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:03:15,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:20,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:25,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:27,302 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:03:27,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:03:27,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:03:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:03:30,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:35,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:40,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:42,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:03:42,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:03:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:03:45,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:50,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:55,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:03:57,303 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:03:57,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:03:57,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:03:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:04:01,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:06,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:09,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:04:09,025 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:04:09,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:04:09,026 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:04:09,724 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:04:11,726 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:04:12,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:12,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:04:12,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:04:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:04:17,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:22,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:27,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:27,304 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:04:27,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:04:27,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:04:27,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:04:32,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:37,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:42,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:42,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:04:42,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:04:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:04:48,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:53,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:04:57,305 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:04:57,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:04:57,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:04:57,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:04:59,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:04,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:09,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:12,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:05:12,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:05:12,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:05:15,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:20,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:22,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:05:22,888 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:05:22,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:05:22,890 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:05:23,766 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:05:25,767 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:05:25,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:27,306 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:05:27,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:05:27,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:05:27,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:05:31,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:36,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:41,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:42,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:05:42,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:05:42,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:05:46,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:51,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:56,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:05:57,307 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:05:57,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:05:57,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:05:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:06:02,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:07,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:12,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:12,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:06:12,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:06:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:06:18,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:23,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:27,308 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:06:27,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:06:27,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:06:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:06:28,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:33,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:36,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:06:36,533 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:06:36,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:06:36,535 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:06:36,811 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:06:37,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:06:38,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:39,813 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:06:41,814 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:06:42,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:06:42,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:06:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:06:43,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:43,815 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:06:48,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:53,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:06:57,309 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:06:57,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:06:57,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:06:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:06:59,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:04,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:09,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:12,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:07:12,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:07:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:07:14,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:19,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:24,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:27,312 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:07:27,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:07:27,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:07:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:07:30,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:35,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:40,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:42,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:07:42,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:07:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:07:45,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:50,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:54,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:07:54,759 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:07:54,759 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:07:54,760 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:07:54,854 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:07:55,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:07:57,312 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:07:57,855 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:07:57,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:07:57,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:07:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:08:01,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:06,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:11,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:12,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:08:12,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:08:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:08:16,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:21,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:26,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:27,313 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:08:27,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:08:27,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:08:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:08:32,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:37,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:42,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:42,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:08:42,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:08:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:08:47,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:52,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:57,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:08:57,315 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:08:57,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:08:57,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:08:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:09:03,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:08,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:08,913 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:09:08,915 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:09:08,915 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:09:08,916 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:09:09,896 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:09:11,897 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:09:12,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:09:12,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:09:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:09:13,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:18,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:23,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:27,315 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:09:27,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:09:27,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:09:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:09:28,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:33,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:38,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:42,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:09:42,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:09:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:09:44,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:49,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:54,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:09:57,317 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:09:57,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:09:57,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:09:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:10:00,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:05,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:10,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:12,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:10:12,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:10:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:10:15,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:20,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:22,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:10:22,754 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:10:22,754 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:10:22,755 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:10:22,939 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:10:25,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:25,941 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:10:27,317 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:10:27,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:10:27,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:10:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:10:31,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:36,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:41,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:42,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:10:42,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:10:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:10:46,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:51,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:56,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:10:57,318 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:10:57,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:10:57,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:10:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:11:01,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:06,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:11,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:12,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:11:12,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:11:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:11:17,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:22,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:27,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:27,319 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:11:27,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:11:27,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:11:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:11:33,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:36,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:11:36,681 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:11:36,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:11:36,682 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:11:36,985 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:11:37,986 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:11:38,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:42,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:11:42,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:11:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:11:44,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:49,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:54,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:11:57,320 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:11:57,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:11:57,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:11:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:11:59,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:04,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:09,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:12,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:12:12,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:12:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:12:15,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:20,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:25,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:27,321 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:12:27,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:12:27,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:12:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:12:30,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:35,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:40,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:42,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:12:42,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:12:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:12:46,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:50,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:12:50,621 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:12:50,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:12:50,622 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:12:51,029 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:12:51,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:52,030 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:12:56,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:12:57,322 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:12:57,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:12:57,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:12:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:13:02,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:07,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:12,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:12,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:13:12,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:13:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:13:17,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:22,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:27,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:27,323 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:13:27,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:13:27,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:13:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:13:32,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:37,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:42,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:42,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:13:42,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:13:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:13:48,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:53,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:13:57,324 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:13:57,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:13:57,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:13:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:13:59,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:04,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:04,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:14:04,393 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:14:04,393 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:14:04,395 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:14:05,073 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:14:06,073 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:14:09,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:12,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:14:12,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:14:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:14:15,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:20,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:25,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:27,325 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:14:27,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:14:27,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:14:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:14:31,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:36,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:41,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:42,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:14:42,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:14:42,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:14:46,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:51,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:56,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:14:57,326 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:14:57,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:14:57,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:14:57,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:15:02,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:07,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:12,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:12,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:15:12,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:15:12,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:15:18,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:18,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:15:18,442 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:15:18,442 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:15:18,443 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:15:19,117 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:15:20,117 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:15:23,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:27,327 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:15:27,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:15:27,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:15:27,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:15:29,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:34,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:39,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:42,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:15:42,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:15:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:15:45,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:50,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:55,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:15:57,328 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:15:57,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:15:57,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:15:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:16:00,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:05,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:10,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:12,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:16:12,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:16:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:16:15,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:20,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:25,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:27,328 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:16:27,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:16:27,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:16:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:16:31,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:32,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:16:32,413 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:16:32,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:16:32,415 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:16:33,162 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:16:34,162 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:16:36,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:41,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:42,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:16:42,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:16:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:16:47,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:52,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:57,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:16:57,330 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:16:57,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:16:57,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:16:57,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:17:03,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:08,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:12,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:17:12,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:17:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:17:13,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:18,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:23,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:27,331 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:17:27,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:17:27,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:17:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:17:28,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:33,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:38,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:42,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:17:42,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:17:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:17:44,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:46,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:17:46,779 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:17:46,779 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:17:46,780 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:17:47,210 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:17:48,210 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:17:49,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:54,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:17:57,331 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:17:57,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:17:57,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:17:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:18:00,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:05,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:10,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:12,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:18:12,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:18:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:18:15,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:20,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:25,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:27,332 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:18:27,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:18:27,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:18:27,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:18:30,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:35,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:40,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:42,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:18:42,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:18:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:18:45,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:50,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:55,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:18:57,333 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:18:57,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:18:57,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:18:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:19:00,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:00,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:19:00,935 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:19:00,935 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:19:00,937 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:19:01,254 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:19:01,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:19:02,255 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:19:04,256 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:19:06,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:06,258 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:19:11,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:12,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:19:12,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:19:12,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:19:17,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:22,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:27,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:27,334 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:19:27,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:19:27,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:19:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:19:33,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:38,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:42,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:19:42,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:19:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:19:43,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:48,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:53,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:19:57,335 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:19:57,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:19:57,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:19:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:19:59,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:04,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:09,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:12,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:20:12,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:20:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:20:15,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:19,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:20:19,232 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:20:19,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:20:19,234 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:20:19,302 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:20:20,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:20,302 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:20:25,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:27,336 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:20:27,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:20:27,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:20:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:20:31,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:36,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:41,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:42,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:20:42,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:20:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:20:46,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:51,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:56,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:20:57,337 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:20:57,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:20:57,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:20:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:21:02,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:07,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:12,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:12,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:21:12,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:21:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:21:17,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:22,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:27,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:27,338 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:21:27,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:21:27,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:21:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:21:33,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:21:33,186 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:21:33,187 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:21:33,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:33,188 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:21:33,347 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:21:34,347 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:21:38,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:42,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:21:42,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:21:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:21:44,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:49,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:54,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:21:57,339 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:21:57,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:21:57,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:21:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:22:00,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:05,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:10,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:12,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:22:12,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:22:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:22:15,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:20,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:25,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:27,340 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:22:27,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:22:27,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:22:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:22:30,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:35,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:40,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:42,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:22:42,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:22:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:22:46,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:47,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:22:47,049 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:22:47,049 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:22:47,050 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:22:47,392 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:22:48,393 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:22:52,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:57,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:22:57,341 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:22:57,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:22:57,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:22:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:23:02,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:07,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:12,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:12,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:23:12,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:23:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:23:17,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:22,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:27,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:27,342 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:23:27,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:23:27,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:23:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:23:33,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:38,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:42,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:23:42,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:23:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:23:43,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:48,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:53,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:23:57,345 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:23:57,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:23:57,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:23:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:23:59,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:24:00,846 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:24:00,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:24:00,848 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:24:01,439 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:24:02,439 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:24:04,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:09,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:12,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:24:12,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:24:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:24:15,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:20,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:25,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:27,346 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:24:27,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:24:27,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:24:27,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:24:30,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:35,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:40,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:42,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:24:42,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:24:42,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:24:45,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:50,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:55,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:24:57,347 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:24:57,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:24:57,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:24:57,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:25:01,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:06,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:11,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:12,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:25:12,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:25:12,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:25:14,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:25:14,678 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:25:14,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:25:14,679 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:25:15,484 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:25:16,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:25:16,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:21,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:26,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:27,347 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:25:27,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:25:27,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:25:27,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:25:32,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:37,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:42,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:42,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:25:42,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:25:43,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:25:48,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:53,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:25:57,348 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:25:57,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:25:57,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:25:58,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:25:58,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:03,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:08,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:12,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:26:12,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:26:13,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:26:13,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:18,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:23,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:27,349 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:26:27,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:26:27,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:26:28,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:26:28,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:28,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:26:28,547 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:26:28,547 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:26:28,548 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:26:29,531 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:26:30,532 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:26:33,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:38,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:42,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:26:42,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:26:43,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:26:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:49,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:54,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:26:57,350 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:26:57,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:26:57,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:26:58,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:26:59,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:04,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:09,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:12,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:27:12,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:27:13,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:27:15,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:20,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:25,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:27,351 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:27:27,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:27:27,973 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:27:28,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:27:30,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:35,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:40,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:42,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:27:42,414 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:27:42,414 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:27:42,415 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:27:42,577 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:27:43,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:27:43,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:27:43,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:27:44,578 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:27:45,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:50,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:55,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:27:57,352 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:27:58,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:27:58,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:27:58,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:28:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:05,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:10,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:13,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:28:13,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:28:13,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:28:16,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:21,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:26,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:27,353 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:28:28,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:28:28,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:28:28,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:28:32,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:37,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:42,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:43,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:28:43,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:28:43,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:28:48,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:53,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:56,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:28:56,499 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:28:56,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:28:56,500 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:28:56,620 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:28:57,354 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:28:58,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:28:58,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:28:58,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:28:58,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:28:58,621 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:29:03,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:08,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:13,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:29:13,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:29:13,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:29:13,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:18,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:23,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:27,355 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:29:28,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:29:28,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:29:28,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:29:29,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:34,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:39,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:43,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:29:43,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:29:43,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:29:45,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:50,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:55,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:29:57,356 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:29:58,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:29:58,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:29:58,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:30:00,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:05,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:10,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:10,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:30:10,470 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:30:10,470 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:30:10,472 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:30:10,664 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:30:12,665 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:30:13,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:30:13,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:30:13,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:30:16,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:21,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:26,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:27,357 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:30:28,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:30:28,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:30:28,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:30:31,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:36,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:41,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:43,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:30:43,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:30:43,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:30:46,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:51,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:56,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:30:57,358 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:30:58,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:30:58,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:30:58,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:31:02,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:07,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:12,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:13,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:31:13,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:31:13,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:31:18,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:23,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:24,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:31:24,283 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:31:24,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:31:24,285 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:31:24,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:31:24,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:31:24,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:31:26,709 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:31:27,359 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:31:28,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:31:28,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:31:28,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:31:28,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:28,711 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:31:33,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:38,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:43,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:31:43,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:31:43,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:31:44,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:49,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:54,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:31:57,360 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:31:58,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:31:58,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:31:58,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:32:00,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:05,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:10,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:13,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:32:13,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:32:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:32:15,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:20,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:25,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:27,361 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:32:28,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:32:28,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:32:28,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:32:31,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:36,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:41,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:42,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:32:42,192 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:32:42,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:32:42,193 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:32:42,754 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:32:42,754 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:32:43,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:32:43,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:32:43,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:32:47,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:52,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:57,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:32:57,361 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:32:58,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:32:58,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:32:58,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:33:02,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:07,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:12,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:13,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:33:13,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:33:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:33:17,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:22,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:27,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:27,363 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:33:28,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:33:28,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:33:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:33:33,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:38,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:43,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:33:43,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:33:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:33:43,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:48,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:53,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:33:56,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:33:56,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:33:56,269 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:33:56,270 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:33:56,798 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:33:56,798 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:33:57,363 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:33:58,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:33:58,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:33:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:33:59,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:04,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:09,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:13,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:34:13,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:34:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:34:14,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:19,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:24,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:27,364 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:34:28,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:34:28,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:34:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:34:29,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:34,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:39,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:43,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:34:43,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:34:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:34:45,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:50,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:55,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:34:57,365 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:34:58,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:34:58,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:34:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:35:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:05,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:10,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:35:10,200 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:35:10,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:35:10,201 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:35:10,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:35:10,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:35:11,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:13,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:35:13,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:35:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:35:17,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:22,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:27,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:27,366 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:35:28,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:35:28,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:35:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:35:33,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:38,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:43,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:35:43,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:35:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:35:43,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:48,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:53,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:35:57,367 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:35:58,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:35:58,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:35:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:35:58,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:03,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:08,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:13,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:36:13,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:36:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:36:14,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:19,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:24,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:36:24,122 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:36:24,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:36:24,123 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:36:24,890 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:36:24,891 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:36:25,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:27,368 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:36:28,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:36:28,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:36:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:36:30,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:35,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:40,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:43,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:36:43,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:36:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:36:45,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:50,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:55,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:36:57,369 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:36:58,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:36:58,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:36:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:37:01,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:06,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:11,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:13,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:37:13,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:37:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:37:16,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:21,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:26,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:27,370 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:37:28,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:37:28,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:37:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:37:31,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:36,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:38,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:37:38,059 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:37:38,060 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:37:38,061 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:37:38,933 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:37:38,933 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:37:42,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:43,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:37:43,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:37:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:37:47,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:52,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:57,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:37:57,371 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:37:58,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:37:58,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:37:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:38:02,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:07,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:12,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:13,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:38:13,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:38:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:38:18,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:23,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:27,372 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:38:28,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:38:28,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:38:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:38:29,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:34,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:39,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:43,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:38:43,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:38:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:38:44,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:49,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:51,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:38:51,888 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:38:51,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:38:51,889 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:38:51,976 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:38:53,977 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:38:54,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:38:57,373 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:38:58,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:38:58,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:38:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:39:00,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:05,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:10,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:13,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:39:13,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:39:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:39:15,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:20,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:25,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:27,375 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:39:28,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:39:28,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:39:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:39:31,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:36,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:41,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:43,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:39:43,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:39:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:39:46,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:51,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:56,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:39:57,375 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:39:58,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:39:58,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:39:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:40:02,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:05,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:40:05,659 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:40:05,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:40:05,661 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:40:06,033 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:40:07,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:08,034 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:40:12,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:13,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:40:13,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:40:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:40:18,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:23,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:27,376 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:40:28,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:40:28,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:40:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:40:28,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:33,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:38,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:43,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:40:43,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:40:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:40:43,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:48,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:53,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:40:57,377 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:40:58,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:40:58,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:40:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:40:59,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:04,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:09,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:13,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:41:13,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:41:13,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:41:15,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:19,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:41:19,442 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:41:19,442 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:41:19,443 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:41:20,077 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:41:20,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:22,078 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:41:25,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:27,377 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:41:28,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:41:28,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:41:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:41:31,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:36,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:41,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:43,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:41:43,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:41:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:41:46,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:51,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:56,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:41:57,378 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:41:58,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:41:58,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:41:58,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:42:02,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:07,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:12,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:13,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:42:13,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:42:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:42:17,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:22,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:27,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:27,379 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:42:28,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:42:28,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:42:28,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:42:33,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: keepalive +2024-08-12 16:42:33,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:33,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:42:33,421 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:42:33,421 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:42:33,423 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:42:34,121 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:42:36,122 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:42:38,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:43,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:42:43,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:42:43,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:42:44,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:49,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:54,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:42:57,380 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:42:58,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:42:58,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:42:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:42:59,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:04,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:09,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:13,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:43:13,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:43:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:43:15,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:20,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:25,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:27,381 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:43:28,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:43:28,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:43:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:43:30,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:35,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:40,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:43,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:43:43,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:43:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:43:46,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:47,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:43:47,377 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:43:47,378 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:43:47,379 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:43:47,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:43:48,164 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:43:50,165 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:43:51,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:52,167 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:43:54,168 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:43:56,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:43:57,382 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:43:58,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:43:58,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:43:58,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:44:02,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:07,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:12,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:13,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:44:13,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:44:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:44:18,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:23,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:27,383 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:44:28,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:44:28,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:44:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:44:28,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:33,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:38,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:43,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:44:43,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:44:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:44:43,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:48,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:53,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:44:57,384 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:44:58,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:44:58,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:44:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:44:59,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:04,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:05,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:45:05,462 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:45:05,462 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:45:05,464 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:45:06,211 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:45:08,212 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:45:09,506 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:13,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:45:13,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:45:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:45:15,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:20,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:25,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:27,385 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:45:28,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:45:28,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:45:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:45:30,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:35,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:40,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:43,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:45:43,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:45:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:45:46,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:51,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:56,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:45:57,386 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:45:58,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:45:58,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:45:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:46:01,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:06,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:11,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:13,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:46:13,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:46:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:46:16,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:19,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:46:19,356 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:46:19,357 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:46:19,358 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:46:20,263 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:46:20,264 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:46:21,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:26,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:27,387 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:46:28,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:46:28,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:46:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:46:32,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:37,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:42,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:43,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:46:43,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:46:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:46:47,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:52,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:57,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:46:57,389 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:46:58,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:46:58,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:46:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:47:03,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:08,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:13,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:47:13,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:47:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:47:14,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:19,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:24,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:27,389 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:47:28,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:47:28,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:47:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:47:30,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:33,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:47:33,259 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:47:33,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:47:33,261 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:47:33,307 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:47:34,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:47:35,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:40,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:43,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:47:43,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:47:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:47:46,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:51,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:56,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:47:57,389 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:47:58,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:47:58,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:47:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:48:02,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:07,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:12,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:13,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:48:13,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:48:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:48:17,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:22,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:27,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:27,390 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:48:28,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:48:28,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:48:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:48:33,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:38,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:43,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:48:43,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:48:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:48:43,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:47,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:48:47,099 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:48:47,099 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:48:47,100 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:48:47,353 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:48:48,353 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:48:49,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:54,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:48:57,391 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:48:58,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:48:58,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:48:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:48:59,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:04,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:09,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:13,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:49:13,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:49:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:49:15,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:20,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:25,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:27,392 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:49:28,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:49:28,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:49:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:49:31,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:36,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:41,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:43,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:49:43,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:49:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:49:46,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:51,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:56,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:49:57,393 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:49:58,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:49:58,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:49:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:50:00,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:50:00,878 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:50:00,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:50:00,879 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:50:01,398 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:50:01,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:02,399 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:50:06,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:11,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:13,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:50:13,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:50:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:50:17,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:22,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:27,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:27,394 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:50:28,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:50:28,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:50:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:50:32,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:37,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:42,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:43,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:50:43,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:50:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:50:48,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:53,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:50:57,395 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:50:58,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:50:58,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:50:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:50:58,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:03,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:08,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:13,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:51:13,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:51:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:51:14,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:14,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:51:14,865 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:51:14,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:51:14,867 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:51:15,447 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:51:16,448 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:51:19,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:24,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:27,396 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:51:28,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:51:28,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:51:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:51:30,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:35,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:40,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:43,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:51:43,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:51:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:51:46,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:51,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:56,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:51:57,397 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:51:58,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:51:58,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:51:58,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:52:01,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:06,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:11,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:13,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:52:13,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:52:13,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:52:16,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:21,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:26,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:27,398 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:52:28,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:52:28,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:52:28,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:52:28,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:52:28,790 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:52:28,790 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:52:28,791 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:52:29,492 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:52:30,493 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:52:31,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:36,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:41,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:43,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:52:43,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:52:43,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:52:47,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:52,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:57,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:52:57,399 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:52:58,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:52:58,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:52:58,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:53:02,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:07,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:12,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:13,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:53:13,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:53:13,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:53:18,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:23,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:27,400 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:53:28,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:53:28,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:53:28,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:53:28,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:33,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:38,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:42,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:53:42,463 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:53:42,463 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:53:42,464 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:53:42,534 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:53:43,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:53:43,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:53:43,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:53:44,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:44,535 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:53:49,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:54,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:53:57,401 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:53:58,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:53:58,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:53:58,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:53:59,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:04,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:09,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:13,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:54:13,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:54:13,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:54:14,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:19,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:24,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:27,402 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:54:28,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:54:28,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:54:28,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:54:30,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:35,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:40,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:43,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:54:43,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:54:43,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:54:45,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:50,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:55,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:54:56,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:54:56,410 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:54:56,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:54:56,411 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:54:56,578 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:54:57,403 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:54:58,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:54:58,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:54:58,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:54:58,579 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:55:01,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:06,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:11,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:13,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:55:13,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:55:13,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:55:16,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:21,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:26,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:27,403 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:55:28,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:55:28,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:55:28,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:55:32,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:37,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:42,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:43,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:55:43,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:55:43,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:55:47,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:52,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:57,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:55:57,404 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:55:58,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:55:58,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:55:58,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:56:03,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:08,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:10,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:56:10,444 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:56:10,444 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:56:10,446 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:56:10,622 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:56:11,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:56:12,623 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:56:13,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:13,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:56:13,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:56:13,337 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:56:14,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:56:16,626 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:56:18,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:23,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:27,405 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:56:28,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:56:28,256 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:56:28,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:56:28,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:33,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:38,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:43,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:56:43,256 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:56:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:56:44,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:49,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:54,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:56:57,406 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:56:58,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:56:58,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:56:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:56:59,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:04,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:09,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:13,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:57:13,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:57:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:57:15,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:20,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:25,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:27,407 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:57:28,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:57:28,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:57:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:57:28,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:57:28,495 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:57:28,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:57:28,497 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:57:28,670 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:57:30,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:30,671 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:57:35,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:40,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:43,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:57:43,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:57:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:57:45,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:50,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:55,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:57:57,408 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:57:58,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:57:58,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:57:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:58:01,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:06,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:11,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:13,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:58:13,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:58:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:58:16,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:21,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:26,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:27,410 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:58:28,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:58:28,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:58:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:58:32,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:37,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:42,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:58:42,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:58:42,268 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:58:42,270 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:58:42,714 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:58:43,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:43,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:58:43,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:58:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:58:44,715 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:58:48,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:53,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:58:57,410 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:58:58,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:58:58,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:58:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:58:58,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:03,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:08,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:13,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:59:13,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:59:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:59:14,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:19,489 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:24,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:27,411 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:59:28,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:59:28,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:59:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:59:30,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:35,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:40,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:43,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:59:43,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:59:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 16:59:46,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:51,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:56,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 16:59:56,265 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 16:59:56,265 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 16:59:56,267 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 16:59:56,771 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 16:59:56,772 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 16:59:57,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 16:59:57,412 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 16:59:58,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 16:59:58,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 16:59:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:00:02,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:07,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:12,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:13,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:00:13,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:00:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:00:17,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:22,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:27,413 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:00:28,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:28,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:00:28,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:00:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:00:33,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:38,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:43,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:00:43,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:00:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:00:44,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:49,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:54,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:00:57,414 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:00:58,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:00:58,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:00:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:00:59,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:04,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:09,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:10,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:01:10,280 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:01:10,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:01:10,281 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:01:10,817 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:01:10,818 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:01:13,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:01:13,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:01:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:01:15,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:20,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:25,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:27,415 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:01:28,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:01:28,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:01:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:01:31,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:36,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:41,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:43,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:01:43,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:01:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:01:46,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:51,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:56,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:01:57,416 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:01:58,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:01:58,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:01:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:02:02,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:07,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:12,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:13,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:02:13,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:02:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:02:17,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:22,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:24,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:02:24,277 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:02:24,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:02:24,279 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:02:24,863 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:02:24,863 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:02:27,417 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:02:28,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:28,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:02:28,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:02:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:02:33,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:38,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:43,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:02:43,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:02:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:02:43,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:48,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:53,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:02:57,418 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:02:58,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:02:58,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:02:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:02:58,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:03,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:08,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:13,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:03:13,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:03:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:03:14,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:19,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:24,504 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:27,419 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:03:28,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:03:28,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:03:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:03:29,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:34,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:37,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:03:37,973 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:03:37,974 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:03:37,975 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:03:38,909 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:03:38,909 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:03:40,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:43,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:03:43,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:03:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:03:45,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:50,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:55,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:03:57,420 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:03:58,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:03:58,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:03:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:04:00,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:05,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:10,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:13,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:04:13,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:04:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:04:15,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:20,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:25,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:27,421 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:04:28,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:04:28,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:04:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:04:31,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:36,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:41,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:43,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:04:43,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:04:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:04:47,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:51,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:04:51,796 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:04:51,796 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:04:51,798 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:04:51,954 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:04:52,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:52,955 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:04:57,422 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:04:58,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:04:58,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:04:58,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:04:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:05:03,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:08,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:13,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:05:13,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:05:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:05:13,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:18,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:23,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:27,423 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:05:28,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:05:28,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:05:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:05:29,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:34,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:39,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:43,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:05:43,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:05:43,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:05:44,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:49,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:54,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:05:57,424 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:05:58,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:05:58,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:05:58,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:05:59,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:04,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:05,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:06:05,793 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:06:05,793 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:06:05,794 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:06:06,001 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:06:07,001 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:06:09,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:13,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:06:13,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:06:13,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:06:15,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:20,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:25,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:27,425 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:06:28,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:06:28,260 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:06:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:06:31,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:36,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:41,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:43,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:06:43,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:06:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:06:47,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:52,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:57,426 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:06:58,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:06:58,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:06:58,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:06:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:07:03,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:08,489 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:13,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:07:13,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:07:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:07:13,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:18,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:19,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:07:19,708 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:07:19,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:07:19,710 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:07:20,048 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:07:21,049 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:07:23,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:27,426 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:07:28,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:07:28,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:07:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:07:29,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:34,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:39,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:43,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:07:43,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:07:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:07:44,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:49,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:54,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:07:57,427 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:07:58,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:07:58,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:07:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:08:00,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:05,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:10,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:13,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:08:13,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:08:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:08:15,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:20,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:25,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:27,428 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:08:28,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:08:28,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:08:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:08:31,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:33,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:08:33,884 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:08:33,884 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:08:33,886 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:08:34,094 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:08:34,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:08:35,095 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:08:37,097 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:08:37,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:38,098 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:08:39,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:08:43,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:43,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:08:43,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:08:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:08:48,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:53,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:08:57,429 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:08:58,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:08:58,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:08:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:08:59,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:04,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:09,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:13,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:09:13,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:09:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:09:15,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:20,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:25,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:27,430 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:09:28,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:09:28,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:09:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:09:30,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:35,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:40,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:43,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:09:43,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:09:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:09:45,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:50,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:51,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:09:51,959 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:09:51,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:09:51,961 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:09:52,144 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:09:53,144 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:09:56,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:09:57,431 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:09:58,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:09:58,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:09:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:10:01,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:06,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:11,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:13,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:10:13,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:10:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:10:17,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:22,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:27,432 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:10:28,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:28,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:10:28,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:10:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:10:33,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:38,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:43,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:10:43,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:10:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:10:44,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:49,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:54,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:10:57,433 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:10:58,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:10:58,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:10:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:11:00,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:05,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:05,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:11:05,687 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:11:05,687 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:11:05,688 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:11:06,191 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:11:07,191 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:11:10,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:13,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:11:13,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:11:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:11:16,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:21,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:26,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:27,434 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:11:28,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:11:28,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:11:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:11:32,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:37,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:42,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:43,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:11:43,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:11:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:11:47,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:52,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:57,435 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:11:58,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:11:58,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:11:58,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:11:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:12:03,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:08,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:13,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:12:13,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:12:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:12:13,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:18,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:19,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:12:19,690 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:12:19,690 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:12:19,692 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:12:20,233 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:12:21,233 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:12:23,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:27,436 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:12:28,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:12:28,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:12:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:12:29,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:34,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:39,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:43,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:12:43,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:12:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:12:44,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:49,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:54,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:12:57,437 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:12:58,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:12:58,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:12:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:13:00,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:05,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:10,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:13,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:13:13,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:13:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:13:16,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:21,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:26,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:27,438 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:13:28,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:13:28,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:13:28,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:13:31,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:33,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:13:33,570 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:13:33,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:13:33,571 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:13:34,278 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:13:35,278 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:13:36,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:41,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:43,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:13:43,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:13:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:13:47,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:52,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:57,439 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:13:58,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:13:58,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:13:58,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:13:58,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:14:03,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:08,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:13,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:14:13,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:14:13,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:14:14,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:19,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:24,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:27,440 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:14:28,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:14:28,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:14:28,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:14:29,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:34,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:39,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:43,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:14:43,264 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:14:43,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:14:45,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:47,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:14:47,365 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:14:47,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:14:47,366 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:14:48,321 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:14:49,321 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:14:51,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:56,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:14:57,441 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:14:58,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:14:58,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:14:58,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:15:01,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:06,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:11,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:13,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:15:13,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:15:13,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:15:17,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:22,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:27,442 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:15:28,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:28,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:15:28,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:15:28,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:15:33,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:38,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:43,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:15:43,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:15:43,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:15:44,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:49,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:54,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:15:57,443 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:15:58,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:15:58,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:15:58,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:15:59,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:01,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:16:01,483 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:16:01,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:16:01,485 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:16:02,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:16:03,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:16:05,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:10,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:13,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:16:13,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:16:13,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:16:15,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:20,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:25,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:27,444 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:16:28,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:16:28,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:16:28,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:16:30,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:35,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:40,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:43,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:16:43,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:16:43,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:16:46,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:51,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:56,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:16:57,445 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:16:58,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:16:58,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:16:58,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:17:02,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:07,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:12,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:13,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:17:13,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:17:13,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:17:15,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:17:15,184 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:17:15,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:17:15,185 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:17:15,409 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:17:16,410 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:17:18,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:23,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:27,447 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:17:28,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:17:28,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:17:28,353 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:17:28,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:33,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:38,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:43,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:17:43,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:17:43,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:17:44,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:49,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:54,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:17:57,446 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:17:58,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:17:58,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:17:58,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:18:00,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:05,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:10,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:13,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:18:13,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:18:13,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:18:15,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:20,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:25,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:27,448 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:18:28,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:18:28,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:18:28,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:18:28,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:18:28,903 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:18:28,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:18:28,905 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:18:29,454 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:18:30,454 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:18:30,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:35,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:40,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:43,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:18:43,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:18:43,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:18:46,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:51,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:56,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:18:57,448 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:18:58,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:18:58,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:18:58,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:19:02,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:07,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:12,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:13,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:19:13,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:19:13,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:19:17,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:22,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:27,449 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:19:28,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:28,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:19:28,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:19:28,354 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:19:33,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:38,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:42,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:19:42,750 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:19:42,750 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:19:42,752 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:19:43,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:19:43,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:19:43,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:19:43,498 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:19:43,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:44,498 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:19:48,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:53,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:19:57,450 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:19:58,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:19:58,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:19:58,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:19:58,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:03,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:08,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:13,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:20:13,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:20:13,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:20:14,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:19,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:24,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:27,451 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:20:28,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:20:28,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:20:28,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:20:30,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:35,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:40,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:43,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:20:43,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:20:43,356 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:20:46,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:51,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:56,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:20:56,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:20:56,606 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:20:56,606 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:20:56,608 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:20:57,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:20:57,452 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:20:57,541 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:20:58,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:20:58,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:20:58,392 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:20:58,541 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:21:00,543 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:21:01,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:02,544 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:21:06,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:11,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:13,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:21:13,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:21:13,380 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:21:16,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:21,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:26,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:27,453 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:21:28,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:21:28,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:21:28,380 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:21:32,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:37,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:42,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:43,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:21:43,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:21:43,380 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:21:48,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:53,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:21:57,454 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:21:58,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:21:58,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:21:58,380 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:21:59,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:04,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:09,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:13,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:22:13,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:22:13,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:22:14,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:22:14,349 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:22:14,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:22:14,351 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:22:14,590 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:22:15,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:16,592 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:22:20,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:25,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:27,455 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:22:28,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:22:28,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:22:28,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:22:30,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:35,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:40,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:43,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:22:43,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:22:43,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:22:46,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:51,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:56,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:22:57,456 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:22:58,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:22:58,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:22:58,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:23:01,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:06,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:11,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:13,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:23:13,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:23:13,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:23:16,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:21,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:26,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:27,457 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:23:28,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:23:28,217 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:23:28,217 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:23:28,218 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:23:28,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:23:28,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:23:28,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:23:28,634 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:23:30,635 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:23:31,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:36,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:41,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:43,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:23:43,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:23:43,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:23:47,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:52,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:57,458 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:23:58,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:23:58,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:23:58,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:23:58,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:24:03,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:08,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:13,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:24:13,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:24:13,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:24:14,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:19,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:24,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:27,459 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:24:28,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:24:28,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:24:28,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:24:30,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:35,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:40,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:42,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:24:42,269 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:24:42,269 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:24:42,271 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:24:42,680 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:24:43,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:24:43,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:24:43,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:24:44,682 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:24:45,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:50,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:55,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:24:57,460 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:24:58,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:24:58,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:24:58,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:25:01,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:06,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:11,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:13,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:25:13,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:25:13,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:25:16,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:21,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:26,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:27,461 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:25:28,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:25:28,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:25:28,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:25:31,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:36,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:41,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:43,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:25:43,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:25:43,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:25:47,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:52,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:56,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:25:56,120 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:25:56,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:25:56,122 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:25:56,725 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:25:57,462 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:25:58,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:25:58,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:25:58,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:25:58,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:25:58,726 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:26:03,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:08,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:13,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:26:13,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:26:13,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:26:13,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:18,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:23,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:27,463 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:26:28,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:26:28,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:26:28,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:26:28,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:33,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:38,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:43,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:26:43,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:26:43,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:26:44,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:49,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:54,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:26:57,464 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:26:58,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:26:58,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:26:58,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:27:00,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:05,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:09,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:27:09,988 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:27:09,989 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:27:09,990 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:27:10,769 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:27:10,770 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:27:10,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:13,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:27:13,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:27:13,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:27:16,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:21,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:26,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:27,465 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:27:28,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:27:28,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:27:28,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:27:31,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:36,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:41,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:43,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:27:43,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:27:43,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:27:47,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:52,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:57,465 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:27:58,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:27:58,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:27:58,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:27:58,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:28:03,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:08,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:13,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:28:13,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:28:13,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:28:13,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:18,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:23,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:23,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:28:23,806 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:28:23,806 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:28:23,808 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:28:23,814 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:28:26,816 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:28:27,466 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:28:28,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:28:28,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:28:28,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:28:29,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:34,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:39,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:43,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:28:43,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:28:43,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:28:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:49,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:54,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:28:57,467 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:28:58,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:28:58,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:28:58,383 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:29:00,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:05,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:10,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:13,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:29:13,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:29:13,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:29:15,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:20,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:25,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:27,468 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:29:28,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:29:28,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:29:28,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:29:31,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:36,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:37,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:29:37,611 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:29:37,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:29:37,613 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:29:37,858 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:29:38,859 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:29:42,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:43,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:29:43,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:29:43,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:29:47,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:52,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:57,469 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:29:58,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:29:58,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:29:58,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:29:58,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:30:03,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:08,647 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:13,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:30:13,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:30:13,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:30:14,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:19,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:24,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:27,470 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:30:28,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:30:28,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:30:28,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:30:29,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:34,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:39,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:43,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:30:43,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:30:43,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:30:45,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:50,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:51,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:30:51,511 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:30:51,512 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:30:51,513 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:30:51,903 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:30:52,903 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:30:56,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:30:57,471 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:30:58,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:30:58,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:30:58,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:31:01,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:06,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:11,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:13,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:31:13,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:31:13,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:31:17,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:22,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:27,472 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:31:28,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:28,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:31:28,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:31:28,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:31:33,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:38,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:43,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:31:43,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:31:43,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:31:44,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:49,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:54,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:31:57,473 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:31:58,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:31:58,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:31:58,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:31:59,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:04,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:05,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:32:05,418 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:32:05,418 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:32:05,420 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:32:05,947 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:32:06,948 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:32:10,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:13,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:32:13,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:32:13,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:32:15,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:20,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:25,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:27,474 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:32:28,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:32:28,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:32:28,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:32:31,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:36,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:41,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:43,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:32:43,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:32:43,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:32:46,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:51,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:56,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:32:57,476 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:32:58,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:32:58,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:32:58,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:33:02,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:07,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:12,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:13,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:33:13,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:33:13,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:33:17,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:19,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:33:19,363 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:33:19,363 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:33:19,365 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:33:19,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:33:19,991 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:33:20,992 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:33:22,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:22,993 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:33:23,994 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:33:24,995 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:33:27,476 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:33:28,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:28,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:33:28,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:33:28,482 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:33:33,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:38,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:43,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:33:43,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:33:43,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:33:44,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:49,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:54,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:33:57,478 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:33:58,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:33:58,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:33:58,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:33:59,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:04,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:09,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:13,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:34:13,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:34:13,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:34:15,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:20,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:25,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:27,478 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:34:28,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:34:28,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:34:28,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:34:30,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:35,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:37,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:34:37,413 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:34:37,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:34:37,415 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:34:38,039 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:34:39,040 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:34:41,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:43,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:34:43,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:34:43,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:34:46,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:51,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:56,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:34:57,479 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:34:58,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:34:58,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:34:58,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:35:02,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:07,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:12,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:13,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:35:13,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:35:13,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:35:17,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:22,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:27,480 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:35:28,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:28,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:35:28,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:35:28,483 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:35:33,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:38,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:43,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:35:43,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:35:43,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:35:43,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:48,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:51,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:35:51,223 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:35:51,223 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:35:51,225 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:35:52,086 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:35:53,087 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:35:54,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:35:57,480 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:35:58,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:35:58,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:35:58,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:35:59,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:04,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:09,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:13,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:36:13,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:36:13,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:36:15,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:20,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:25,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:27,481 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:36:28,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:36:28,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:36:28,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:36:30,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:35,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:40,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:43,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:36:43,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:36:43,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:36:45,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:50,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:55,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:36:57,482 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:36:58,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:36:58,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:36:58,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:37:01,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:05,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:37:05,039 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:37:05,040 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:37:05,041 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:37:05,129 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:37:07,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:07,130 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:37:12,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:13,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:37:13,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:37:13,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:37:17,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:22,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:27,483 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:37:28,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:28,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:37:28,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:37:28,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:37:33,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:38,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:43,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:37:43,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:37:43,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:37:44,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:49,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:54,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:37:57,484 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:37:58,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:37:58,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:37:58,484 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:38:00,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:05,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:10,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:13,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:38:13,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:38:13,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:38:16,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:19,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:38:19,063 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:38:19,064 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:38:19,065 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:38:19,174 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:38:21,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:38:22,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:27,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:27,485 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:38:28,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:38:28,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:38:28,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:38:32,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:37,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:42,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:43,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:38:43,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:38:43,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:38:47,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:52,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:57,486 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:38:58,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:38:58,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:38:58,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:38:58,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:39:03,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:08,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:13,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:39:13,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:39:13,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:39:14,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:19,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:24,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:27,487 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:39:28,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:39:28,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:39:28,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:39:30,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:32,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:39:32,866 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:39:32,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:39:32,868 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:39:33,218 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:39:35,220 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:39:35,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:40,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:43,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:39:43,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:39:43,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:39:46,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:51,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:56,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:39:57,488 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:39:58,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:39:58,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:39:58,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:40:02,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:07,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:12,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:13,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:40:13,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:40:13,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:40:17,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:22,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:27,490 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:40:28,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:28,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:40:28,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:40:28,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:40:33,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:38,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:43,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:40:43,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:40:43,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:40:44,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:46,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:40:46,668 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:40:46,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:40:46,671 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:40:47,262 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:40:49,263 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:40:50,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:55,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:40:57,490 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:40:58,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:40:58,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:40:58,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:41:00,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:05,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:10,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:13,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:41:13,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:41:13,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:41:16,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:21,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:26,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:27,491 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:41:28,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:41:28,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:41:28,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:41:31,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:36,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:41,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:43,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:41:43,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:41:43,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:41:47,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:52,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:57,491 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:41:58,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:41:58,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:41:58,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:41:58,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:42:00,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:42:00,446 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:42:00,447 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:42:00,448 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:42:01,307 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:42:01,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:42:03,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:08,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:13,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:42:13,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:42:13,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:42:13,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:18,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:23,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:27,492 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:42:28,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:42:28,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:42:28,486 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:42:28,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:33,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:38,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:43,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:42:43,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:42:43,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:42:43,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:48,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:53,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:42:57,493 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:42:58,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:42:58,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:42:58,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:42:58,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:03,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:08,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:13,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:43:13,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:43:13,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:43:14,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:43:14,041 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:43:14,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:43:14,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:14,043 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:43:14,351 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:43:15,351 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:43:19,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:24,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:27,494 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:43:28,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:43:28,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:43:28,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:43:29,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:34,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:39,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:43,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:43:43,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:43:43,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:43:45,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:50,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:55,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:43:57,495 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:43:58,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:43:58,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:43:58,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:44:00,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:05,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:10,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:13,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:44:13,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:44:13,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:44:15,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:20,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:25,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:27,497 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:44:27,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:44:27,955 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:44:27,955 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:44:27,957 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:44:28,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:44:28,394 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:44:28,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:44:28,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:44:29,395 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:44:31,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:36,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:41,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:43,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:44:43,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:44:43,488 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:44:46,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:51,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:56,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:44:57,497 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:44:58,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:44:58,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:44:58,487 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:45:01,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:06,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:11,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:13,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:45:13,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:45:13,488 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:45:17,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:22,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:27,498 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:45:28,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:28,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:45:28,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:45:28,488 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:45:33,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:38,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:41,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:45:41,858 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:45:41,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:45:41,860 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:45:42,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:45:42,438 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:45:43,439 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:45:43,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:45:43,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:45:43,634 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:45:43,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:45,440 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:45:46,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:45:47,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:45:48,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:53,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:45:57,499 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:45:58,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:45:58,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:45:58,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:45:59,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:04,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:09,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:13,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:46:13,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:46:13,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:46:15,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:20,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:25,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:27,500 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:46:28,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:46:28,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:46:28,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:46:31,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:36,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:41,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:43,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:46:43,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:46:43,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:46:46,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:51,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:56,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:46:57,501 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:46:58,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:46:58,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:46:58,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:46:59,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:46:59,746 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:46:59,746 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:46:59,748 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:47:00,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:47:01,486 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:47:02,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:07,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:12,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:13,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:47:13,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:47:13,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:47:17,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:22,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:27,503 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:47:28,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:28,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:47:28,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:47:28,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:47:33,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:38,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:43,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:47:43,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:47:43,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:47:43,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:48,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:53,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:47:57,503 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:47:58,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:47:58,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:47:58,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:47:58,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:03,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:08,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:13,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:48:13,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:48:13,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:48:13,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:48:13,793 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:48:13,793 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:48:13,795 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:48:14,530 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:48:14,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:15,531 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:48:19,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:24,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:27,504 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:48:28,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:48:28,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:48:28,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:48:30,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:35,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:40,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:43,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:48:43,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:48:43,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:48:45,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:50,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:55,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:48:57,506 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:48:58,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:48:58,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:48:58,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:49:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:05,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:10,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:13,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:49:13,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:49:13,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:49:16,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:21,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:26,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:27,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:49:27,506 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:49:27,507 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:49:27,509 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:49:27,509 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:49:27,573 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:49:28,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:49:28,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:49:28,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:49:29,575 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:49:32,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:37,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:42,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:43,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:49:43,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:49:43,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:49:47,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:52,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:57,507 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:49:58,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:49:58,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:49:58,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:49:58,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:50:03,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:08,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:13,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:50:13,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:50:13,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:50:14,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:19,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:24,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:27,508 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:50:28,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:50:28,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:50:28,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:50:30,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:35,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:40,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:41,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:50:41,398 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:50:41,399 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:50:41,400 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:50:41,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:50:43,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:50:43,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:50:43,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:50:43,619 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:50:46,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:51,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:56,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:50:57,509 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:50:58,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:50:58,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:50:58,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:51:01,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:06,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:11,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:13,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:51:13,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:51:13,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:51:16,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:21,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:26,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:27,510 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:51:28,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:51:28,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:51:28,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:51:31,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:36,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:41,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:43,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:51:43,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:51:43,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:51:47,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:52,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:55,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:51:55,200 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:51:55,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:51:55,202 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:51:55,662 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:51:57,511 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:51:57,663 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:51:58,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:51:58,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:51:58,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:51:58,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:52:03,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:08,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:13,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:52:13,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:52:13,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:52:13,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:18,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:23,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:27,512 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:52:28,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:52:28,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:52:28,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:52:29,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:34,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:39,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:43,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:52:43,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:52:43,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:52:44,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:49,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:54,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:52:57,513 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:52:58,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:52:58,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:52:58,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:53:00,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:05,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:09,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:53:09,009 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:53:09,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:53:09,011 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:53:09,707 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:53:09,707 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:53:11,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:13,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:53:13,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:53:13,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:53:16,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:21,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:26,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:27,514 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:53:28,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:53:28,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:53:28,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:53:31,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:36,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:41,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:43,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:53:43,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:53:43,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:53:47,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:52,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:57,515 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:53:58,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:53:58,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:53:58,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:53:58,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:54:03,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:08,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:13,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:54:13,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:54:13,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:54:13,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:18,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:22,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:54:22,998 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:54:22,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:54:23,000 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:54:23,751 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:54:23,752 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:54:24,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:27,516 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:54:28,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:54:28,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:54:28,590 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:54:29,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:34,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:39,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:43,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:54:43,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:54:43,590 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:54:44,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:49,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:54,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:54:57,516 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:54:58,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:54:58,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:54:58,590 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:55:00,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:05,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:10,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:13,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:55:13,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:55:13,590 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:55:16,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:21,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:26,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:27,517 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:55:28,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:55:28,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:55:28,590 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:55:31,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:36,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:36,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:55:36,868 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:55:36,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:55:36,869 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:55:37,797 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:55:37,797 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:55:41,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:43,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:55:43,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:55:43,623 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:55:46,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:51,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:56,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:55:57,518 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:55:58,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:55:58,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:55:58,623 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:56:02,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:07,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:12,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:13,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:56:13,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:56:13,623 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:56:18,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:23,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:27,519 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:56:28,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:56:28,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:56:28,623 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:56:28,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:33,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:38,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:43,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:56:43,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:56:43,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:56:44,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:49,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:50,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:56:50,671 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:56:50,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:56:50,672 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:56:50,844 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:56:51,844 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:56:55,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:56:57,520 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:56:58,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:56:58,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:56:58,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:57:00,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:05,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:10,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:13,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:57:13,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:57:13,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:57:15,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:20,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:25,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:27,521 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:57:28,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:57:28,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:57:28,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:57:31,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:36,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:41,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:43,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:57:43,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:57:43,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:57:46,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:51,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:56,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:57:57,522 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:57:58,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:57:58,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:57:58,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:58:01,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:04,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:58:04,562 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:58:04,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:58:04,563 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:58:04,889 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:58:05,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:58:05,890 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:58:07,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:07,891 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:58:09,892 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:58:13,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:13,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:58:13,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:58:13,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:58:18,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:23,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:27,523 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:58:28,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:58:28,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:58:28,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:58:28,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:33,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:38,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:43,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:58:43,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:58:43,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:58:44,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:49,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:54,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:58:57,524 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:58:58,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:58:58,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:58:58,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:59:00,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:05,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:10,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:13,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:59:13,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:59:13,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:59:15,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:20,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:22,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 17:59:22,369 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 17:59:22,370 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 17:59:22,371 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 17:59:22,936 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 17:59:23,936 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 17:59:26,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:27,525 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:59:28,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:59:28,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:59:28,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:59:31,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:36,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:41,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:43,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:59:43,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:59:43,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 17:59:46,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:51,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:56,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 17:59:57,526 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 17:59:58,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 17:59:58,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 17:59:58,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:00:01,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:06,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:11,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:13,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:00:13,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:00:13,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:00:16,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:21,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:26,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:27,526 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:00:28,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:00:28,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:00:28,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:00:32,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:36,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:00:36,259 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:00:36,259 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:00:36,260 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:00:36,979 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:00:38,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:38,980 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:00:43,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:00:43,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:00:43,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:00:43,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:48,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:53,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:00:57,527 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:00:58,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:00:58,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:00:58,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:00:59,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:04,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:09,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:13,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:01:13,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:01:13,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:01:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:20,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:25,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:27,528 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:01:28,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:01:28,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:01:28,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:01:30,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:35,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:40,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:43,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:01:43,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:01:43,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:01:45,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:49,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:01:50,000 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:01:50,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:01:50,002 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:01:50,023 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:01:51,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:53,024 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:01:56,003 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:01:57,529 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:01:58,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:01:58,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:01:58,625 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:02:01,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:06,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:11,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:13,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:02:13,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:02:13,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:02:16,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:21,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:26,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:27,530 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:02:28,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:02:28,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:02:28,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:02:32,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:37,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:42,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:43,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:02:43,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:02:43,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:02:48,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:53,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:02:57,532 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:02:58,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:02:58,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:02:58,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:02:59,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:04,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:03:04,084 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:03:04,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:03:04,086 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:03:05,070 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:03:05,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:07,071 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:03:10,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:13,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:03:13,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:03:13,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:03:15,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:20,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:25,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:27,532 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:03:28,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:03:28,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:03:28,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:03:31,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:36,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:41,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:43,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:03:43,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:03:43,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:03:47,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:52,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:57,534 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:03:58,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:03:58,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:03:58,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:03:58,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:04:03,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:08,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:13,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:04:13,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:04:13,626 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:04:13,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:17,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:04:17,752 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:04:17,752 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:04:17,754 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:04:18,129 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:04:19,129 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:04:19,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:24,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:27,534 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:04:28,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:04:28,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:04:28,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:04:29,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:34,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:39,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:43,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:04:43,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:04:43,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:04:44,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:49,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:54,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:04:57,535 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:04:58,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:04:58,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:04:58,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:05:00,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:05,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:10,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:13,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:05:13,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:05:13,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:05:15,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:20,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:25,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:27,536 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:05:28,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:05:28,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:05:28,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:05:30,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:31,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:05:31,534 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:05:31,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:05:31,536 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:05:32,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:05:33,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:05:36,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:41,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:43,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:05:43,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:05:43,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:05:46,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:51,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:56,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:05:57,537 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:05:58,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:05:58,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:05:58,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:06:02,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:07,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:12,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:13,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:06:13,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:06:13,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:06:17,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:22,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:27,564 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:06:28,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:28,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:06:28,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:06:28,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:06:33,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:38,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:43,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:06:43,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:06:43,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:06:44,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:06:45,610 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:06:45,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:06:45,612 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:06:46,222 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:06:47,223 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:06:50,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:55,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:06:57,565 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:06:58,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:06:58,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:06:58,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:07:00,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:05,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:10,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:13,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:07:13,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:07:13,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:07:16,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:21,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:26,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:27,566 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:07:28,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:07:28,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:07:28,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:07:32,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:37,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:42,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:43,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:07:43,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:07:43,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:07:47,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:52,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:57,567 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:07:58,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:07:58,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:07:58,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:07:58,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:07:59,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:07:59,344 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:07:59,345 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:07:59,346 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:08:00,281 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:08:01,281 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:08:03,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:08,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:13,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:08:13,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:08:13,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:08:13,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:18,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:23,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:27,567 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:08:28,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:08:28,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:08:28,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:08:29,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:34,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:39,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:43,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:08:43,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:08:43,723 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:08:44,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:49,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:54,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:08:57,568 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:08:58,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:08:58,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:08:58,723 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:09:00,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:05,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:10,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:13,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:09:13,218 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:09:13,218 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:09:13,220 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:09:13,325 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:09:13,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:09:13,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:09:13,723 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:09:15,326 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:09:16,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:21,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:26,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:27,569 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:09:28,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:09:28,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:09:28,723 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:09:32,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:37,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:42,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:43,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:09:43,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:09:43,723 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:09:47,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:52,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:57,570 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:09:58,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:09:58,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:09:58,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:09:58,723 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:10:03,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:08,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:13,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:10:13,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:10:13,724 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:10:14,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:19,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:24,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:26,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:10:26,992 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:10:26,993 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:10:26,994 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:10:27,370 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:10:27,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:10:27,571 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:10:28,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:10:28,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:10:28,791 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:10:29,371 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:10:30,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:30,372 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:10:31,372 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:10:33,373 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:10:36,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:41,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:43,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:10:43,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:10:43,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:10:46,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:51,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:56,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:10:57,572 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:10:58,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:10:58,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:10:58,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:11:01,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:06,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:11,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:13,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:11:13,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:11:13,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:11:17,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:22,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:27,573 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:11:28,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:28,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:11:28,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:11:28,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:11:34,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:39,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:43,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:11:43,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:11:43,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:11:44,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:11:44,953 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:11:44,953 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:11:44,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:44,955 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:11:45,415 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:11:47,416 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:11:49,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:54,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:11:57,574 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:11:58,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:11:58,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:11:58,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:12:00,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:05,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:10,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:13,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:12:13,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:12:13,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:12:16,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:21,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:26,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:27,575 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:12:28,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:12:28,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:12:28,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:12:32,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:37,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:42,003 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:43,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:12:43,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:12:43,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:12:47,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:52,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:57,576 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:12:58,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:12:58,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:12:58,681 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:12:58,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:12:58,682 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:12:58,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:12:58,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:12:58,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:12:59,459 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:13:01,461 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:13:03,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:08,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:13,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:13:13,729 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:13:13,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:13:13,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:18,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:23,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:27,577 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:13:28,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:13:28,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:13:28,823 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:13:29,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:34,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:39,003 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:43,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:13:43,822 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:13:43,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:13:44,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:49,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:54,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:13:57,578 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:13:58,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:13:58,822 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:13:58,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:13:59,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:04,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:09,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:12,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:14:12,607 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:14:12,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:14:12,609 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:14:13,503 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:14:13,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:14:13,823 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:14:13,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:14:15,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:15,505 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:14:20,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:25,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:27,581 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:14:28,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:14:28,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:14:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:14:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:35,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:40,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:43,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:14:43,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:14:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:14:45,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:50,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:55,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:14:57,580 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:14:58,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:14:58,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:14:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:15:01,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:06,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:11,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:13,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:15:13,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:15:13,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:15:16,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:21,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:26,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:26,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:15:26,429 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:15:26,429 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:15:26,431 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:15:26,548 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:15:27,548 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:15:27,580 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:15:28,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:15:28,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:15:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:15:31,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:36,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:41,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:43,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:15:43,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:15:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:15:47,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:52,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:57,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:15:57,582 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:15:58,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:15:58,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:15:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:16:02,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:07,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:12,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:13,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:16:13,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:16:13,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:16:18,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:23,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:27,582 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:16:28,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:28,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:16:28,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:16:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:16:34,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:39,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:40,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:16:40,245 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:16:40,246 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:16:40,247 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:16:40,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:16:41,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:16:43,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:16:43,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:16:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:16:44,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:49,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:54,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:16:57,583 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:16:58,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:16:58,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:16:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:16:59,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:04,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:09,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:13,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:17:13,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:17:13,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:17:15,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:20,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:25,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:27,584 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:17:28,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:17:28,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:17:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:17:30,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:35,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:40,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:43,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:17:43,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:17:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:17:46,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:51,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:53,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:17:53,951 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:17:53,951 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:17:53,952 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:17:54,640 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:17:55,641 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:17:56,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:17:57,586 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:17:58,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:17:58,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:17:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:18:02,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:07,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:12,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:13,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:18:13,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:18:13,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:18:18,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:23,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:27,586 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:18:28,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:28,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:18:28,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:18:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:18:34,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:39,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:43,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:18:43,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:18:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:18:45,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:50,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:55,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:18:57,588 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:18:58,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:18:58,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:18:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:19:00,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:05,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:08,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:19:08,006 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:19:08,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:19:08,008 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:19:08,685 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:19:09,685 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:19:11,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:13,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:19:13,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:19:13,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:19:17,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:22,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:27,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:27,588 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:19:28,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:19:28,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:19:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:19:32,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:37,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:42,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:43,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:19:43,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:19:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:19:48,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:53,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:57,589 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:19:58,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:19:58,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:19:58,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:19:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:20:04,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:09,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:13,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:20:13,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:20:13,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:20:14,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:19,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:21,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:20:21,775 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:20:21,776 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:20:21,777 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:20:22,732 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:20:23,733 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:20:24,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:27,589 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:20:28,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:20:28,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:20:28,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:20:30,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:35,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:40,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:43,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:20:43,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:20:43,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:20:46,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:51,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:56,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:20:57,590 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:20:58,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:20:58,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:20:58,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:21:01,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:06,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:11,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:13,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:21:13,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:21:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:21:16,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:21,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:26,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:27,591 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:21:28,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:21:28,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:21:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:21:32,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:35,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:21:35,655 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:21:35,655 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:21:35,656 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:21:35,776 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:21:37,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:37,777 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:21:42,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:43,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:21:43,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:21:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:21:48,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:53,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:57,592 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:21:58,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:21:58,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:21:58,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:21:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:22:04,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:09,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:13,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:22:13,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:22:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:22:15,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:20,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:25,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:27,593 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:22:28,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:22:28,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:22:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:22:30,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:35,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:40,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:43,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:22:43,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:22:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:22:45,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:49,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:22:49,621 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:22:49,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:22:49,623 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:22:49,822 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:22:50,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:50,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:22:51,823 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:22:52,824 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:22:53,825 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:22:55,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:22:55,826 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:22:57,594 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:22:58,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:22:58,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:22:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:23:01,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:06,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:11,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:13,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:23:13,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:23:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:23:17,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:22,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:27,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:27,595 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:23:28,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:23:28,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:23:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:23:32,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:37,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:42,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:43,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:23:43,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:23:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:23:48,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:53,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:57,596 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:23:58,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:23:58,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:23:58,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:23:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:24:04,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:07,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:24:07,958 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:24:07,959 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:24:07,960 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:24:08,871 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:24:09,871 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:24:10,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:13,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:24:13,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:24:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:24:15,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:20,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:25,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:27,597 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:24:28,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:24:28,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:24:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:24:31,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:36,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:41,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:43,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:24:43,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:24:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:24:46,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:51,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:56,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:24:57,598 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:24:58,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:24:58,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:24:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:25:02,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:07,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:12,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:13,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:25:13,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:25:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:25:17,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:21,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:25:21,735 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:25:21,735 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:25:21,736 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:25:21,915 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:25:22,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:23,916 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:25:27,600 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:25:28,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:28,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:25:28,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:25:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:25:34,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:39,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:43,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:25:43,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:25:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:25:44,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:49,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:54,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:25:57,600 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:25:58,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:25:58,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:25:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:26:00,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:05,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:10,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:13,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:26:13,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:26:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:26:16,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:21,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:26,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:27,601 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:26:28,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:26:28,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:26:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:26:31,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:35,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:26:35,614 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:26:35,614 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:26:35,615 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:26:35,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:26:36,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:37,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:26:41,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:43,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:26:43,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:26:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:26:47,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:52,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:57,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:26:57,602 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:26:58,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:26:58,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:26:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:27:03,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:08,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:13,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:13,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:27:13,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:27:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:27:19,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:24,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:27,603 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:27:28,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:27:28,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:27:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:27:29,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:34,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:39,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:43,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:27:43,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:27:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:27:45,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:49,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:27:49,554 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:27:49,554 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:27:49,556 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:27:50,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:27:50,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:52,007 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:27:55,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:27:57,604 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:27:58,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:27:58,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:27:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:28:01,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:06,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:11,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:13,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:28:13,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:28:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:28:17,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:22,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:27,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:27,605 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:28:28,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:28:28,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:28:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:28:32,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:37,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:42,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:43,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:28:43,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:28:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:28:48,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:53,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:57,606 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:28:58,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:28:58,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:28:58,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:28:58,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:29:03,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:29:03,485 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:29:03,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:29:03,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:03,487 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:29:04,056 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:29:06,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:29:08,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:13,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:29:13,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:29:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:29:14,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:19,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:24,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:27,607 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:29:28,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:29:28,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:29:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:29:29,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:34,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:39,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:43,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:29:43,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:29:43,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:29:44,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:49,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:54,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:29:57,608 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:29:58,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:29:58,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:29:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:29:59,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:04,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:09,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:13,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:30:13,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:30:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:30:15,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:17,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:30:17,722 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:30:17,722 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:30:17,724 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:30:18,104 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:30:20,105 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:30:20,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:25,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:27,609 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:30:28,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:30:28,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:30:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:30:31,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:36,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:41,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:43,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:30:43,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:30:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:30:46,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:51,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:56,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:30:57,610 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:30:58,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:30:58,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:30:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:31:02,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:07,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:12,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:13,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:31:13,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:31:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:31:17,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:22,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:27,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:27,611 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:31:28,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:31:28,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:31:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:31:31,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:31:31,982 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:31:31,982 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:31:31,983 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:31:32,150 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:31:33,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:34,151 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:31:38,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:43,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:43,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:31:43,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:31:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:31:48,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:53,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:57,612 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:31:58,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:31:58,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:31:58,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:31:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:32:04,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:09,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:13,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:32:13,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:32:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:32:14,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:19,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:24,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:27,613 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:32:28,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:32:28,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:32:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:32:29,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:34,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:39,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:43,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:32:43,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:32:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:32:45,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:45,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:32:45,690 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:32:45,690 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:32:45,691 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:32:46,195 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:32:48,196 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:32:50,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:55,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:32:57,613 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:32:58,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:32:58,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:32:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:33:01,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:06,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:11,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:13,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:33:13,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:33:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:33:17,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:22,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:27,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:27,614 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:33:28,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:33:28,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:33:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:33:33,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:38,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:43,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:43,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:33:43,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:33:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:33:48,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:53,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:57,617 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:33:58,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:33:58,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:33:58,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:33:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:33:59,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:33:59,626 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:33:59,627 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:33:59,628 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:34:00,239 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:34:00,239 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:34:03,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:08,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:13,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:34:13,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:34:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:34:14,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:19,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:24,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:27,616 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:34:28,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:34:28,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:34:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:34:29,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:34,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:39,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:43,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:34:43,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:34:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:34:44,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:49,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:54,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:34:57,618 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:34:58,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:34:58,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:34:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:34:59,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:04,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:09,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:13,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:35:13,527 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:35:13,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:35:13,528 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:35:13,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:35:13,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:35:13,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:35:14,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:35:14,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:14,282 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:35:14,283 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:35:16,284 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:35:18,285 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:35:19,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:24,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:27,618 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:35:28,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:35:28,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:35:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:35:30,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:35,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:40,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:43,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:35:43,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:35:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:35:45,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:50,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:55,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:35:57,619 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:35:58,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:35:58,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:35:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:36:01,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:06,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:11,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:13,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:36:13,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:36:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:36:16,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:21,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:26,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:27,620 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:36:28,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:36:28,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:36:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:36:31,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:36:31,322 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:36:31,323 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:36:31,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:31,324 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:36:31,331 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:36:32,332 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:36:36,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:41,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:43,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:36:43,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:36:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:36:47,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:52,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:57,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:36:57,621 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:36:58,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:36:58,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:36:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:37:03,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:08,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:13,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:13,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:37:13,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:37:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:37:18,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:23,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:27,622 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:37:28,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:28,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:37:28,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:37:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:37:34,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:39,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:43,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:37:43,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:37:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:37:44,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:45,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:37:45,074 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:37:45,074 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:37:45,075 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:37:45,376 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:37:46,377 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:37:50,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:55,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:37:57,623 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:37:58,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:37:58,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:37:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:38:01,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:06,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:11,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:13,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:38:13,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:38:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:38:16,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:21,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:26,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:27,624 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:38:28,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:38:28,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:38:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:38:32,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:37,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:42,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:43,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:38:43,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:38:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:38:48,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:53,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:57,625 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:38:58,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:38:58,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:38:58,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:38:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:38:59,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:38:59,061 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:38:59,062 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:38:59,064 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:38:59,421 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:39:01,422 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:39:04,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:09,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:13,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:39:13,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:39:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:39:14,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:19,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:24,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:27,626 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:39:28,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:39:28,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:39:28,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:39:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:34,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:39,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:43,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:39:43,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:39:43,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:39:45,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:50,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:55,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:39:57,627 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:39:58,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:39:58,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:39:58,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:40:00,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:05,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:10,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:13,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:40:13,084 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:40:13,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:40:13,085 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:40:13,466 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:40:13,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:40:13,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:40:13,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:40:15,467 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:40:16,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:21,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:26,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:27,628 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:40:28,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:40:28,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:40:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:40:31,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:36,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:41,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:43,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:40:43,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:40:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:40:47,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:52,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:57,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:40:57,630 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:40:58,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:40:58,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:40:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:41:03,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:08,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:13,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:13,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:41:13,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:41:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:41:18,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:23,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:27,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:41:27,386 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:41:27,386 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:41:27,387 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:41:27,510 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:41:27,630 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:41:28,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:28,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:41:28,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:41:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:41:29,511 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:41:34,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:39,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:43,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:41:43,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:41:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:41:44,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:49,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:54,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:41:57,631 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:41:58,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:41:58,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:41:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:41:59,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:04,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:09,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:13,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:42:13,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:42:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:42:15,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:20,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:25,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:27,632 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:42:28,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:42:28,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:42:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:42:31,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:36,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:41,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:41,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:42:41,292 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:42:41,293 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:42:41,294 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:42:41,554 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:42:43,556 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:42:43,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:42:43,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:42:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:42:46,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:51,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:56,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:42:57,633 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:42:58,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:42:58,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:42:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:43:01,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:06,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:11,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:13,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:43:13,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:43:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:43:16,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:21,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:26,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:27,634 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:43:28,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:43:28,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:43:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:43:32,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:37,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:42,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:43,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:43:43,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:43:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:43:48,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:53,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:55,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:43:55,302 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:43:55,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:43:55,304 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:43:55,600 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:43:57,601 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:43:57,635 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:43:58,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:43:58,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:43:58,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:43:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:44:04,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:09,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:13,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:44:13,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:44:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:44:15,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:20,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:25,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:27,636 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:44:28,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:44:28,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:44:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:44:30,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:35,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:40,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:43,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:44:43,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:44:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:44:45,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:50,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:55,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:44:57,637 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:44:58,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:44:58,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:44:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:45:01,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:06,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:09,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:45:09,129 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:45:09,130 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:45:09,131 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:45:09,645 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:45:11,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:11,647 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:45:13,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:45:13,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:45:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:45:17,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:22,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:27,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:27,637 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:45:28,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:45:28,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:45:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:45:32,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:37,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:42,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:43,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:45:43,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:45:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:45:47,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:52,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:57,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:45:57,638 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:45:58,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:45:58,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:45:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:46:02,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:07,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:12,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:13,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:46:13,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:46:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:46:18,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:22,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:46:22,831 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:46:22,831 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:46:22,833 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:46:23,689 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:46:23,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:25,690 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:46:27,639 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:46:28,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:46:28,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:46:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:46:29,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:34,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:39,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:43,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:46:43,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:46:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:46:45,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:50,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:55,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:46:57,640 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:46:58,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:46:58,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:46:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:47:01,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:06,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:11,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:13,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:47:13,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:47:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:47:16,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:21,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:26,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:27,641 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:47:28,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:47:28,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:47:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:47:32,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:36,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:47:36,847 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:47:36,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:47:36,848 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:47:37,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:37,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:47:37,734 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:47:39,735 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:47:40,736 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:47:42,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:43,738 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:47:43,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:47:43,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:47:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:47:48,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:53,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:57,644 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:47:58,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:47:58,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:47:58,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:47:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:48:04,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:09,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:13,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:48:13,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:48:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:48:14,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:19,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:24,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:27,643 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:48:28,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:48:28,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:48:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:48:30,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:35,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:40,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:43,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:48:43,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:48:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:48:46,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:51,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:55,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:48:55,042 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:48:55,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:48:55,044 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:48:55,783 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:48:56,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:48:57,644 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:48:57,784 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:48:58,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:48:58,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:48:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:49:02,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:07,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:12,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:13,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:49:13,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:49:13,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:49:17,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:22,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:27,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:27,646 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:49:28,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:49:28,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:49:28,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:49:32,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:37,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:42,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:43,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:49:43,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:49:43,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:49:48,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:53,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:57,646 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:49:58,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:49:58,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:49:58,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:49:58,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:50:04,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:08,997 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:50:08,999 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:50:08,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:50:09,005 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:50:09,827 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:50:10,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:11,828 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:50:13,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:50:13,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:50:13,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:50:15,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:20,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:25,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:27,647 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:50:28,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:50:28,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:50:28,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:50:30,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:35,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:40,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:43,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:50:43,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:50:43,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:50:46,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:51,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:56,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:50:57,648 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:50:58,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:50:58,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:50:58,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:51:02,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:07,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:12,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:13,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:51:13,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:51:13,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:51:17,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:22,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:23,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:51:23,109 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:51:23,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:51:23,110 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:51:23,876 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:51:25,878 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:51:27,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:27,649 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:51:28,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:51:28,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:51:28,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:51:33,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:38,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:43,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:43,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:51:43,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:51:43,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:51:49,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:54,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:51:57,650 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:51:58,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:51:58,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:51:58,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:51:59,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:04,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:09,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:13,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:52:13,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:52:13,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:52:15,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:20,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:25,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:27,651 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:52:28,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:52:28,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:52:28,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:52:31,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:36,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:36,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:52:36,979 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:52:36,979 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:52:36,981 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:52:37,919 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:52:39,920 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:52:42,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:43,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:52:43,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:52:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:52:47,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:52,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:57,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:52:57,652 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:52:58,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:52:58,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:52:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:53:03,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:08,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:13,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:13,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:53:13,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:53:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:53:18,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:23,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:27,653 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:53:28,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:28,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:53:28,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:53:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:53:34,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:39,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:43,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:53:43,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:53:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:53:45,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:50,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:50,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:53:50,898 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:53:50,898 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:53:50,900 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:53:50,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:53:51,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:53:55,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:53:57,653 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:53:58,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:53:58,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:53:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:54:01,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:06,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:11,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:13,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:54:13,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:54:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:54:16,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:21,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:26,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:27,654 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:54:28,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:54:28,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:54:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:54:32,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:37,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:42,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:43,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:54:43,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:54:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:54:47,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:52,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:57,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:54:57,655 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:54:58,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:54:58,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:54:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:55:03,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:04,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:55:04,846 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:55:04,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:55:04,847 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:55:05,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:55:06,007 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:55:08,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:13,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:55:13,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:55:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:55:14,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:19,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:24,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:27,656 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:55:28,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:55:28,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:55:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:55:29,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:34,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:39,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:43,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:55:43,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:55:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:55:45,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:50,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:55,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:55:57,657 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:55:58,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:55:58,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:55:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:56:01,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:06,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:11,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:13,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:56:13,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:56:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:56:16,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:18,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:56:18,749 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:56:18,749 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:56:18,750 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:56:19,050 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:56:20,051 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:56:21,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:26,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:27,658 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:56:28,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:56:28,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:56:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:56:32,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:37,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:42,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:43,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:56:43,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:56:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:56:48,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:53,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:57,659 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:56:58,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:56:58,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:56:58,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:56:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:57:04,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:09,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:13,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:57:13,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:57:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:57:15,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:20,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:25,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:27,660 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:57:28,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:57:28,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:57:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:57:31,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:32,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:57:32,664 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:57:32,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:57:32,665 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:57:33,095 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:57:34,096 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:57:36,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:41,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:43,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:57:43,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:57:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:57:47,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:52,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:57,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:57:57,661 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:57:58,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:57:58,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:57:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:58:03,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:08,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:13,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:13,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:58:13,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:58:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:58:18,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:23,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:27,662 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:58:28,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:28,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:58:28,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:58:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:58:34,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:39,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:43,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:58:43,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:58:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:58:44,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:46,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 18:58:46,602 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 18:58:46,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 18:58:46,603 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 18:58:47,140 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 18:58:48,141 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 18:58:49,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:54,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:58:57,663 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:58:58,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:58:58,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:58:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:59:00,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:05,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:10,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:13,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:59:13,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:59:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:59:15,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:20,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:25,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:27,664 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:59:28,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:59:28,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:59:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:59:30,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:35,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:40,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:43,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:59:43,927 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:59:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 18:59:45,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:50,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:55,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 18:59:57,665 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 18:59:58,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 18:59:58,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 18:59:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:00:00,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:00,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:00:00,477 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:00:00,477 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:00:00,479 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:00:01,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:00:01,184 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:00:02,185 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:00:04,187 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:00:05,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:06,188 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:00:10,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:13,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:00:13,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:00:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:00:16,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:21,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:26,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:27,666 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:00:28,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:00:28,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:00:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:00:32,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:37,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:42,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:43,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:00:43,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:00:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:00:48,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:53,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:57,667 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:00:58,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:00:58,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:00:58,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:00:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:01:04,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:09,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:13,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:01:13,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:01:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:01:15,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:18,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:01:18,422 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:01:18,423 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:01:18,424 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:01:19,231 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:01:20,232 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:01:20,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:25,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:27,668 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:01:28,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:01:28,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:01:28,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:01:31,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:36,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:41,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:43,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:01:43,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:01:43,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:01:47,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:52,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:57,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:01:57,669 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:01:58,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:01:58,928 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:01:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:02:02,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:07,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:12,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:13,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:02:13,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:02:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:02:17,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:22,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:27,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:27,670 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:02:28,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:02:28,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:02:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:02:32,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:02:32,298 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:02:32,298 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:02:32,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:32,300 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:02:33,276 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:02:34,276 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:02:37,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:42,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:43,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:02:43,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:02:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:02:48,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:53,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:57,671 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:02:58,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:02:58,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:02:58,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:02:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:03:04,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:09,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:13,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:03:13,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:03:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:03:15,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:20,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:25,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:27,672 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:03:28,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:03:28,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:03:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:03:30,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:35,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:40,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:43,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:03:43,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:03:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:03:46,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:46,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:03:46,261 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:03:46,262 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:03:46,263 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:03:46,319 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:03:48,320 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:03:51,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:56,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:03:57,673 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:03:58,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:03:58,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:03:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:04:02,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:07,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:12,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:13,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:04:13,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:04:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:04:17,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:22,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:27,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:27,674 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:04:28,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:04:28,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:04:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:04:33,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:38,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:43,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:43,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:04:43,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:04:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:04:48,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:53,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:57,675 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:04:58,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:04:58,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:04:58,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:04:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:05:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:05:00,272 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:05:00,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:05:00,274 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:05:00,365 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:05:02,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:05:04,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:09,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:13,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:05:13,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:05:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:05:15,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:20,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:25,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:27,676 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:05:28,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:05:28,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:05:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:05:31,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:36,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:41,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:43,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:05:43,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:05:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:05:47,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:52,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:57,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:05:57,677 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:05:58,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:05:58,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:05:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:06:02,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:07,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:12,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:13,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:06:13,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:06:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:06:14,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:06:14,197 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:06:14,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:06:14,222 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:06:14,409 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:06:16,410 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:06:17,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:22,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:27,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:27,678 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:06:28,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:06:28,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:06:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:06:33,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:38,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:43,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:43,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:06:43,930 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:06:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:06:49,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:54,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:06:57,679 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:06:58,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:06:58,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:06:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:06:59,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:04,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:09,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:13,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:07:13,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:07:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:07:15,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:20,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:25,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:27,680 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:07:27,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:07:27,736 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:07:27,736 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:07:27,737 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:07:28,455 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:07:28,455 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:07:28,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:07:28,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:07:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:07:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:35,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:40,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:43,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:07:43,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:07:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:07:45,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:50,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:55,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:07:57,681 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:07:58,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:07:58,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:07:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:08:01,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:06,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:11,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:13,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:08:13,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:08:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:08:16,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:21,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:26,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:27,682 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:08:28,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:08:28,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:08:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:08:32,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:37,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:41,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:08:41,542 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:08:41,542 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:08:41,543 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:08:42,500 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:08:42,500 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:08:42,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:43,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:08:43,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:08:43,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:08:48,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:53,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:57,682 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:08:58,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:08:58,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:08:58,931 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:08:58,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:09:04,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:09,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:13,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:09:13,932 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:09:13,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:09:15,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:20,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:25,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:27,683 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:09:28,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:09:28,932 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:09:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:09:30,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:35,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:40,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:43,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:09:43,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:09:43,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:09:46,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:51,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:55,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:09:55,407 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:09:55,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:09:55,409 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:09:55,545 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:09:56,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:09:56,546 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:09:57,684 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:09:58,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:09:58,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:09:58,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:10:02,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:07,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:12,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:13,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:10:13,939 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:10:13,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:10:17,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:22,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:27,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:27,685 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:10:28,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:10:28,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:10:28,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:10:32,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:37,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:42,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:43,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:10:43,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:10:43,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:10:48,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:53,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:57,686 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:10:58,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:10:58,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:10:58,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:10:58,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:11:04,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:09,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:11:09,056 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:11:09,057 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:11:09,058 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:11:09,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:11:10,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:10,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:11:13,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:11:13,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:11:13,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:11:15,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:20,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:25,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:27,688 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:11:28,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:11:28,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:11:28,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:11:31,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:36,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:41,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:43,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:11:43,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:11:43,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:11:47,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:52,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:57,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:11:57,688 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:11:58,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:11:58,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:11:58,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:12:02,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:07,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:12,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:13,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:12:13,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:12:13,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:12:18,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:22,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:12:22,936 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:12:22,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:12:22,938 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:12:23,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:23,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:12:23,641 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:12:24,642 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:12:26,643 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:12:27,689 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:12:28,645 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:12:28,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:12:28,940 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:12:28,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:12:29,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:34,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:39,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:43,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:12:43,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:12:43,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:12:45,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:50,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:55,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:12:57,690 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:12:58,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:12:58,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:12:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:13:01,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:06,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:11,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:13,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:13:13,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:13:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:13:17,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:22,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:27,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:27,691 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:13:28,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:13:28,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:13:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:13:32,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:37,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:41,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:13:41,428 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:13:41,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:13:41,430 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:13:41,690 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:13:42,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:42,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:13:43,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:13:43,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:13:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:13:48,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:53,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:57,692 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:13:58,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:13:58,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:13:58,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:13:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:14:04,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:09,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:13,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:14:13,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:14:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:14:14,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:19,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:24,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:27,693 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:14:28,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:14:28,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:14:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:14:30,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:35,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:40,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:43,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:14:43,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:14:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:14:46,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:51,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:55,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:14:55,152 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:14:55,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:14:55,153 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:14:55,733 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:14:56,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:14:56,734 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:14:57,694 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:14:58,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:14:58,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:14:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:15:02,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:07,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:12,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:13,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:15:13,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:15:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:15:18,131 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:23,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:27,695 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:15:28,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:28,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:15:28,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:15:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:15:34,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:39,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:43,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:15:43,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:15:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:15:44,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:49,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:54,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:15:57,695 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:15:58,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:15:58,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:15:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:16:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:05,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:09,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:16:09,242 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:16:09,243 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:16:09,245 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:16:09,777 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:16:10,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:10,777 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:16:13,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:16:13,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:16:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:16:16,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:21,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:26,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:27,696 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:16:28,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:16:28,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:16:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:16:32,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:37,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:42,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:43,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:16:43,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:16:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:16:47,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:52,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:57,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:16:57,697 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:16:58,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:16:58,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:16:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:17:03,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:08,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:13,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:13,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:17:13,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:17:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:17:18,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:23,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:17:23,156 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:17:23,157 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:17:23,158 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:17:23,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:23,822 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:17:25,824 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:17:27,698 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:17:28,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:28,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:17:28,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:17:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:17:34,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:39,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:43,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:17:43,942 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:17:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:17:45,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:50,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:55,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:17:57,699 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:17:58,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:17:58,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:17:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:18:01,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:06,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:11,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:13,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:18:13,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:18:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:18:16,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:21,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:26,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:27,700 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:18:28,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:18:28,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:18:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:18:31,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:36,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:36,981 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:18:36,983 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:18:36,983 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:18:36,984 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:18:37,868 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:18:39,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:18:42,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:43,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:18:43,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:18:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:18:47,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:52,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:57,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:18:57,701 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:18:58,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:18:58,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:18:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:19:03,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:08,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:13,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:13,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:19:13,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:19:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:19:18,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:23,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:27,702 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:19:28,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:28,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:19:28,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:19:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:19:34,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:39,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:43,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:19:43,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:19:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:19:45,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:50,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:50,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:19:50,954 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:19:50,955 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:19:50,956 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:19:51,914 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:19:53,915 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:19:55,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:19:57,703 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:19:58,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:19:58,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:19:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:20:01,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:06,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:11,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:13,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:20:13,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:20:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:20:17,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:22,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:27,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:27,704 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:20:28,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:20:28,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:20:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:20:32,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:37,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:42,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:43,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:20:43,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:20:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:20:48,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:53,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:57,705 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:20:58,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:20:58,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:20:58,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:20:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:21:04,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:05,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:21:05,011 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:21:05,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:21:05,013 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:21:05,959 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:21:07,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:21:10,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:13,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:21:13,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:21:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:21:15,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:20,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:25,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:27,706 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:21:28,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:21:28,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:21:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:21:31,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:36,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:41,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:43,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:21:43,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:21:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:21:47,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:52,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:57,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:21:57,707 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:21:58,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:21:58,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:21:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:22:03,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:08,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:13,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:13,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:22:13,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:22:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:22:18,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:22:18,936 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:22:18,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:22:18,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:18,938 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:22:19,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:22:22,008 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:22:23,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:27,708 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:22:28,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:22:28,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:22:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:22:29,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:34,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:39,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:43,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:22:43,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:22:43,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:22:44,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:49,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:54,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:22:57,709 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:22:58,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:22:58,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:22:58,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:22:59,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:04,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:09,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:13,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:23:13,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:23:13,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:23:15,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:20,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:25,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:27,710 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:23:28,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:23:28,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:23:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:23:31,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:32,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:23:32,808 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:23:32,809 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:23:32,810 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:23:33,051 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:23:34,052 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:23:36,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:41,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:43,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:23:43,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:23:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:23:47,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:52,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:57,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:23:57,711 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:23:58,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:23:58,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:23:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:24:03,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:08,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:13,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:13,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:24:13,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:24:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:24:19,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:24,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:27,712 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:24:28,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:24:28,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:24:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:24:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:35,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:40,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:43,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:24:43,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:24:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:24:46,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:46,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:24:46,535 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:24:46,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:24:46,536 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:24:47,094 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:24:47,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:24:48,095 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:24:49,096 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:24:50,097 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:24:51,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:52,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:24:56,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:24:57,712 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:24:58,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:24:58,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:24:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:25:02,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:07,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:12,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:13,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:25:13,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:25:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:25:18,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:23,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:27,714 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:25:28,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:28,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:25:28,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:25:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:25:34,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:39,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:43,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:25:43,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:25:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:25:45,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:50,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:55,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:25:57,714 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:25:58,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:25:58,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:25:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:26:00,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:04,504 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:26:04,507 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:26:04,507 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:26:04,508 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:26:05,141 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:26:05,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:06,142 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:26:10,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:13,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:26:13,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:26:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:26:16,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:21,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:26,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:27,715 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:26:28,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:26:28,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:26:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:26:32,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:37,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:42,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:43,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:26:43,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:26:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:26:47,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:52,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:57,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:26:57,716 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:26:58,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:26:58,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:26:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:27:03,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:08,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:13,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:13,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:27:13,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:27:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:27:18,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:27:18,340 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:27:18,340 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:27:18,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:18,341 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:27:19,186 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:27:20,187 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:27:23,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:27,717 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:27:28,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:28,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:27:28,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:27:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:27:34,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:39,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:43,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:27:43,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:27:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:27:45,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:50,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:55,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:27:57,718 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:27:58,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:27:58,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:27:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:28:00,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:05,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:10,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:13,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:28:13,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:28:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:28:16,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:21,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:26,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:27,719 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:28:28,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:28:28,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:28:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:28:32,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:32,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:28:32,216 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:28:32,216 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:28:32,217 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:28:32,231 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:28:34,233 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:28:37,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:42,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:43,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:28:43,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:28:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:28:48,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:53,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:57,720 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:28:58,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:28:58,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:28:58,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:28:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:29:04,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:09,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:13,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:29:13,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:29:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:29:14,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:19,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:24,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:27,721 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:29:28,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:29:28,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:29:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:29:30,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:35,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:40,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:43,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:29:43,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:29:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:29:45,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:46,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:29:46,362 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:29:46,362 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:29:46,363 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:29:47,279 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:29:48,279 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:29:50,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:55,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:29:57,722 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:29:58,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:29:58,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:29:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:30:01,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:06,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:11,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:13,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:30:13,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:30:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:30:17,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:22,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:27,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:27,723 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:30:28,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:30:28,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:30:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:30:32,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:37,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:42,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:43,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:30:43,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:30:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:30:47,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:52,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:57,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:30:57,724 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:30:58,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:30:58,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:30:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:31:00,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:31:00,635 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:31:00,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:31:00,636 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:31:01,327 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:31:02,327 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:31:02,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:07,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:12,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:13,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:31:13,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:31:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:31:18,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:23,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:27,725 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:31:28,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:28,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:31:28,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:31:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:31:34,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:39,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:43,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:31:43,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:31:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:31:44,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:49,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:54,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:31:57,729 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:31:58,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:31:58,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:31:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:31:59,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:04,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:09,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:13,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:32:13,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:32:13,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:32:14,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:32:14,522 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:32:14,522 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:32:14,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:14,524 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:32:15,374 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:32:16,374 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:32:19,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:24,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:27,730 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:32:28,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:32:28,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:32:28,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:32:30,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:35,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:40,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:43,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:32:43,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:32:43,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:32:46,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:51,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:56,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:32:57,731 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:32:58,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:32:58,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:32:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:33:01,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:06,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:11,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:13,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:33:13,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:33:13,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:33:16,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:21,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:26,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:27,732 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:33:28,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:33:28,655 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:33:28,655 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:33:28,656 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:33:28,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:33:28,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:33:28,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:33:29,422 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:33:30,422 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:33:31,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:36,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:41,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:43,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:33:43,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:33:43,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:33:47,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:52,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:57,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:33:57,733 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:33:58,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:33:58,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:33:58,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:34:02,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:07,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:12,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:13,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:34:13,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:34:13,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:34:18,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:23,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:27,735 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:34:28,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:28,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:34:28,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:34:28,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:34:34,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:39,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:42,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:34:42,493 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:34:42,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:34:42,495 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:34:43,468 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:34:43,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:34:43,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:34:43,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:34:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:44,469 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:34:49,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:54,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:34:57,736 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:34:58,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:34:58,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:34:58,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:35:00,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:05,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:10,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:13,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:35:13,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:35:13,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:35:16,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:21,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:26,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:27,738 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:35:28,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:35:28,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:35:28,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:35:31,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:36,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:41,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:43,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:35:43,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:35:43,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:35:46,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:51,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:56,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:35:56,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:35:56,394 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:35:56,394 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:35:56,396 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:35:56,513 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:35:57,738 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:35:58,515 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:35:58,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:35:58,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:35:58,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:36:01,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:06,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:11,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:13,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:36:13,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:36:13,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:36:17,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:22,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:27,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:27,739 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:36:28,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:36:28,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:36:28,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:36:32,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:37,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:42,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:43,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:36:43,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:36:43,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:36:48,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:53,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:57,740 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:36:58,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:36:58,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:36:58,950 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:36:58,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:37:03,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:08,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:10,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:37:10,384 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:37:10,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:37:10,385 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:37:10,559 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:37:10,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:37:12,561 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:37:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:37:13,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:37:13,993 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:37:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:14,562 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:37:16,563 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:37:19,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:24,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:27,741 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:37:28,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:37:28,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:37:29,004 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:37:30,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:35,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:40,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:43,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:37:43,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:37:43,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:37:46,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:51,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:56,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:37:57,742 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:37:58,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:37:58,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:37:58,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:38:02,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:07,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:12,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:13,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:38:13,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:38:13,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:38:17,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:22,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:27,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:27,743 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:38:28,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:38:28,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:38:28,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:38:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:38:29,200 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:38:29,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:38:29,201 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:38:29,612 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:38:30,613 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:38:32,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:37,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:42,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:43,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:38:43,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:38:43,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:38:48,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:53,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:57,744 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:38:58,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:38:58,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:38:58,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:38:58,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:39:04,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:09,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:13,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:39:13,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:39:13,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:39:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:19,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:24,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:27,773 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:39:28,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:39:28,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:39:28,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:39:29,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:34,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:39,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:43,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:39:43,026 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:39:43,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:39:43,028 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:39:43,659 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:39:43,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:39:43,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:39:43,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:39:44,659 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:39:45,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:50,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:55,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:39:57,774 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:39:58,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:39:58,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:39:58,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:40:01,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:06,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:11,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:13,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:40:13,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:40:13,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:40:16,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:21,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:26,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:27,775 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:40:28,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:40:28,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:40:28,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:40:32,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:37,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:42,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:43,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:40:43,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:40:43,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:40:47,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:52,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:56,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:40:56,835 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:40:56,835 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:40:56,837 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:40:57,706 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:40:57,776 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:40:57,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:40:58,706 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:40:58,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:40:58,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:40:58,962 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:41:03,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:08,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:13,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:13,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:41:13,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:41:13,963 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:41:18,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:23,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:27,779 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:41:28,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:28,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:41:28,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:41:28,963 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:41:34,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:39,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:43,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:41:43,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:41:43,963 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:41:44,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:49,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:54,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:41:57,780 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:41:58,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:41:58,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:41:58,963 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:42:00,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:05,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:10,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:11,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:42:11,023 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:42:11,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:42:11,025 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:42:11,752 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:42:12,752 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:42:13,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:42:13,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:42:13,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:42:16,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:21,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:26,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:27,779 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:42:28,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:42:28,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:42:28,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:42:32,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:37,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:42,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:43,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:42:43,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:42:43,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:42:48,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:53,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:57,781 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:42:58,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:42:58,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:42:58,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:42:58,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:43:04,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:09,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:13,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:43:13,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:43:13,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:43:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:19,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:24,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:24,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:43:24,901 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:43:24,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:43:24,903 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:43:25,799 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:43:26,800 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:43:27,781 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:43:28,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:43:28,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:43:28,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:43:29,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:34,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:39,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:43,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:43:43,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:43:43,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:43:45,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:50,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:55,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:43:57,782 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:43:58,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:43:58,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:43:58,964 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:44:01,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:06,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:11,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:13,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:44:13,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:44:13,965 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:44:16,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:21,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:26,160 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:27,783 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:44:28,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:44:28,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:44:28,965 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:44:31,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:36,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:39,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:44:39,035 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:44:39,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:44:39,037 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:44:39,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:44:40,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:44:42,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:43,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:44:43,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:44:43,965 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:44:47,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:52,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:57,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:44:57,784 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:44:59,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:44:59,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:44:59,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:45:02,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:07,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:12,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:14,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:45:14,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:45:14,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:45:18,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:23,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:27,785 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:45:28,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:29,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:45:29,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:45:29,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:45:34,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:39,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:44,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:45:44,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:45:44,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:45:45,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:50,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:53,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:45:53,131 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:45:53,131 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:45:53,133 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:45:53,892 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:45:54,893 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:45:56,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:45:57,785 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:45:59,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:45:59,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:45:59,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:46:01,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:06,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:11,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:14,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:46:14,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:46:14,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:46:17,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:22,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:27,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:27,786 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:46:29,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:46:29,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:46:29,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:46:32,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:37,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:42,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:44,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:46:44,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:46:44,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:46:48,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:53,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:57,787 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:46:58,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:46:59,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:46:59,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:46:59,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:47:04,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:07,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:47:07,678 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:47:07,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:47:07,680 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:47:07,942 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:47:08,943 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:47:09,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:14,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:47:14,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:47:14,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:47:15,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:20,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:25,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:27,788 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:47:29,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:47:29,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:47:29,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:47:31,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:36,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:41,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:44,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:47:44,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:47:44,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:47:47,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:52,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:57,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:47:57,789 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:47:59,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:47:59,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:47:59,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:48:02,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:07,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:12,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:14,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:48:14,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:48:14,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:48:18,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:21,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:48:21,691 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:48:21,692 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:48:21,694 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:48:21,990 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:48:22,990 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:48:23,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:27,790 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:48:29,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:48:29,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:48:29,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:48:29,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:34,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:39,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:44,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:48:44,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:48:44,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:48:44,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:49,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:54,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:48:57,791 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:48:59,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:48:59,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:48:59,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:49:00,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:05,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:10,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:14,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:49:14,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:49:14,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:49:15,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:20,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:25,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:27,792 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:49:29,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:49:29,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:49:29,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:49:30,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:35,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:35,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:49:35,671 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:49:35,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:49:35,672 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:49:36,036 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:49:36,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:49:37,037 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:49:39,038 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:49:40,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:41,040 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:49:44,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:49:44,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:49:44,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:49:46,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:51,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:56,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:49:57,793 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:49:59,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:49:59,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:49:59,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:50:02,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:07,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:12,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:14,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:50:14,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:50:14,011 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:50:17,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:22,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:27,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:27,794 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:50:29,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:50:29,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:50:29,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:50:33,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:38,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:43,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:44,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:50:44,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:50:44,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:50:48,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:53,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:53,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:50:53,538 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:50:53,538 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:50:53,539 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:50:54,098 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:50:55,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:50:57,795 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:50:58,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:50:59,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:50:59,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:50:59,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:51:04,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:09,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:14,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:51:14,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:51:14,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:51:15,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:20,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:25,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:27,796 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:51:29,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:51:29,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:51:29,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:51:30,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:35,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:40,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:44,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:51:44,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:51:44,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:51:46,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:51,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:56,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:51:57,797 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:51:59,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:51:59,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:51:59,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:52:02,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:07,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:07,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:52:07,511 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:52:07,511 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:52:07,513 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:52:08,146 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:52:09,147 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:52:12,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:14,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:52:14,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:52:14,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:52:18,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:23,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:27,798 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:52:28,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:29,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:52:29,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:52:29,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:52:34,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:39,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:44,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:52:44,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:52:44,013 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:52:44,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:49,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:54,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:52:57,799 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:52:59,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:52:59,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:52:59,013 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:53:00,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:05,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:10,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:14,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:53:14,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:53:14,013 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:53:16,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:21,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:21,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:53:21,634 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:53:21,635 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:53:21,636 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:53:22,193 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:53:23,194 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:53:26,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:27,800 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:53:29,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:53:29,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:53:29,013 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:53:32,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:37,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:42,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:44,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:53:44,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:53:44,013 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:53:48,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:53,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:57,801 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:53:58,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:53:59,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:53:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:53:59,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:54:04,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:09,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:14,012 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:54:14,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:54:14,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:54:14,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:19,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:24,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:27,802 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:54:29,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:54:29,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:54:29,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:54:30,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:35,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:35,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:54:35,581 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:54:35,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:54:35,583 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:54:36,240 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:54:37,241 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:54:40,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:44,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:54:44,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:54:44,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:54:46,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:51,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:56,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:54:57,803 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:54:59,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:54:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:54:59,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:55:01,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:06,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:11,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:14,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:55:14,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:55:14,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:55:16,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:21,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:26,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:27,804 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:55:29,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:55:29,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:55:29,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:55:31,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:36,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:41,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:44,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:55:44,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:55:44,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:55:47,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:49,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:55:49,993 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:55:49,994 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:55:49,996 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:55:50,291 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:55:52,292 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:55:52,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:57,805 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:55:58,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:55:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:55:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:55:59,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:56:04,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:09,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:14,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:56:14,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:56:14,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:56:14,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:19,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:24,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:27,806 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:56:29,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:56:29,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:56:29,014 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:56:29,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:34,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:39,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:44,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:56:44,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:56:44,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:56:45,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:50,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:55,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:56:57,808 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:56:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:56:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:56:59,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:57:00,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:03,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:57:03,869 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:57:03,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:57:03,872 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:57:04,336 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:57:05,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:06,337 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:57:10,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:14,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:57:14,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:57:14,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:57:16,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:21,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:26,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:27,808 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:57:29,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:57:29,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:57:29,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:57:32,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:37,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:42,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:44,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:57:44,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:57:44,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:57:47,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:52,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:57,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:57:57,810 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:57:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:57:59,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:57:59,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:58:03,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:08,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:13,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:14,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:58:14,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:58:14,015 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:58:17,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:58:17,632 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:58:17,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:58:17,633 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:58:18,381 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:58:18,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:20,382 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:58:23,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:27,810 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:58:29,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:58:29,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:58:29,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:58:29,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:34,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:39,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:44,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:58:44,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:58:44,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:58:45,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:50,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:55,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:58:57,811 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:58:59,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:58:59,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:58:59,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:59:00,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:05,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:10,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:14,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:59:14,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:59:14,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:59:16,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:21,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:26,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:27,812 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:59:29,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:59:29,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:59:29,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:59:31,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 19:59:31,763 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 19:59:31,763 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 19:59:31,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:31,765 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 19:59:32,426 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 19:59:34,427 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 19:59:36,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:41,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:44,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:59:44,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:59:44,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 19:59:47,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:52,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:57,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 19:59:57,813 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 19:59:59,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 19:59:59,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 19:59:59,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:00:03,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:08,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:13,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:14,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:00:14,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:00:14,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:00:19,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:24,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:27,814 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:00:29,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:00:29,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:00:29,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:00:30,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:35,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:40,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:44,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:00:44,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:00:44,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:00:45,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:00:45,562 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:00:45,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:00:45,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:45,565 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:00:46,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:00:48,474 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:00:50,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:55,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:00:57,814 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:00:59,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:00:59,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:00:59,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:01:01,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:06,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:11,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:14,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:01:14,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:01:14,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:01:17,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:22,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:27,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:27,816 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:01:29,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:01:29,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:01:29,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:01:33,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:38,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:43,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:44,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:01:44,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:01:44,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:01:48,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:53,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:57,816 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:01:58,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:01:59,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:01:59,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:01:59,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:01:59,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:01:59,525 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:01:59,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:01:59,527 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:02:00,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:02:00,521 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:02:02,523 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:02:03,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:04,524 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:02:06,525 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:02:08,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:14,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:02:14,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:02:14,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:02:14,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:19,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:24,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:27,817 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:02:29,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:02:29,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:02:29,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:02:30,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:35,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:40,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:44,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:02:44,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:02:44,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:02:45,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:50,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:55,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:02:57,819 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:02:59,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:02:59,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:02:59,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:03:01,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:06,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:11,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:14,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:03:14,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:03:14,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:03:17,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:17,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:03:17,718 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:03:17,718 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:03:17,720 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:03:18,573 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:03:20,574 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:03:22,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:27,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:27,819 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:03:29,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:03:29,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:03:29,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:03:33,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:38,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:43,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:44,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:03:44,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:03:44,018 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:03:49,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:54,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:03:57,820 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:03:59,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:03:59,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:03:59,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:04:00,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:05,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:10,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:14,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:04:14,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:04:14,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:04:16,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:21,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:26,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:27,823 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:04:29,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:04:29,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:04:29,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:04:31,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:32,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:04:32,039 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:04:32,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:04:32,041 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:04:32,622 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:04:34,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:04:37,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:42,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:44,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:04:44,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:04:44,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:04:47,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:52,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:57,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:04:57,824 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:04:59,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:04:59,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:04:59,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:05:02,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:07,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:12,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:14,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:05:14,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:05:14,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:05:18,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:23,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:27,825 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:05:28,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:29,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:05:29,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:05:29,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:05:34,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:39,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:44,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:05:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:05:44,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:05:45,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:46,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:05:46,102 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:05:46,102 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:05:46,104 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:05:46,669 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:05:48,670 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:05:51,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:56,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:05:57,826 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:05:59,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:05:59,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:05:59,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:06:01,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:06,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:11,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:14,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:06:14,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:06:14,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:06:16,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:21,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:26,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:27,827 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:06:29,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:06:29,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:06:29,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:06:31,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:36,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:41,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:44,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:06:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:06:44,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:06:47,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:52,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:57,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:06:57,828 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:06:59,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:06:59,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:06:59,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:07:00,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:07:00,029 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:07:00,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:07:00,030 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:07:00,714 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:07:02,715 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:07:03,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:08,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:13,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:14,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:07:14,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:07:14,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:07:18,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:23,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:27,829 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:07:28,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:29,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:07:29,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:07:29,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:07:34,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:39,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:44,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:07:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:07:44,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:07:44,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:49,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:54,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:07:57,830 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:07:59,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:07:59,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:07:59,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:07:59,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:04,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:09,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:13,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:08:13,941 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:08:13,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:08:13,943 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:08:14,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:08:14,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:08:14,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:08:14,758 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:08:15,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:16,759 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:08:20,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:25,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:27,831 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:08:29,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:08:29,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:08:29,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:08:31,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:36,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:41,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:44,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:08:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:08:44,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:08:46,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:51,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:56,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:08:57,831 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:08:59,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:08:59,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:08:59,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:09:02,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:07,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:12,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:14,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:09:14,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:09:14,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:09:17,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:22,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:27,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:27,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:09:27,686 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:09:27,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:09:27,688 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:09:27,802 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:09:27,832 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:09:28,803 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:09:29,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:09:29,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:09:29,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:09:32,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:37,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:42,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:44,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:09:44,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:09:44,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:09:48,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:53,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:57,833 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:09:58,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:09:59,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:09:59,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:09:59,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:10:04,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:09,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:14,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:10:14,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:10:14,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:10:14,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:19,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:24,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:27,834 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:10:29,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:10:29,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:10:29,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:10:30,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:35,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:40,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:41,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:10:41,673 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:10:41,674 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:10:41,675 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:10:41,846 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:10:42,847 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:10:44,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:10:44,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:10:44,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:10:46,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:51,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:56,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:10:57,835 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:10:59,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:10:59,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:10:59,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:11:01,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:06,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:11,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:14,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:11:14,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:11:14,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:11:16,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:21,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:26,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:27,836 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:11:29,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:11:29,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:11:29,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:11:32,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:37,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:42,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:11:44,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:11:44,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:11:47,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:52,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:56,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:11:56,108 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:11:56,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:11:56,110 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:11:56,894 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:11:57,838 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:11:57,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:11:58,895 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:11:59,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:11:59,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:11:59,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:12:03,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:08,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:13,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:14,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:12:14,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:12:14,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:12:18,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:23,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:27,838 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:12:28,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:29,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:12:29,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:12:29,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:12:34,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:39,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:12:44,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:12:44,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:12:44,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:49,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:54,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:12:57,840 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:12:59,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:12:59,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:12:59,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:12:59,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:04,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:09,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:10,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:13:10,015 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:13:10,016 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:13:10,017 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:13:10,939 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:13:10,940 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:13:14,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:13:14,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:13:14,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:13:15,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:20,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:25,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:27,840 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:13:29,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:13:29,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:13:29,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:13:31,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:36,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:41,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:44,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:13:44,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:13:44,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:13:47,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:52,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:57,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:13:57,841 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:13:59,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:13:59,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:13:59,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:14:03,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:08,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:13,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:14,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:14:14,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:14:14,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:14:18,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:23,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:24,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:14:24,083 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:14:24,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:14:24,085 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:14:24,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:14:24,984 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:14:24,984 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:14:26,985 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:14:27,842 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:14:28,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:28,987 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:14:29,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:14:29,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:14:29,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:14:34,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:39,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:44,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:14:44,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:14:44,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:14:44,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:49,293 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:54,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:14:57,843 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:14:59,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:14:59,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:14:59,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:15:00,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:05,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:10,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:14,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:15:14,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:15:14,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:15:15,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:20,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:25,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:27,844 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:15:29,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:15:29,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:15:29,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:15:30,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:35,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:40,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:41,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:15:41,943 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:15:41,943 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:15:41,945 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:15:42,030 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:15:43,030 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:15:44,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:15:44,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:15:44,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:15:46,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:51,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:56,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:15:57,845 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:15:59,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:15:59,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:15:59,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:16:02,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:07,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:12,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:14,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:16:14,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:16:14,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:16:17,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:22,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:27,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:27,846 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:16:29,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:16:29,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:16:29,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:16:33,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:38,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:43,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:44,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:16:44,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:16:44,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:16:49,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:54,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:16:55,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:16:55,824 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:16:55,824 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:16:55,826 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:16:56,074 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:16:57,074 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:16:57,847 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:16:59,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:16:59,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:16:59,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:16:59,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:04,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:09,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:14,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:17:14,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:17:14,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:17:14,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:19,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:24,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:27,848 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:17:29,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:17:29,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:17:29,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:17:30,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:35,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:40,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:44,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:17:44,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:17:44,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:17:46,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:51,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:56,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:17:57,849 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:17:59,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:17:59,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:17:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:18:01,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:06,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:09,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:18:09,709 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:18:09,710 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:18:09,711 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:18:10,121 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:18:11,122 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:18:11,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:14,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:18:14,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:18:14,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:18:17,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:22,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:27,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:27,850 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:18:29,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:18:29,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:18:29,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:18:32,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:37,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:42,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:44,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:18:44,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:18:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:18:48,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:53,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:57,851 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:18:58,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:18:59,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:18:59,083 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:18:59,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:19:04,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:09,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:14,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:19:14,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:19:14,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:19:14,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:19,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:23,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:19:23,590 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:19:23,590 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:19:23,592 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:19:24,168 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:19:24,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:25,168 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:19:27,853 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:19:29,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:19:29,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:19:29,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:19:30,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:35,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:40,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:44,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:19:44,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:19:44,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:19:46,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:51,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:56,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:19:57,853 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:19:59,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:19:59,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:19:59,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:20:01,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:06,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:11,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:14,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:20:14,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:20:14,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:20:16,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:21,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:26,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:27,854 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:20:29,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:20:29,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:20:29,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:20:32,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:37,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:37,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:20:37,867 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:20:37,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:20:37,869 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:20:38,214 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:20:39,215 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:20:42,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:44,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:20:44,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:20:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:20:48,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:53,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:57,855 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:20:58,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:20:59,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:20:59,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:20:59,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:21:04,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:09,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:14,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:21:14,084 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:21:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:21:14,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:19,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:24,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:27,856 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:21:29,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:21:29,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:21:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:21:30,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:35,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:40,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:44,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:21:44,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:21:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:21:46,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:51,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:51,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:21:51,768 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:21:51,768 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:21:51,769 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:21:52,258 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:21:53,258 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:21:56,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:21:57,857 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:21:59,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:21:59,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:21:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:22:02,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:07,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:12,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:14,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:22:14,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:22:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:22:17,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:22,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:27,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:27,858 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:22:29,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:22:29,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:22:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:22:33,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:38,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:43,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:44,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:22:44,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:22:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:22:49,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:54,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:22:57,858 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:22:59,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:22:59,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:22:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:22:59,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:04,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:05,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:23:05,558 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:23:05,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:23:05,560 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:23:06,301 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:23:07,302 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:23:09,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:14,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:23:14,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:23:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:23:15,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:20,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:25,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:27,859 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:23:29,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:23:29,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:23:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:23:30,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:35,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:40,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:44,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:23:44,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:23:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:23:46,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:51,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:56,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:23:57,861 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:23:59,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:23:59,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:23:59,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:24:01,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:06,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:11,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:14,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:24:14,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:24:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:24:17,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:19,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:24:19,500 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:24:19,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:24:19,502 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:24:20,344 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:24:21,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:24:22,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:27,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:27,861 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:24:29,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:24:29,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:24:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:24:33,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:38,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:43,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:44,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:24:44,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:24:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:24:48,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:53,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:57,862 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:24:58,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:24:59,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:24:59,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:24:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:25:04,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:09,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:14,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:25:14,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:25:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:25:14,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:19,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:24,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:27,863 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:25:29,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:25:29,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:25:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:25:30,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:33,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:25:33,500 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:25:33,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:25:33,501 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:25:34,388 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:25:35,389 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:25:35,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:40,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:44,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:25:44,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:25:44,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:25:46,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:51,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:56,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:25:57,864 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:25:59,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:25:59,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:25:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:26:01,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:06,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:11,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:14,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:26:14,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:26:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:26:17,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:22,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:27,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:27,865 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:26:29,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:26:29,086 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:26:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:26:33,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:38,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:43,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:44,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:26:44,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:26:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:26:47,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:26:47,186 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:26:47,186 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:26:47,187 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:26:47,432 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:26:47,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:26:48,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:49,433 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:26:51,434 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:26:53,435 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:26:54,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:26:57,866 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:26:59,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:26:59,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:26:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:26:59,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:04,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:09,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:14,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:27:14,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:27:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:27:14,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:19,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:24,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:27,867 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:27:29,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:27:29,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:27:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:27:29,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:34,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:39,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:44,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:27:44,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:27:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:27:45,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:50,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:55,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:27:57,868 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:27:59,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:27:59,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:27:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:28:01,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:05,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:28:05,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:28:05,269 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:28:05,270 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:28:05,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:28:07,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:07,482 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:28:12,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:14,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:28:14,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:28:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:28:18,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:23,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:27,869 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:28:28,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:29,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:28:29,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:28:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:28:34,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:39,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:44,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:28:44,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:28:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:28:44,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:49,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:54,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:28:57,870 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:28:59,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:28:59,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:28:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:29:00,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:05,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:10,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:14,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:29:14,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:29:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:29:15,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:19,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:29:19,497 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:29:19,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:29:19,499 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:29:19,530 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:29:20,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:21,532 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:29:25,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:27,871 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:29:29,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:29:29,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:29:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:29:31,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:36,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:41,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:44,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:29:44,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:29:44,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:29:47,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:52,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:57,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:29:57,872 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:29:59,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:29:59,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:29:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:30:02,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:07,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:12,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:14,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:30:14,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:30:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:30:17,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:22,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:27,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:27,873 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:30:29,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:30:29,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:30:29,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:30:33,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:33,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:30:33,296 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:30:33,296 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:30:33,297 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:30:33,577 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:30:36,579 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:30:38,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:43,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:44,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:30:44,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:30:44,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:30:49,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:54,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:30:57,874 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:30:59,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:30:59,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:30:59,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:30:59,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:04,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:09,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:14,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:31:14,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:31:14,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:31:14,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:19,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:24,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:27,875 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:31:29,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:31:29,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:31:29,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:31:30,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:35,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:40,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:44,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:31:44,088 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:31:44,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:31:46,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:47,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:31:47,174 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:31:47,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:31:47,175 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:31:47,623 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:31:48,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:31:52,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:57,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:31:57,875 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:31:59,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:31:59,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:31:59,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:32:02,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:07,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:12,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:14,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:32:14,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:32:14,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:32:18,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:23,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:27,876 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:32:28,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:29,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:32:29,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:32:29,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:32:34,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:39,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:44,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:32:44,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:32:44,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:32:44,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:49,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:54,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:32:57,877 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:32:59,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:32:59,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:32:59,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:32:59,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:00,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:33:00,980 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:33:00,980 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:33:00,981 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:33:01,670 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:33:02,671 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:33:05,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:10,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:14,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:33:14,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:33:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:33:15,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:20,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:25,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:27,878 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:33:29,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:33:29,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:33:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:33:31,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:36,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:41,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:44,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:33:44,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:33:44,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:33:46,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:51,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:56,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:33:57,879 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:33:59,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:33:59,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:33:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:34:01,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:06,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:11,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:14,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:34:14,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:34:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:34:14,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:34:14,622 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:34:14,622 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:34:14,623 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:34:14,714 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:34:16,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:16,715 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:34:21,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:26,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:27,880 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:34:29,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:34:29,136 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:34:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:34:32,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:37,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:42,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:44,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:34:44,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:34:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:34:47,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:52,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:57,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:34:57,882 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:34:59,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:34:59,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:34:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:35:03,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:08,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:13,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:14,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:35:14,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:35:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:35:19,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:24,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:27,882 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:35:28,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:35:28,600 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:35:28,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:35:28,602 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:35:28,760 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:35:29,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:35:29,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:35:29,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:35:30,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:30,761 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:35:35,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:40,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:44,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:35:44,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:35:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:35:45,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:50,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:55,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:35:57,883 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:35:59,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:35:59,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:35:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:36:01,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:06,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:11,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:14,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:36:14,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:36:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:36:16,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:21,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:26,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:27,885 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:36:29,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:36:29,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:36:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:36:31,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:36,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:41,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:42,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:36:42,885 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:36:42,885 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:36:42,886 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:36:43,807 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:36:44,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:36:44,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:36:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:36:44,808 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:36:47,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:52,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:57,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:36:57,885 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:36:59,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:36:59,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:36:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:37:03,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:08,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:13,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:14,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:37:14,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:37:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:37:18,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:23,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:27,886 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:37:28,504 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:29,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:37:29,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:37:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:37:34,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:39,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:44,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:37:44,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:37:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:37:45,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:50,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:55,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:37:56,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:37:56,956 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:37:56,956 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:37:56,957 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:37:57,853 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:37:57,887 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:37:58,853 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:37:59,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:37:59,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:37:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:38:00,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:05,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:10,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:14,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:38:14,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:38:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:38:16,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:21,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:26,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:27,888 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:38:29,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:38:29,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:38:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:38:32,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:37,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:42,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:44,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:38:44,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:38:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:38:48,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:53,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:57,889 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:38:58,506 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:38:59,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:38:59,138 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:38:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:39:04,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:09,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:10,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:39:10,801 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:39:10,801 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:39:10,802 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:39:10,894 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:39:11,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:39:12,895 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:39:14,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:39:14,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:39:14,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:39:14,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:14,897 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:39:16,898 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:39:19,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:24,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:27,890 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:39:29,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:39:29,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:39:29,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:39:30,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:35,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:40,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:44,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:39:44,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:39:44,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:39:46,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:51,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:56,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:39:57,891 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:39:59,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:39:59,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:39:59,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:40:01,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:06,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:11,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:14,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:40:14,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:40:14,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:40:17,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:22,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:27,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:27,891 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:40:28,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:40:28,873 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:40:28,874 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:40:28,875 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:40:28,941 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:40:29,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:40:29,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:40:29,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:40:30,942 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:40:32,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:37,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:42,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:44,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:40:44,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:40:44,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:40:47,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:52,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:57,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:40:57,892 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:40:59,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:40:59,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:40:59,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:41:03,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:08,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:13,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:14,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:41:14,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:41:14,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:41:18,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:23,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:27,893 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:41:28,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:29,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:41:29,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:41:29,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:41:34,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:39,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:42,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:41:42,773 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:41:42,773 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:41:42,775 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:41:42,985 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:41:44,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:41:44,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:41:44,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:41:44,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:44,986 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:41:49,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:54,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:41:57,895 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:41:59,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:41:59,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:41:59,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:41:59,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:04,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:09,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:14,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:42:14,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:42:14,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:42:15,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:20,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:25,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:27,895 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:42:29,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:42:29,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:42:29,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:42:30,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:35,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:40,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:44,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:42:44,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:42:44,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:42:45,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:50,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:55,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:42:56,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:42:56,817 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:42:56,818 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:42:56,820 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:42:57,032 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:42:57,896 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:42:59,033 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:42:59,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:42:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:42:59,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:43:00,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:05,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:10,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:14,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:43:14,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:43:14,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:43:16,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:21,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:26,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:27,897 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:43:29,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:43:29,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:43:29,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:43:31,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:36,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:41,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:44,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:43:44,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:43:44,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:43:47,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:52,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:57,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:43:57,898 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:43:59,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:43:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:43:59,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:44:02,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:07,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:10,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:44:10,856 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:44:10,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:44:10,858 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:44:11,075 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:44:12,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:13,076 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:44:14,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:44:14,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:44:14,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:44:18,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:23,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:27,899 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:44:28,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:29,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:44:29,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:44:29,146 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:44:34,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:39,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:44,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:44:44,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:44:44,147 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:44:44,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:49,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:54,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:44:57,900 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:44:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:44:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:44:59,147 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:45:00,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:05,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:10,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:14,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:45:14,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:45:14,147 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:45:15,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:20,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:25,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:45:25,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:45:25,268 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:45:25,270 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:45:26,128 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:45:26,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:27,129 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:45:27,901 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:45:29,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:45:29,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:45:29,147 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:45:31,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:36,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:41,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:44,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:45:44,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:45:44,147 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:45:46,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:51,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:56,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:45:57,901 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:45:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:45:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:45:59,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:46:01,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:06,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:11,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:14,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:46:14,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:46:14,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:46:16,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:21,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:26,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:27,903 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:46:29,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:46:29,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:46:29,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:46:31,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:36,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:39,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:46:39,199 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:46:39,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:46:39,201 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:46:40,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:46:41,176 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:46:42,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:44,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:46:44,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:46:44,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:46:47,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:52,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:57,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:46:57,903 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:46:59,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:46:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:46:59,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:47:03,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:08,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:13,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:14,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:47:14,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:47:14,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:47:19,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:24,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:27,904 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:47:29,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:47:29,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:47:29,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:47:30,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:35,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:40,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:44,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:47:44,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:47:44,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:47:46,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:51,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:53,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:47:53,125 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:47:53,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:47:53,127 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:47:53,220 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:47:55,221 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:47:57,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:47:57,905 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:47:59,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:47:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:47:59,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:48:02,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:07,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:12,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:14,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:48:14,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:48:14,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:48:18,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:23,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:27,906 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:48:28,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:29,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:48:29,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:48:29,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:48:34,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:39,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:44,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:48:44,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:48:44,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:48:44,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:49,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:54,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:48:57,907 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:48:59,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:48:59,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:48:59,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:49:00,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:05,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:06,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:49:06,853 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:49:06,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:49:06,855 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:49:07,279 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:49:09,280 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:49:10,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:14,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:49:14,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:49:14,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:49:16,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:21,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:26,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:27,908 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:49:29,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:49:29,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:49:29,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:49:31,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:36,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:41,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:44,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:49:44,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:49:44,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:49:47,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:52,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:57,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:49:57,909 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:49:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:49:59,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:49:59,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:50:03,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:08,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:13,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:14,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:50:14,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:50:14,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:50:18,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:20,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:50:20,786 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:50:20,786 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:50:20,787 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:50:21,324 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:50:23,326 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:50:23,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:27,910 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:50:29,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:50:29,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:50:29,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:50:29,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:34,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:39,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:44,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:50:44,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:50:44,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:50:44,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:49,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:54,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:50:57,911 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:50:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:50:59,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:50:59,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:50:59,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:04,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:09,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:14,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:51:14,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:51:14,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:51:15,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:20,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:25,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:27,912 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:51:29,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:51:29,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:51:29,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:51:30,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:34,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:51:34,523 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:51:34,523 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:51:34,525 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:51:35,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:51:35,369 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:51:35,369 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:51:36,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:37,370 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:51:39,371 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:51:41,373 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:51:41,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:44,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:51:44,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:51:44,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:51:47,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:52,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:57,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:51:57,912 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:51:59,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:51:59,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:51:59,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:52:03,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:08,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:13,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:14,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:52:14,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:52:14,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:52:19,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:24,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:27,913 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:52:29,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:52:29,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:52:29,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:52:29,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:34,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:39,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:44,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:52:44,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:52:44,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:52:45,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:50,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:53,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:52:53,306 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:52:53,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:52:53,307 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:52:53,419 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:52:55,420 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:52:56,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:52:57,914 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:52:59,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:52:59,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:52:59,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:53:01,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:06,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:11,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:14,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:53:14,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:53:14,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:53:17,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:22,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:27,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:27,915 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:53:29,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:53:29,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:53:29,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:53:33,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:38,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:43,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:44,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:53:44,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:53:44,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:53:49,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:54,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:53:57,916 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:53:59,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:53:59,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:53:59,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:53:59,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:04,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:07,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:54:07,631 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:54:07,632 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:54:07,633 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:54:08,467 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:54:09,468 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:54:09,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:14,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:54:14,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:54:14,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:54:15,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:20,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:25,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:27,917 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:54:29,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:54:29,161 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:54:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:54:30,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:35,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:40,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:44,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:54:44,161 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:54:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:54:46,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:51,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:56,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:54:57,918 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:54:59,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:54:59,161 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:54:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:55:02,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:07,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:12,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:14,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:55:14,161 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:55:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:55:17,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:21,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:55:21,479 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:55:21,479 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:55:21,480 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:55:21,513 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:55:22,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:23,514 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:55:27,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:27,919 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:55:29,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:55:29,161 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:55:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:55:33,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:38,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:43,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:44,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:55:44,161 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:55:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:55:49,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:54,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:55:57,920 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:55:59,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:55:59,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:55:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:55:59,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:04,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:09,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:14,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:56:14,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:56:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:56:14,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:19,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:24,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:27,921 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:56:29,161 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:56:29,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:56:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:56:29,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:34,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:35,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:56:35,261 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:56:35,261 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:56:35,262 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:56:35,557 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:56:37,558 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:56:40,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:44,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:56:44,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:56:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:56:45,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:50,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:55,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:56:57,922 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:56:59,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:56:59,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:56:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:57:00,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:05,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:10,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:14,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:57:14,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:57:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:57:16,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:21,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:26,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:27,924 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:57:29,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:57:29,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:57:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:57:31,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:36,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:41,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:44,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:57:44,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:57:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:57:47,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:49,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:57:49,236 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:57:49,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:57:49,238 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:57:49,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:57:51,603 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:57:53,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:57,924 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:57:58,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:57:59,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:57:59,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:57:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:58:04,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:09,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:14,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:58:14,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:58:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:58:15,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:20,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:25,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:27,925 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:58:29,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:58:29,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:58:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:58:30,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:35,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:40,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:44,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:58:44,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:58:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:58:46,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:51,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:56,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:58:57,925 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:58:59,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:58:59,162 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:58:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:59:02,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:03,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 20:59:03,042 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 20:59:03,042 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 20:59:03,044 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 20:59:03,646 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 20:59:05,648 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 20:59:08,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:13,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:14,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:59:14,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:59:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:59:18,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:23,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:27,926 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:59:28,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:29,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:59:29,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:59:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:59:34,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:39,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:44,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:59:44,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:59:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 20:59:44,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:49,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:54,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 20:59:57,927 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 20:59:59,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 20:59:59,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 20:59:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:00:00,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:05,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:10,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:14,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:00:14,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:00:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:00:15,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:16,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:00:16,866 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:00:16,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:00:16,867 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:00:17,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:00:17,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:00:20,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:25,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:27,928 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:00:29,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:00:29,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:00:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:00:31,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:36,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:41,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:44,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:00:44,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:00:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:00:47,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:52,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:57,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:00:57,929 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:00:59,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:00:59,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:00:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:01:02,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:07,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:14,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:01:14,163 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:01:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:01:18,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:23,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:27,930 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:01:28,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:29,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:01:29,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:01:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:01:31,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:01:31,019 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:01:31,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:01:31,021 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:01:31,737 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:01:31,737 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:01:34,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:39,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:44,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:01:44,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:01:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:01:44,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:49,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:54,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:01:57,931 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:01:59,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:01:59,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:01:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:02:00,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:05,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:10,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:14,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:02:14,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:02:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:02:15,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:20,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:25,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:27,932 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:02:29,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:02:29,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:02:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:02:31,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:36,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:41,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:44,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:02:44,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:02:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:02:45,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:02:45,142 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:02:45,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:02:45,144 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:02:45,783 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:02:45,783 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:02:47,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:52,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:57,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:02:57,932 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:02:59,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:02:59,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:02:59,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:03:02,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:07,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:12,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:14,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:03:14,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:03:14,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:03:18,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:23,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:27,933 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:03:28,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:29,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:03:29,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:03:29,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:03:34,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:39,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:44,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:03:44,164 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:03:44,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:03:44,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:49,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:54,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:57,934 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:03:59,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:03:59,127 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:03:59,128 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:03:59,129 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:03:59,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:03:59,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:03:59,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:03:59,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:03:59,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:03:59,829 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:03:59,829 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:04:01,830 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:04:02,831 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:04:03,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:04:05,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:05,834 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:04:10,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:14,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:04:14,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:04:14,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:04:16,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:21,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:26,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:27,935 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:04:29,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:04:29,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:04:29,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:04:32,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:37,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:42,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:44,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:04:44,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:04:44,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:04:47,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:52,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:57,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:04:57,937 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:04:59,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:04:59,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:04:59,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:05:02,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:07,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:12,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:14,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:05:14,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:05:14,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:05:17,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:05:17,196 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:05:17,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:05:17,198 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:05:17,897 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:05:17,898 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:05:18,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:23,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:27,937 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:05:28,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:29,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:05:29,165 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:05:29,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:05:34,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:39,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:44,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:05:44,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:05:44,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:05:45,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:50,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:55,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:05:57,939 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:05:59,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:05:59,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:05:59,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:06:00,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:05,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:10,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:14,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:06:14,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:06:14,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:06:15,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:20,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:25,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:27,939 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:06:29,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:06:29,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:06:29,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:06:31,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:06:31,086 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:06:31,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:06:31,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:31,088 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:06:31,950 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:06:31,950 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:06:36,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:41,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:44,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:06:44,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:06:44,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:06:46,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:51,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:56,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:06:57,940 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:06:59,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:06:59,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:06:59,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:07:01,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:06,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:11,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:14,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:07:14,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:07:14,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:07:17,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:22,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:27,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:27,941 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:07:29,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:07:29,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:07:29,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:07:33,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:38,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:43,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:44,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:07:44,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:07:44,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:07:44,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:07:44,956 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:07:44,956 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:07:44,958 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:07:44,993 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:07:45,993 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:07:48,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:53,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:07:57,942 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:07:59,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:07:59,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:07:59,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:07:59,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:04,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:09,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:14,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:08:14,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:08:14,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:08:15,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:20,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:25,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:27,943 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:08:29,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:08:29,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:08:29,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:08:30,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:35,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:40,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:44,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:08:44,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:08:44,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:08:46,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:51,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:56,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:08:57,944 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:08:59,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:08:59,106 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:08:59,106 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:08:59,107 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:08:59,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:08:59,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:08:59,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:09:00,038 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:09:00,038 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:09:01,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:06,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:11,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:14,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:09:14,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:09:14,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:09:16,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:21,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:26,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:27,945 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:09:29,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:09:29,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:09:29,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:09:31,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:36,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:41,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:44,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:09:44,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:09:44,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:09:47,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:52,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:57,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:09:57,946 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:09:59,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:09:59,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:09:59,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:10:02,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:07,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:12,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:13,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:10:13,691 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:10:13,691 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:10:13,693 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:10:14,086 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:10:14,086 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:10:14,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:10:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:10:14,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:10:18,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:23,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:27,947 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:10:28,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:29,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:10:29,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:10:29,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:10:34,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:39,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:44,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:10:44,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:10:44,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:10:45,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:50,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:55,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:10:57,948 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:10:59,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:10:59,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:10:59,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:11:00,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:05,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:10,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:11:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:11:14,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:11:15,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:20,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:25,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:27,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:11:27,808 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:11:27,808 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:11:27,810 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:11:27,948 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:11:28,131 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:11:29,132 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:11:29,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:11:29,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:11:29,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:11:31,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:36,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:41,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:44,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:11:44,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:11:44,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:11:46,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:51,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:56,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:11:57,950 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:11:59,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:11:59,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:11:59,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:12:02,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:07,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:12,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:12:14,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:12:14,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:12:18,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:23,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:27,951 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:12:28,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:29,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:12:29,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:12:29,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:12:34,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:39,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:41,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:12:41,706 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:12:41,706 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:12:41,708 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:12:42,178 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:12:43,179 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:12:44,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:12:44,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:12:44,170 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:12:45,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:50,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:55,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:12:57,951 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:12:59,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:12:59,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:12:59,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:13:00,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:05,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:10,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:13:14,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:13:14,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:13:15,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:20,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:25,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:27,952 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:13:29,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:13:29,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:13:29,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:13:31,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:36,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:41,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:44,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:13:44,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:13:44,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:13:46,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:51,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:55,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:13:55,762 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:13:55,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:13:55,764 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:13:56,226 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:13:56,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:13:57,227 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:13:57,953 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:13:59,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:13:59,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:13:59,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:14:02,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:07,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:12,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:14,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:14:14,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:14:14,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:14:18,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:23,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:27,954 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:14:28,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:29,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:14:29,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:14:29,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:14:34,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:39,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:44,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:14:44,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:14:44,171 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:14:45,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:50,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:55,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:14:57,955 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:14:59,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:14:59,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:14:59,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:15:01,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:06,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:09,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:15:09,870 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:15:09,871 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:15:09,872 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:15:10,273 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:15:11,273 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:15:11,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:14,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:15:14,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:15:14,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:15:17,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:22,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:27,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:27,956 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:15:29,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:15:29,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:15:29,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:15:32,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:37,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:42,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:44,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:15:44,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:15:44,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:15:48,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:53,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:57,957 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:15:58,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:15:59,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:15:59,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:15:59,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:16:04,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:09,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:14,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:16:14,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:16:14,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:16:14,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:19,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:24,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:16:24,035 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:16:24,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:16:24,037 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:16:24,320 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:16:24,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:16:24,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:25,320 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:16:27,322 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:16:27,958 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:16:29,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:16:29,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:16:29,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:16:29,323 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:16:30,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:35,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:40,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:44,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:16:44,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:16:44,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:16:45,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:50,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:55,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:16:57,987 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:16:59,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:16:59,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:16:59,172 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:17:00,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:05,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:10,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:14,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:17:14,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:17:14,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:17:16,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:21,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:26,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:27,988 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:17:29,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:17:29,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:17:29,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:17:31,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:36,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:41,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:42,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:17:42,334 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:17:42,334 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:17:42,336 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:17:42,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:17:44,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:17:44,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:17:44,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:17:45,369 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:17:47,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:52,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:57,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:17:57,991 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:17:59,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:17:59,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:17:59,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:18:02,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:07,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:12,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:14,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:18:14,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:18:14,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:18:18,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:23,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:27,993 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:18:28,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:29,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:18:29,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:18:29,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:18:34,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:39,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:44,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:18:44,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:18:44,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:18:44,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:49,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:54,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:18:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:18:56,946 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:18:56,946 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:18:56,948 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:18:57,418 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:18:57,993 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:18:59,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:18:59,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:18:59,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:18:59,419 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:19:00,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:05,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:10,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:14,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:19:14,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:19:14,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:19:15,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:20,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:25,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:27,994 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:19:29,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:19:29,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:19:29,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:19:30,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:35,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:40,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:44,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:19:44,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:19:44,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:19:45,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:50,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:55,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:19:57,995 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:19:59,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:19:59,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:19:59,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:20:01,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:06,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:10,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:20:10,605 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:20:10,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:20:10,606 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:20:11,463 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:20:11,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:13,464 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:20:14,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:20:14,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:20:14,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:20:17,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:22,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:27,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:27,996 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:20:29,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:20:29,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:20:29,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:20:32,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:37,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:42,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:44,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:20:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:20:44,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:20:47,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:52,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:57,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:20:57,997 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:20:59,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:20:59,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:20:59,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:21:03,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:08,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:13,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:14,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:21:14,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:21:14,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:21:19,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:24,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:24,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:21:24,735 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:21:24,736 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:21:24,737 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:21:25,507 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:21:27,508 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:21:27,998 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:21:29,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:21:29,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:21:29,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:21:29,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:34,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:39,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:44,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:21:44,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:21:44,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:21:45,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:50,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:55,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:21:57,999 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:21:59,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:21:59,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:21:59,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:22:01,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:06,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:11,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:14,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:22:14,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:22:14,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:22:16,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:21,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:26,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:28,000 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:22:29,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:22:29,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:22:29,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:22:31,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:36,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:38,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:22:38,962 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:22:38,963 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:22:38,964 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:22:39,550 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:22:41,552 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:22:41,965 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:44,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:22:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:22:44,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:22:47,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:52,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:57,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:22:58,001 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:22:59,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:22:59,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:22:59,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:23:03,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:08,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:13,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:14,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:23:14,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:23:14,175 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:23:19,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:24,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:28,002 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:23:29,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:23:29,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:23:29,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:23:30,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:35,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:40,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:44,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:23:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:23:44,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:23:46,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:51,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:52,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:23:52,876 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:23:52,877 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:23:52,878 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:23:53,595 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:23:55,596 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:23:56,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:23:58,003 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:23:59,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:23:59,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:23:59,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:24:02,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:07,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:12,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:14,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:24:14,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:24:14,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:24:17,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:22,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:27,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:28,004 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:24:29,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:24:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:24:29,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:24:33,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:38,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:43,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:44,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:24:44,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:24:44,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:24:48,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:53,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:58,005 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:24:58,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:24:59,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:24:59,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:24:59,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:25:04,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:06,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:25:06,825 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:25:06,825 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:25:06,827 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:25:07,641 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:25:09,642 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:25:09,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:14,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:25:14,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:25:14,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:25:15,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:20,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:25,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:28,006 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:25:29,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:25:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:25:29,176 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:25:31,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:36,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:41,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:25:44,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:25:44,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:25:47,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:52,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:57,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:25:58,007 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:25:59,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:25:59,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:25:59,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:26:02,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:07,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:12,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:14,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:26:14,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:26:14,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:26:17,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:21,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:26:21,117 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:26:21,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:26:21,119 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:26:21,688 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:26:23,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:23,689 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:26:28,007 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:26:28,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:29,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:26:29,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:26:29,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:26:34,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:39,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:44,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:26:44,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:26:44,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:26:45,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:50,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:55,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:26:58,008 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:26:59,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:26:59,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:26:59,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:27:00,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:05,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:10,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:14,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:27:14,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:27:14,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:27:15,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:20,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:25,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:28,010 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:27:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:27:29,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:27:29,177 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:27:30,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:35,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:35,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:27:35,458 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:27:35,459 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:27:35,460 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:27:35,735 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:27:37,736 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:27:40,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:44,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:27:44,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:27:44,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:27:46,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:51,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:56,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:27:58,011 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:27:59,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:27:59,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:27:59,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:28:02,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:07,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:12,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:14,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:28:14,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:28:14,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:28:17,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:22,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:27,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:28,012 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:28:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:28:29,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:28:29,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:28:32,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:37,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:42,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:44,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:28:44,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:28:44,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:28:48,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:49,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:28:49,456 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:28:49,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:28:49,458 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:28:49,780 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:28:50,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:28:51,781 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:28:53,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:53,783 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:28:55,784 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:28:58,012 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:28:58,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:28:59,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:28:59,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:28:59,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:29:04,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:09,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:14,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:29:14,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:29:14,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:29:14,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:19,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:24,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:28,015 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:29:29,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:29:29,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:29:29,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:29:30,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:35,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:40,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:44,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:29:44,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:29:44,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:29:46,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:51,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:56,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:29:58,014 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:29:59,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:29:59,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:29:59,178 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:30:01,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:06,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:07,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:30:07,393 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:30:07,394 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:30:07,395 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:30:07,826 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:30:09,827 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:30:12,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:14,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:30:14,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:30:14,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:30:18,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:23,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:28,015 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:30:28,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:29,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:30:29,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:30:29,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:30:34,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:39,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:44,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:30:44,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:30:44,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:30:44,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:49,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:54,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:30:58,016 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:30:59,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:30:59,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:30:59,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:31:00,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:05,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:10,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:14,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:31:14,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:31:14,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:31:16,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:21,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:31:21,410 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:31:21,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:31:21,412 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:31:21,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:31:22,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:23,871 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:31:27,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:28,017 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:31:29,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:31:29,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:31:29,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:31:32,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:37,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:42,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:44,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:31:44,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:31:44,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:31:48,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:53,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:58,018 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:31:58,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:31:59,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:31:59,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:31:59,179 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:32:04,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:09,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:14,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:32:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:32:14,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:32:15,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:20,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:25,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:28,019 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:32:29,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:32:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:32:29,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:32:30,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:35,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:35,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:32:35,452 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:32:35,452 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:32:35,454 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:32:35,915 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:32:37,916 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:32:40,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:44,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:32:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:32:44,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:32:46,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:51,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:56,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:32:58,020 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:32:59,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:32:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:32:59,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:33:02,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:07,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:12,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:14,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:33:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:33:14,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:33:18,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:23,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:28,021 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:33:28,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:29,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:33:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:33:29,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:33:34,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:39,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:44,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:33:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:33:44,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:33:44,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:49,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:49,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:33:49,489 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:33:49,490 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:33:49,491 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:33:49,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:33:51,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:33:54,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:33:58,022 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:33:59,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:33:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:33:59,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:34:00,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:05,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:10,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:14,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:34:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:34:14,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:34:15,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:20,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:25,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:28,023 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:34:29,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:34:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:34:29,180 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:34:31,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:36,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:41,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:44,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:34:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:34:44,181 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:34:47,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:52,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:57,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:34:58,024 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:34:59,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:34:59,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:34:59,181 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:35:02,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:03,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:35:03,603 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:35:03,603 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:35:03,605 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:35:04,014 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:35:06,015 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:35:07,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:12,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:14,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:35:14,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:35:14,181 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:35:18,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:23,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:28,026 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:35:28,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:29,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:35:29,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:35:29,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:35:34,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:39,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:44,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:35:44,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:35:44,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:35:44,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:49,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:54,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:35:58,026 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:35:59,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:35:59,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:35:59,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:35:59,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:04,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:09,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:14,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:36:14,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:36:14,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:36:15,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:17,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:36:17,774 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:36:17,774 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:36:17,776 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:36:18,060 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:36:20,061 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:36:20,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:25,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:28,027 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:36:29,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:36:29,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:36:29,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:36:31,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:36,449 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:41,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:36:44,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:36:44,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:36:47,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:52,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:57,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:36:58,028 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:36:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:36:59,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:36:59,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:37:03,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:08,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:13,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:37:14,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:37:14,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:37:18,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:23,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:28,029 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:37:28,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:37:29,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:37:29,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:37:31,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:37:31,774 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:37:31,774 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:37:31,776 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:37:32,120 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:37:34,121 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:37:34,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:39,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:37:44,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:37:44,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:37:45,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:50,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:55,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:37:58,030 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:37:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:37:59,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:37:59,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:38:01,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:06,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:11,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:14,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:38:14,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:38:14,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:38:16,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:21,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:26,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:28,031 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:38:29,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:38:29,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:38:29,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:38:31,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:36,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:41,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:44,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:38:44,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:38:44,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:38:45,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:38:45,901 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:38:45,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:38:45,903 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:38:46,166 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:38:46,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:48,167 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:38:51,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:56,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:38:58,031 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:38:59,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:38:59,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:38:59,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:39:02,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:07,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:12,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:14,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:39:14,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:39:14,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:39:17,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:22,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:27,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:28,032 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:39:29,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:39:29,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:39:29,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:39:32,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:37,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:42,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:44,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:39:44,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:39:44,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:39:48,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:53,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:58,033 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:39:58,973 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:39:59,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:39:59,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:39:59,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:39:59,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:39:59,951 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:39:59,951 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:39:59,953 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:40:00,211 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:40:02,212 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:40:04,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:09,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:14,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:40:14,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:40:14,183 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:40:15,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:20,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:25,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:28,034 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:40:29,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:40:29,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:40:29,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:40:31,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:36,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:41,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:44,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:40:44,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:40:44,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:40:46,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:51,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:56,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:40:58,035 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:40:59,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:40:59,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:40:59,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:41:02,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:07,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:12,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:13,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:41:13,678 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:41:13,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:41:13,680 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:41:14,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:41:14,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:41:14,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:41:14,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:41:14,258 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:41:14,258 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:41:16,259 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:41:17,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:18,260 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:41:22,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:27,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:28,036 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:41:29,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:41:29,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:41:29,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:41:33,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:38,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:43,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:44,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:41:44,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:41:44,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:41:48,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:53,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:58,037 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:41:58,974 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:41:59,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:41:59,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:41:59,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:42:04,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:09,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:14,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:42:14,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:42:14,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:42:15,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:20,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:25,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:28,038 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:42:29,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:42:29,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:42:29,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:42:30,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:31,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:42:31,617 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:42:31,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:42:31,619 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:42:32,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:42:32,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:42:35,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:40,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:44,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:42:44,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:42:44,184 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:42:46,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:51,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:56,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:42:58,041 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:42:59,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:42:59,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:42:59,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:43:02,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:07,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:12,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:14,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:43:14,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:43:14,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:43:17,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:22,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:27,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:28,041 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:43:29,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:43:29,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:43:29,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:43:32,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:37,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:42,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:44,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:43:44,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:43:44,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:43:46,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:43:46,200 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:43:46,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:43:46,203 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:43:46,356 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:43:48,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:48,357 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:43:53,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:58,041 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:43:58,975 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:43:59,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:43:59,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:43:59,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:44:04,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:09,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:14,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:44:14,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:44:14,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:44:15,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:20,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:25,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:28,043 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:44:29,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:44:29,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:44:29,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:44:31,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:36,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:41,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:44,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:44:44,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:44:44,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:44:47,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:52,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:57,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:44:58,043 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:44:59,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:44:59,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:44:59,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:45:00,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:45:00,067 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:45:00,068 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:45:00,069 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:45:00,402 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:45:02,403 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:45:03,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:08,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:13,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:14,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:45:14,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:45:14,186 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:45:18,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:23,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:28,044 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:45:28,976 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:29,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:45:29,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:45:29,186 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:45:34,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:39,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:44,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:45:44,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:45:44,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:45:44,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:49,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:54,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:45:58,045 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:45:59,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:45:59,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:45:59,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:45:59,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:04,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:09,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:14,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:46:14,027 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:46:14,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:46:14,029 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:46:14,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:46:14,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:46:14,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:46:14,447 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:46:14,447 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:46:15,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:20,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:25,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:28,046 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:46:29,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:46:29,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:46:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:46:31,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:36,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:41,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:44,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:46:44,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:46:44,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:46:47,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:52,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:57,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:46:58,052 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:46:59,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:46:59,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:46:59,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:47:02,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:07,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:12,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:14,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:47:14,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:47:14,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:47:17,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:22,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:27,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:28,048 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:47:28,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:47:28,096 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:47:28,096 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:47:28,098 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:47:28,492 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:47:29,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:47:29,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:47:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:47:29,493 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:47:32,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:37,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:42,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:44,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:47:44,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:47:44,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:47:48,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:53,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:58,049 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:47:58,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:47:59,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:47:59,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:47:59,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:48:04,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:09,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:14,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:48:14,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:48:14,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:48:15,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:20,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:25,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:28,050 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:48:29,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:48:29,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:48:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:48:30,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:35,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:40,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:42,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:48:42,043 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:48:42,043 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:48:42,044 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:48:42,535 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:48:43,536 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:48:44,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:48:44,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:48:44,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:48:45,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:50,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:55,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:48:58,051 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:48:59,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:48:59,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:48:59,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:49:01,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:06,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:11,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:14,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:49:14,195 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:49:14,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:49:17,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:22,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:27,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:28,052 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:49:29,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:49:29,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:49:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:49:32,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:37,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:42,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:44,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:49:44,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:49:44,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:49:47,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:52,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:55,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:49:55,943 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:49:55,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:49:55,945 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:49:56,580 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:49:57,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:49:57,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:49:58,052 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:49:59,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:49:59,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:49:59,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:50:03,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:08,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:13,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:14,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:50:14,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:50:14,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:50:19,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:24,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:28,053 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:50:29,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:50:29,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:50:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:50:29,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:34,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:39,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:44,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:50:44,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:50:44,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:50:44,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:49,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:54,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:50:58,056 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:50:59,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:50:59,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:50:59,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:50:59,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:04,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:09,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:09,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:51:09,888 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:51:09,888 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:51:09,889 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:51:10,626 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:51:11,626 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:51:14,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:51:14,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:51:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:51:15,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:20,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:25,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:28,055 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:51:29,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:51:29,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:51:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:51:31,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:36,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:41,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:44,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:51:44,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:51:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:51:46,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:51,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:56,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:51:58,057 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:51:59,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:51:59,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:51:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:52:01,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:06,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:11,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:14,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:52:14,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:52:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:52:16,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:21,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:24,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:52:24,228 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:52:24,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:52:24,229 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:52:24,675 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:52:25,676 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:52:27,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:28,057 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:52:29,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:52:29,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:52:29,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:52:32,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:37,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:42,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:44,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:52:44,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:52:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:52:47,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:52,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:57,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:52:58,058 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:52:59,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:52:59,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:52:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:53:02,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:07,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:12,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:14,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:53:14,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:53:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:53:18,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:23,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:28,059 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:53:28,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:29,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:53:29,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:53:29,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:53:34,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:38,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:53:38,300 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:53:38,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:53:38,302 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:53:38,720 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:53:38,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:53:39,721 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:53:39,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:41,722 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:53:43,723 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:53:44,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:53:44,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:53:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:53:45,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:50,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:55,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:53:58,060 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:53:59,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:53:59,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:53:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:54:00,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:05,472 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:10,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:14,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:54:14,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:54:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:54:16,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:21,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:26,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:28,061 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:54:29,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:54:29,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:54:29,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:54:32,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:37,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:42,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:44,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:54:44,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:54:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:54:48,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:53,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:56,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:54:56,140 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:54:56,141 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:54:56,142 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:54:56,769 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:54:57,770 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:54:58,062 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:54:58,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:54:59,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:54:59,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:54:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:55:04,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:09,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:14,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:55:14,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:55:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:55:14,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:19,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:24,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:28,063 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:55:29,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:55:29,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:55:29,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:55:30,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:35,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:40,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:44,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:55:44,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:55:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:55:46,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:51,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:56,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:55:58,064 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:55:59,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:55:59,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:55:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:56:01,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:06,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:10,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:56:10,096 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:56:10,096 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:56:10,099 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:56:10,815 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:56:11,815 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:56:12,137 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:14,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:56:14,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:56:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:56:17,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:22,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:27,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:28,065 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:56:29,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:56:29,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:56:29,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:56:33,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:38,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:43,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:44,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:56:44,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:56:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:56:49,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:54,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:56:58,066 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:56:59,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:56:59,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:56:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:56:59,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:04,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:09,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:14,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:57:14,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:57:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:57:15,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:20,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:23,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:57:23,968 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:57:23,968 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:57:23,969 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:57:24,863 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:57:25,863 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:57:26,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:28,066 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:57:29,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:57:29,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:57:29,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:57:31,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:36,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:41,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:44,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:57:44,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:57:44,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:57:47,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:52,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:57,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:57:58,069 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:57:59,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:57:59,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:57:59,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:58:03,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:08,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:13,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:14,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:58:14,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:58:14,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:58:19,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:24,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:28,068 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:58:29,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:58:29,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:58:29,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:58:29,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:34,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:37,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:58:37,743 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:58:37,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:58:37,746 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:58:37,907 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:58:39,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:39,908 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:58:44,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:58:44,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:58:44,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:58:45,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:50,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:55,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:58:58,069 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:58:59,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:58:59,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:58:59,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:59:01,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:06,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:11,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:14,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:59:14,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:59:14,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:59:16,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:21,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:26,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:28,071 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:59:29,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:59:29,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:59:29,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:59:32,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:37,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:42,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:44,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:59:44,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:59:44,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 21:59:47,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:51,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 21:59:51,875 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 21:59:51,875 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 21:59:51,876 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 21:59:51,954 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 21:59:52,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:53,955 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 21:59:57,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 21:59:58,071 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 21:59:59,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 21:59:59,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 21:59:59,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:00:03,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:08,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:13,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:14,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:00:14,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:00:14,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:00:19,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:24,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:28,072 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:00:29,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:00:29,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:00:29,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:00:30,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:35,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:40,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:44,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:00:44,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:00:44,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:00:45,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:50,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:55,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:00:58,073 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:00:59,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:00:59,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:00:59,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:01:01,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:06,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:01:06,257 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:01:06,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:01:06,259 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:01:07,004 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:01:07,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:08,005 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:01:12,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:14,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:01:14,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:01:14,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:01:17,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:22,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:27,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:28,074 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:01:29,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:01:29,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:01:29,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:01:32,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:37,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:42,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:44,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:01:44,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:01:44,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:01:48,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:53,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:58,075 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:01:58,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:01:59,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:01:59,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:01:59,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:02:04,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:09,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:14,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:02:14,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:02:14,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:02:15,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:20,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:02:20,201 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:02:20,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:02:20,203 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:02:21,054 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:02:21,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:22,054 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:02:26,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:28,076 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:02:29,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:02:29,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:02:29,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:02:31,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:36,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:41,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:44,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:02:44,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:02:44,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:02:47,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:52,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:57,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:02:58,077 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:02:59,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:02:59,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:02:59,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:03:03,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:08,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:13,485 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:14,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:03:14,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:03:14,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:03:18,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:23,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:28,078 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:03:28,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:29,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:03:29,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:03:29,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:03:33,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:03:33,997 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:03:33,997 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:03:33,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:33,999 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:03:34,102 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:03:36,103 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:03:39,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:44,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:03:44,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:03:44,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:03:44,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:49,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:54,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:03:58,079 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:03:59,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:03:59,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:03:59,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:04:00,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:05,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:10,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:14,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:04:14,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:04:14,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:04:15,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:20,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:25,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:28,080 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:04:29,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:04:29,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:04:29,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:04:31,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:36,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:41,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:44,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:04:44,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:04:44,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:04:47,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:47,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:04:47,834 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:04:47,834 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:04:47,836 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:04:48,148 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:04:50,149 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:04:52,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:57,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:04:58,080 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:04:59,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:04:59,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:04:59,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:05:03,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:08,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:13,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:14,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:05:14,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:05:14,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:05:18,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:23,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:28,081 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:05:28,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:29,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:05:29,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:05:29,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:05:34,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:39,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:44,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:05:44,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:05:44,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:05:45,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:50,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:55,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:05:58,082 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:05:59,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:05:59,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:05:59,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:06:01,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:01,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:06:01,587 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:06:01,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:06:01,588 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:06:02,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:06:02,194 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:06:04,195 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:06:05,196 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:06:06,196 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:06:06,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:08,198 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:06:11,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:14,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:06:14,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:06:14,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:06:17,472 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:22,472 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:27,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:28,083 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:06:29,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:06:29,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:06:29,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:06:32,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:37,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:42,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:44,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:06:44,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:06:44,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:06:48,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:53,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:58,084 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:06:58,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:06:59,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:06:59,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:06:59,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:07:04,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:09,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:14,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:07:14,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:07:14,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:07:14,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:19,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:19,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:07:19,784 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:07:19,785 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:07:19,786 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:07:20,242 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:07:22,243 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:07:24,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:28,085 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:07:29,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:07:29,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:07:29,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:07:30,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:35,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:40,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:44,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:07:44,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:07:44,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:07:45,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:50,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:55,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:07:58,086 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:07:59,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:07:59,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:07:59,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:08:01,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:06,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:11,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:14,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:08:14,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:08:14,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:08:17,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:22,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:27,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:28,087 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:08:29,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:08:29,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:08:29,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:08:32,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:33,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:08:33,817 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:08:33,817 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:08:33,819 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:08:34,289 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:08:36,291 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:08:37,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:42,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:44,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:08:44,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:08:44,289 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:08:48,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:53,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:58,088 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:08:58,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:08:59,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:08:59,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:08:59,289 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:09:04,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:09,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:14,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:09:14,329 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:09:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:09:14,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:19,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:24,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:28,089 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:09:29,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:09:29,329 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:09:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:09:30,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:35,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:40,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:44,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:09:44,329 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:09:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:09:45,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:47,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:09:47,949 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:09:47,949 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:09:47,950 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:09:48,339 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:09:50,340 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:09:50,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:55,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:09:58,090 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:09:59,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:09:59,329 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:09:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:10:01,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:06,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:11,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:14,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:10:14,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:10:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:10:16,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:21,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:26,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:28,091 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:10:29,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:10:29,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:10:29,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:10:32,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:37,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:42,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:44,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:10:44,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:10:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:10:48,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:53,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:58,092 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:10:58,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:10:59,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:10:59,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:10:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:11:02,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:11:02,105 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:11:02,105 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:11:02,106 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:11:02,386 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:11:04,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:04,388 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:11:09,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:14,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:11:14,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:11:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:11:14,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:19,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:24,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:28,093 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:11:29,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:11:29,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:11:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:11:30,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:35,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:40,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:44,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:11:44,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:11:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:11:45,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:50,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:55,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:11:58,094 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:11:59,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:11:59,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:11:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:12:01,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:06,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:11,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:14,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:12:14,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:12:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:12:15,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:12:15,866 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:12:15,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:12:15,867 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:12:16,433 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:12:16,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:18,434 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:12:21,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:26,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:28,095 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:12:29,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:12:29,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:12:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:12:32,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:37,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:42,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:44,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:12:44,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:12:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:12:48,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:53,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:58,096 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:12:58,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:12:59,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:12:59,330 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:12:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:13:04,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:09,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:14,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:13:14,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:13:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:13:14,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:19,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:24,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:28,097 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:13:29,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:13:29,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:13:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:13:29,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:13:29,589 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:13:29,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:13:29,590 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:13:29,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:30,478 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:13:30,478 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:13:34,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:39,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:44,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:13:44,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:13:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:13:45,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:50,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:55,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:13:58,097 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:13:59,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:13:59,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:13:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:14:01,505 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:06,505 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:11,506 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:14,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:14:14,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:14:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:14:16,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:21,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:26,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:28,098 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:14:29,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:14:29,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:14:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:14:31,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:36,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:41,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:43,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:14:43,721 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:14:43,721 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:14:43,722 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:14:44,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:14:44,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:14:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:14:44,523 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:14:44,524 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:14:47,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:52,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:57,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:14:58,099 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:14:59,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:14:59,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:14:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:15:02,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:07,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:12,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:14,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:15:14,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:15:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:15:18,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:23,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:28,100 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:15:28,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:29,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:15:29,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:15:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:15:34,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:39,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:44,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:15:44,332 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:15:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:15:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:49,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:54,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:15:57,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:15:57,434 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:15:57,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:15:57,436 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:15:57,569 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:15:58,101 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:15:58,570 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:15:59,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:15:59,332 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:15:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:16:00,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:05,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:10,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:14,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:16:14,332 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:16:14,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:16:16,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:21,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:26,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:28,102 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:16:29,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:16:29,332 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:16:29,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:16:31,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:36,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:41,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:44,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:16:44,332 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:16:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:16:47,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:52,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:57,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:16:58,103 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:16:59,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:16:59,333 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:16:59,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:17:02,592 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:07,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:11,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:17:11,645 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:17:11,646 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:17:11,647 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:17:12,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:17:12,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:17:12,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:14,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:17:14,333 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:17:14,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:17:18,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:23,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:28,104 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:17:28,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:29,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:17:29,333 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:17:29,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:17:34,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:39,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:44,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:17:44,333 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:17:44,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:17:44,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:49,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:54,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:17:58,105 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:17:59,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:17:59,333 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:17:59,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:18:00,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:05,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:10,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:14,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:18:14,333 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:18:14,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:18:15,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:20,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:25,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:25,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:18:25,596 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:18:25,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:18:25,598 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:18:25,663 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:18:26,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:18:26,664 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:18:28,106 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:18:28,666 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:18:29,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:18:29,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:18:29,441 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:18:29,667 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:18:30,668 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:18:30,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:35,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:40,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:44,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:18:44,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:18:44,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:18:46,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:51,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:56,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:18:58,107 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:18:59,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:18:59,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:18:59,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:19:01,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:06,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:11,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:14,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:19:14,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:19:14,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:19:16,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:21,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:26,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:28,108 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:19:29,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:19:29,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:19:29,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:19:32,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:37,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:42,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:43,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:19:43,785 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:19:43,785 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:19:43,787 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:19:44,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:19:44,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:19:44,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:19:44,711 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:19:44,711 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:19:47,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:52,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:57,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:19:58,109 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:19:59,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:19:59,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:19:59,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:20:03,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:08,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:13,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:14,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:20:14,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:20:14,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:20:18,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:23,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:28,110 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:20:28,997 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:29,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:20:29,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:20:29,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:20:34,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:39,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:44,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:20:44,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:20:44,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:20:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:50,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:55,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:20:57,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:20:57,597 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:20:57,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:20:57,598 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:20:57,754 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:20:58,111 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:20:58,755 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:20:59,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:20:59,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:20:59,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:21:00,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:05,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:10,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:14,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:21:14,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:21:14,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:21:16,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:21,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:26,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:28,112 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:21:29,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:21:29,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:21:29,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:21:31,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:36,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:41,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:44,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:21:44,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:21:44,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:21:47,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:52,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:57,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:21:58,113 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:21:59,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:21:59,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:21:59,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:22:02,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:07,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:11,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:22:11,415 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:22:11,415 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:22:11,417 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:22:11,798 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:22:12,799 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:22:13,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:14,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:22:14,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:22:14,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:22:18,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:23,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:28,113 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:22:28,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:29,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:22:29,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:22:29,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:22:34,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:39,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:44,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:22:44,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:22:44,347 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:22:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:50,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:55,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:22:58,114 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:22:59,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:22:59,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:22:59,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:23:00,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:05,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:10,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:14,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:23:14,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:23:14,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:23:15,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:20,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:25,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:23:25,269 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:23:25,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:23:25,271 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:23:25,843 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:23:26,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:26,844 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:23:28,115 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:23:29,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:23:29,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:23:29,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:23:31,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:36,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:41,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:44,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:23:44,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:23:44,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:23:46,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:51,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:56,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:23:58,117 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:23:59,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:23:59,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:23:59,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:24:01,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:06,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:11,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:14,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:24:14,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:24:14,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:24:17,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:22,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:27,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:28,117 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:24:29,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:24:29,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:24:29,348 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:24:32,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:37,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:39,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:24:39,324 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:24:39,324 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:24:39,326 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:24:39,892 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:24:41,893 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:24:43,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:44,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:24:44,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:24:44,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:24:48,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:53,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:58,118 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:24:58,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:24:59,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:24:59,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:24:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:25:04,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:09,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:14,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:25:14,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:25:14,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:25:14,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:19,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:24,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:28,119 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:25:29,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:25:29,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:25:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:25:29,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:34,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:39,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:44,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:25:44,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:25:44,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:25:44,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:49,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:53,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:25:53,714 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:25:53,714 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:25:53,715 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:25:53,940 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:25:54,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:25:55,942 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:25:58,120 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:25:59,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:25:59,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:25:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:26:00,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:05,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:10,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:14,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:26:14,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:26:14,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:26:15,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:20,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:25,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:28,121 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:26:29,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:26:29,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:26:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:26:31,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:36,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:41,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:44,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:26:44,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:26:44,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:26:47,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:52,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:57,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:26:58,122 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:26:59,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:26:59,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:26:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:27:02,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:07,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:27:07,551 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:27:07,552 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:27:07,553 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:27:07,986 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:27:08,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:09,987 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:27:13,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:14,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:27:14,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:27:14,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:27:18,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:23,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:28,123 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:27:29,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:29,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:27:29,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:27:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:27:34,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:39,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:44,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:27:44,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:27:44,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:27:44,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:49,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:54,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:27:58,124 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:27:59,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:27:59,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:27:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:28:00,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:05,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:10,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:14,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:28:14,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:28:14,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:28:16,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:21,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:28:21,423 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:28:21,424 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:28:21,425 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:28:22,032 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:28:22,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:24,033 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:28:27,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:28,125 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:28:29,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:28:29,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:28:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:28:32,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:37,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:42,603 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:44,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:28:44,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:28:44,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:28:47,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:52,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:57,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:28:58,127 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:28:59,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:28:59,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:28:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:29:02,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:07,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:12,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:14,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:29:14,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:29:14,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:29:17,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:22,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:27,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:28,126 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:29:29,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:29:29,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:29:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:29:33,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:35,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:29:35,444 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:29:35,444 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:29:35,446 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:29:36,078 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:29:38,080 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:29:39,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:44,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:29:44,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:29:44,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:29:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:49,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:54,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:29:58,128 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:29:59,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:29:59,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:29:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:29:59,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:04,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:09,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:14,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:30:14,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:30:14,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:30:14,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:19,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:24,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:28,129 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:30:29,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:30:29,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:30:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:30:30,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:35,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:40,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:44,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:30:44,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:30:44,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:30:45,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:49,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:30:49,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:30:49,268 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:30:49,269 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:30:49,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:30:50,128 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:30:50,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:52,129 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:30:53,130 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:30:54,131 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:30:56,132 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:30:56,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:30:58,129 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:30:59,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:30:59,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:30:59,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:31:01,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:06,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:11,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:14,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:31:14,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:31:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:31:17,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:22,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:27,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:28,130 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:31:29,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:31:29,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:31:29,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:31:32,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:37,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:42,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:44,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:31:44,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:31:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:31:48,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:53,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:58,131 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:31:59,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:31:59,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:31:59,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:31:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:32:04,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:07,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:32:07,350 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:32:07,350 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:32:07,351 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:32:08,179 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:32:10,181 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:32:10,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:14,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:32:14,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:32:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:32:15,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:20,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:25,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:28,132 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:32:29,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:32:29,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:32:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:32:31,592 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:36,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:41,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:44,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:32:44,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:32:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:32:46,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:51,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:56,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:32:58,133 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:32:59,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:32:59,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:32:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:33:01,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:06,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:11,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:14,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:33:14,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:33:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:33:17,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:21,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:33:21,181 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:33:21,181 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:33:21,182 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:33:21,226 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:33:23,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:24,228 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:33:28,134 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:33:29,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:29,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:33:29,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:33:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:33:34,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:39,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:44,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:33:44,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:33:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:33:45,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:50,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:55,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:33:58,135 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:33:59,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:33:59,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:33:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:34:01,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:06,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:11,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:14,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:34:14,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:34:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:34:16,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:21,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:26,624 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:28,136 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:34:29,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:34:29,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:34:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:34:31,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:35,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:34:35,553 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:34:35,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:34:35,554 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:34:36,275 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:34:37,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:38,276 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:34:42,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:44,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:34:44,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:34:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:34:47,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:52,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:57,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:34:58,137 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:34:59,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:34:59,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:34:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:35:03,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:08,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:13,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:14,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:35:14,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:35:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:35:18,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:23,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:28,139 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:35:29,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:29,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:35:29,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:35:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:35:34,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:39,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:44,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:35:44,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:35:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:35:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:49,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:35:49,263 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:35:49,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:35:49,264 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:35:49,323 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:35:50,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:52,324 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:35:55,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:35:58,140 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:35:59,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:35:59,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:35:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:36:00,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:05,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:10,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:14,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:36:14,410 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:36:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:36:16,587 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:21,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:26,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:28,140 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:36:29,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:36:29,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:36:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:36:31,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:36,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:41,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:44,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:36:44,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:36:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:36:46,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:51,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:56,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:36:58,141 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:36:59,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:36:59,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:36:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:37:02,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:03,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:37:03,186 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:37:03,186 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:37:03,188 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:37:03,370 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:37:04,371 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:37:08,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:13,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:14,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:37:14,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:37:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:37:18,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:23,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:28,142 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:37:29,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:29,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:37:29,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:37:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:37:34,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:39,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:44,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:37:44,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:37:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:37:45,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:50,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:55,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:37:58,143 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:37:59,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:37:59,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:37:59,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:38:00,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:05,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:10,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:14,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:38:14,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:38:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:38:16,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:17,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:38:17,127 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:38:17,128 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:38:17,129 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:38:17,417 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:38:18,418 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:38:22,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:27,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:28,144 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:38:29,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:38:29,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:38:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:38:32,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:37,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:42,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:44,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:38:44,411 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:38:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:38:47,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:52,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:57,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:38:58,145 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:38:59,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:38:59,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:38:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:39:03,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:08,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:13,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:14,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:39:14,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:39:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:39:18,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:23,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:28,146 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:39:29,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:29,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:39:29,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:39:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:39:31,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:39:31,102 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:39:31,102 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:39:31,103 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:39:31,463 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:39:32,463 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:39:34,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:39,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:44,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:39:44,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:39:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:39:44,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:49,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:54,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:39:58,147 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:39:59,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:39:59,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:39:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:40:00,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:05,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:10,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:14,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:40:14,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:40:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:40:15,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:20,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:25,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:28,150 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:40:29,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:40:29,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:40:29,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:40:31,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:36,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:41,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:44,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:40:44,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:40:44,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:40:44,993 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:40:44,995 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:40:44,995 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:40:44,996 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:40:45,508 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:40:46,509 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:40:47,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:52,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:57,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:40:58,149 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:40:59,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:40:59,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:40:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:41:02,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:07,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:12,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:14,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:41:14,412 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:41:14,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:41:17,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:22,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:27,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:28,151 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:41:29,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:41:29,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:41:29,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:41:33,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:38,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:43,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:44,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:41:44,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:41:44,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:41:48,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:53,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:58,151 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:41:59,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:41:59,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:41:59,059 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:41:59,059 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:41:59,060 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:41:59,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:41:59,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:41:59,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:41:59,556 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:42:00,557 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:42:04,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:09,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:14,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:42:14,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:42:14,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:42:14,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:19,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:24,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:28,152 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:42:29,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:42:29,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:42:29,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:42:30,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:35,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:40,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:44,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:42:44,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:42:44,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:42:45,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:50,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:55,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:42:58,153 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:42:59,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:42:59,413 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:42:59,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:43:00,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:05,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:10,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:13,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:43:13,384 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:43:13,384 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:43:13,386 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:43:13,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:43:13,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:43:14,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:43:14,457 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:43:14,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:43:14,603 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:43:15,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:16,605 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:43:17,606 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:43:18,607 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:43:21,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:26,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:28,154 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:43:29,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:43:29,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:43:29,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:43:31,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:36,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:41,651 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:44,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:43:44,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:43:44,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:43:47,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:52,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:57,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:43:58,155 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:43:59,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:43:59,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:43:59,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:44:02,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:07,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:12,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:14,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:44:14,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:44:14,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:44:17,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:22,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:27,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:28,156 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:44:29,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:44:29,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:44:29,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:44:31,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:44:31,419 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:44:31,420 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:44:31,421 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:44:31,651 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:44:32,652 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:44:33,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:38,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:43,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:44,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:44:44,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:44:44,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:44:48,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:53,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:58,157 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:44:59,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:44:59,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:44:59,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:44:59,433 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:45:04,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:09,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:14,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:45:14,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:45:14,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:45:14,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:19,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:24,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:28,158 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:45:29,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:45:29,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:45:29,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:45:30,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:35,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:40,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:44,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:45:44,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:45:44,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:45:45,179 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:45:45,182 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:45:45,182 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:45:45,183 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:45:45,697 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:45:46,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:46,697 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:45:51,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:56,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:45:58,159 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:45:59,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:45:59,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:45:59,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:46:01,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:06,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:11,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:14,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:46:14,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:46:14,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:46:17,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:22,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:27,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:28,160 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:46:29,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:46:29,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:46:29,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:46:32,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:37,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:42,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:44,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:46:44,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:46:44,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:46:48,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:53,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:58,161 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:46:59,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:46:59,078 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:46:59,079 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:46:59,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:46:59,081 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:46:59,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:46:59,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:46:59,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:46:59,742 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:47:00,743 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:47:04,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:09,696 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:14,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:47:14,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:47:14,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:47:14,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:19,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:24,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:28,162 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:47:29,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:47:29,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:47:29,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:47:30,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:35,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:40,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:44,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:47:44,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:47:44,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:47:45,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:50,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:55,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:47:58,163 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:47:59,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:47:59,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:47:59,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:48:01,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:06,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:11,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:12,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:48:12,928 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:48:12,929 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:48:12,930 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:48:13,788 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:48:14,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:48:14,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:48:14,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:48:14,788 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:48:16,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:21,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:26,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:28,163 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:48:29,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:48:29,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:48:29,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:48:32,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:37,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:42,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:44,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:48:44,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:48:44,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:48:47,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:52,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:57,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:48:58,164 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:48:59,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:48:59,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:48:59,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:49:03,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:08,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:13,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:14,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:49:14,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:49:14,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:49:18,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:23,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:26,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:49:26,658 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:49:26,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:49:26,660 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:49:26,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:49:28,165 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:49:28,833 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:49:29,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:29,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:49:29,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:49:29,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:49:34,688 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:39,689 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:44,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:49:44,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:49:44,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:49:45,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:50,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:55,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:49:58,166 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:49:59,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:49:59,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:49:59,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:50:01,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:06,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:11,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:14,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:50:14,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:50:14,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:50:16,699 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:21,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:26,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:28,167 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:50:29,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:50:29,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:50:29,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:50:31,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:36,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:41,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:50:41,141 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:50:41,142 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:50:41,143 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:50:41,883 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:50:42,144 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:42,884 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:50:44,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:50:44,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:50:44,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:50:47,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:52,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:57,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:50:58,168 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:50:59,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:50:59,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:50:59,435 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:51:02,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:07,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:12,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:14,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:51:14,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:51:14,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:51:17,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:22,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:27,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:28,170 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:51:29,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:51:29,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:51:29,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:51:33,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:38,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:43,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:44,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:51:44,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:51:44,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:51:48,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:53,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:55,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:51:55,089 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:51:55,089 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:51:55,091 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:51:55,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:51:56,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:51:58,170 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:51:59,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:51:59,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:51:59,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:51:59,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:52:04,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:09,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:14,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:52:14,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:52:14,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:52:14,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:19,640 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:24,641 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:28,171 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:52:29,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:52:29,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:52:29,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:52:29,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:34,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:39,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:44,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:52:44,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:52:44,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:52:44,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:49,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:54,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:52:58,172 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:52:59,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:52:59,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:52:59,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:52:59,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:04,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:08,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:53:08,866 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:53:08,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:53:08,868 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:53:08,974 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:53:09,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:10,975 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:53:14,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:53:14,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:53:14,436 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:53:15,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:20,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:25,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:28,173 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:53:29,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:53:29,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:53:29,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:53:31,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:36,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:41,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:44,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:53:44,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:53:44,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:53:46,627 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:51,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:56,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:53:58,174 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:53:59,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:53:59,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:53:59,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:54:02,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:07,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:12,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:14,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:54:14,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:54:14,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:54:17,644 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:22,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:22,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:54:22,813 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:54:22,813 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:54:22,815 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:54:23,020 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:54:25,022 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:54:27,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:28,175 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:54:29,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:54:29,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:54:29,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:54:33,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:38,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:43,696 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:44,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:54:44,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:54:44,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:54:48,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:53,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:58,176 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:54:59,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:54:59,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:54:59,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:54:59,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:55:04,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:09,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:14,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:55:14,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:55:14,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:55:14,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:19,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:24,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:28,177 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:55:29,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:55:29,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:55:29,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:55:29,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:34,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:36,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:55:36,778 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:55:36,778 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:55:36,780 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:55:37,065 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:55:37,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:55:39,066 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:55:40,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:41,067 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:55:43,069 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:55:44,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:55:44,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:55:44,437 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:55:45,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:50,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:55,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:55:58,178 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:55:59,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:55:59,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:55:59,438 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:56:01,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:06,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:11,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:14,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:56:14,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:56:14,438 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:56:16,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:21,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:26,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:28,179 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:56:29,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:56:29,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:56:29,438 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:56:32,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:37,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:42,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:44,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:56:44,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:56:44,438 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:56:47,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:52,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:54,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:56:54,540 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:56:54,540 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:56:54,542 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:56:55,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:56:55,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:56:58,179 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:56:58,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:56:59,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:56:59,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:56:59,438 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:57:03,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:08,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:13,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:14,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:57:14,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:57:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:57:18,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:23,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:28,208 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:57:29,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:29,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:57:29,492 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:57:29,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:57:34,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:39,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:44,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:57:44,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:57:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:57:44,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:49,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:54,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:57:58,209 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:57:59,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:57:59,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:57:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:58:00,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:05,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:08,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:58:08,409 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:58:08,409 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:58:08,410 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:58:09,157 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:58:09,157 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:58:11,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:14,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:58:14,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:58:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:58:16,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:21,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:26,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:28,212 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:58:29,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:58:29,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:58:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:58:31,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:36,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:41,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:44,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:58:44,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:58:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:58:47,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:52,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:57,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:58:58,211 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:58:59,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:58:59,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:58:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:59:02,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:07,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:12,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:14,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:59:14,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:59:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:59:18,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:22,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 22:59:22,911 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 22:59:22,911 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 22:59:22,913 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 22:59:23,206 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 22:59:23,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:25,207 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 22:59:28,212 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:59:29,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:29,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:59:29,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:59:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:59:34,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:39,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:44,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:59:44,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:59:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 22:59:45,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:50,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:55,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 22:59:58,214 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 22:59:59,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 22:59:59,493 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 22:59:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:00:01,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:06,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:11,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:14,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:00:14,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:00:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:00:16,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:21,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:26,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:28,214 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:00:29,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:00:29,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:00:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:00:31,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:36,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:00:36,657 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:00:36,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:00:36,658 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:00:37,250 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:00:37,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:38,251 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:00:42,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:44,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:00:44,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:00:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:00:48,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:53,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:58,215 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:00:59,216 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:00:59,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:00:59,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:00:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:01:04,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:09,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:14,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:01:14,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:01:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:01:14,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:19,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:24,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:28,216 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:01:29,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:01:29,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:01:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:01:30,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:35,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:40,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:44,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:01:44,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:01:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:01:46,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:50,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:01:50,598 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:01:50,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:01:50,599 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:01:51,297 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:01:52,298 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:01:52,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:57,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:01:58,217 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:01:59,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:01:59,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:01:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:02:02,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:07,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:12,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:14,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:02:14,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:02:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:02:17,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:22,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:27,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:28,218 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:02:29,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:02:29,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:02:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:02:32,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:37,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:42,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:44,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:02:44,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:02:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:02:48,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:53,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:58,219 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:02:59,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:02:59,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:02:59,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:02:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:03:04,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:03:04,401 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:03:04,401 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:03:04,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:04,403 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:03:05,344 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:03:06,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:03:09,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:14,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:03:14,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:03:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:03:14,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:19,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:24,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:28,220 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:03:29,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:03:29,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:03:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:03:29,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:34,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:39,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:44,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:03:44,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:03:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:03:45,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:50,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:55,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:03:58,221 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:03:59,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:03:59,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:03:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:04:00,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:05,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:10,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:14,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:04:14,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:04:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:04:16,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:18,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:04:18,198 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:04:18,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:04:18,199 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:04:18,390 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:04:20,392 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:04:22,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:27,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:28,222 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:04:29,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:04:29,495 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:04:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:04:32,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:37,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:42,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:44,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:04:44,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:04:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:04:47,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:52,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:57,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:04:58,223 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:04:59,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:04:59,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:04:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:05:03,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:08,690 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:13,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:14,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:05:14,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:05:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:05:18,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:23,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:28,224 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:05:29,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:29,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:05:29,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:05:29,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:05:32,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:05:32,140 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:05:32,140 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:05:32,142 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:05:32,433 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:05:34,435 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:05:35,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:40,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:44,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:05:44,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:05:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:05:45,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:50,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:55,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:05:58,225 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:05:59,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:05:59,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:05:59,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:06:01,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:06,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:11,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:14,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:06:14,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:06:14,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:06:16,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:21,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:26,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:28,226 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:06:29,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:06:29,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:06:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:06:32,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:37,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:42,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:44,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:06:44,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:06:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:06:46,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:06:46,125 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:06:46,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:06:46,126 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:06:46,479 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:06:48,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:48,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:06:53,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:58,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:06:58,227 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:06:59,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:06:59,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:06:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:07:03,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:08,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:13,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:14,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:07:14,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:07:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:07:18,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:23,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:28,228 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:07:29,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:29,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:07:29,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:07:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:07:34,699 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:39,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:44,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:07:44,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:07:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:07:44,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:49,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:54,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:07:58,229 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:07:59,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:07:59,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:07:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:08:00,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:08:00,456 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:08:00,456 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:08:00,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:00,458 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:08:00,527 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:08:01,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:08:02,529 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:08:04,530 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:08:05,531 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:08:05,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:06,532 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:08:10,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:14,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:08:14,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:08:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:08:15,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:20,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:25,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:28,230 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:08:29,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:08:29,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:08:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:08:31,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:36,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:41,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:44,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:08:44,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:08:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:08:46,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:51,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:56,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:08:58,231 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:08:59,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:08:59,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:08:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:09:02,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:07,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:12,699 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:14,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:09:14,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:09:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:09:17,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:18,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:09:18,621 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:09:18,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:09:18,622 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:09:19,575 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:09:20,576 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:09:23,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:28,232 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:09:29,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:29,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:09:29,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:09:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:09:34,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:39,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:44,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:09:44,498 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:09:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:09:44,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:49,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:54,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:09:58,233 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:09:59,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:09:59,498 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:09:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:10:00,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:05,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:10,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:14,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:10:14,498 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:10:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:10:16,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:21,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:26,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:28,234 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:10:29,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:10:29,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:10:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:10:31,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:32,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:10:32,526 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:10:32,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:10:32,528 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:10:32,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:10:34,620 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:10:37,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:42,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:44,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:10:44,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:10:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:10:47,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:52,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:57,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:10:58,235 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:10:59,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:10:59,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:10:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:11:02,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:07,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:12,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:14,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:11:14,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:11:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:11:18,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:23,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:28,236 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:11:29,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:29,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:11:29,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:11:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:11:34,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:39,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:44,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:11:44,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:11:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:11:44,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:46,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:11:46,454 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:11:46,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:11:46,456 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:11:46,664 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:11:48,665 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:11:50,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:55,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:11:58,236 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:11:59,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:11:59,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:11:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:12:00,691 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:05,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:10,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:14,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:12:14,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:12:14,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:12:15,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:20,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:25,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:28,237 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:12:29,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:12:29,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:12:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:12:30,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:35,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:40,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:44,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:12:44,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:12:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:12:45,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:50,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:55,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:12:58,238 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:12:59,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:12:59,499 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:12:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:13:00,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:13:00,300 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:13:00,300 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:13:00,301 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:13:00,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:13:01,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:02,710 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:13:06,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:11,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:14,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:13:14,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:13:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:13:16,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:21,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:26,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:28,239 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:13:29,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:13:29,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:13:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:13:32,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:37,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:42,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:44,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:13:44,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:13:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:13:48,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:53,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:58,240 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:13:59,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:13:59,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:13:59,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:13:59,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:14:04,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:09,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:14,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:14:14,246 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:14:14,246 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:14:14,247 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:14:14,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:14:14,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:14:14,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:14:14,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:14,754 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:14:16,755 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:14:19,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:24,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:28,242 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:14:29,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:14:29,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:14:29,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:14:30,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:35,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:40,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:44,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:14:44,500 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:14:44,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:14:46,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:51,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:56,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:14:58,242 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:14:59,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:14:59,502 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:14:59,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:15:01,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:06,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:11,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:14,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:15:14,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:15:14,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:15:17,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:22,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:27,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:28,243 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:15:28,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:15:28,657 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:15:28,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:15:28,658 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:15:28,801 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:15:29,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:15:29,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:15:29,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:15:30,803 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:15:33,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:38,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:43,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:44,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:15:44,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:15:44,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:15:49,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:54,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:15:58,244 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:15:59,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:15:59,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:15:59,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:15:59,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:04,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:09,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:14,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:16:14,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:16:14,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:16:14,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:19,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:24,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:28,245 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:16:29,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:16:29,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:16:29,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:16:33,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:38,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:43,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:16:43,077 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:16:43,077 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:16:43,079 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:16:43,849 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:16:44,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:44,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:16:44,501 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:16:44,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:16:44,850 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:16:49,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:54,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:16:58,246 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:16:59,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:16:59,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:16:59,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:17:00,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:05,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:10,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:14,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:17:14,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:17:14,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:17:15,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:20,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:25,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:28,247 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:17:29,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:17:29,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:17:29,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:17:31,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:36,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:41,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:44,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:17:44,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:17:44,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:17:47,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:52,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:56,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:17:56,885 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:17:56,885 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:17:56,886 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:17:56,893 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:17:57,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:17:58,248 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:17:58,895 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:17:59,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:17:59,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:17:59,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:18:03,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:08,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:13,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:14,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:18:14,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:18:14,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:18:19,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:24,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:28,249 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:18:29,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:18:29,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:18:29,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:18:29,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:34,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:39,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:44,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:18:44,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:18:44,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:18:45,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:50,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:55,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:18:58,250 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:18:59,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:18:59,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:18:59,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:19:00,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:05,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:10,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:10,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:19:10,798 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:19:10,798 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:19:10,800 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:19:10,939 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:19:12,940 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:19:14,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:19:14,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:19:14,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:19:16,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:21,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:26,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:28,251 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:19:29,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:19:29,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:19:29,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:19:32,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:37,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:42,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:44,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:19:44,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:19:44,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:19:47,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:52,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:57,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:19:58,252 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:19:59,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:19:59,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:19:59,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:20:03,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:08,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:13,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:14,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:20:14,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:20:14,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:20:18,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:23,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:24,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:20:24,707 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:20:24,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:20:24,709 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:20:24,983 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:20:25,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:20:26,985 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:20:27,986 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:20:28,253 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:20:28,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:28,986 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:20:29,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:20:29,556 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:20:29,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:20:30,988 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:20:34,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:39,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:44,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:20:44,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:20:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:20:44,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:49,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:54,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:20:58,253 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:20:59,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:20:59,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:20:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:21:00,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:05,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:10,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:14,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:21:14,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:21:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:21:15,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:20,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:25,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:28,256 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:21:29,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:21:29,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:21:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:21:31,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:36,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:41,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:42,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:21:42,720 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:21:42,720 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:21:42,721 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:21:43,032 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:21:44,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:21:44,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:21:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:21:45,033 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:21:47,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:52,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:57,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:21:58,255 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:21:59,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:21:59,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:21:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:22:02,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:07,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:12,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:14,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:22:14,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:22:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:22:17,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:22,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:27,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:28,257 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:22:29,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:22:29,557 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:22:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:22:33,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:38,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:43,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:44,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:22:44,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:22:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:22:48,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:53,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:56,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:22:56,649 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:22:56,650 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:22:56,651 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:22:57,077 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:22:58,257 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:22:59,078 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:22:59,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:22:59,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:22:59,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:22:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:23:04,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:09,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:14,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:23:14,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:23:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:23:15,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:20,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:25,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:28,258 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:23:29,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:23:29,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:23:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:23:31,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:36,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:41,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:44,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:23:44,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:23:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:23:46,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:51,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:56,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:23:58,259 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:23:59,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:23:59,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:23:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:24:01,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:06,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:11,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:24:11,279 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:24:11,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:24:11,281 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:24:12,127 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:24:12,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:13,128 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:24:14,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:24:14,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:24:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:24:17,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:22,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:27,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:28,260 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:24:29,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:24:29,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:24:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:24:32,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:37,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:42,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:44,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:24:44,558 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:24:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:24:47,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:52,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:57,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:24:58,261 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:24:59,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:24:59,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:24:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:25:03,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:08,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:13,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:14,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:25:14,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:25:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:25:18,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:23,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:25,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:25:25,288 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:25:25,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:25:25,290 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:25:26,173 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:25:27,173 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:25:28,262 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:25:29,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:29,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:25:29,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:25:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:25:34,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:39,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:44,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:25:44,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:25:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:25:45,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:50,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:55,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:25:58,263 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:25:59,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:25:59,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:25:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:26:00,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:05,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:10,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:14,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:26:14,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:26:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:26:15,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:20,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:25,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:28,264 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:26:29,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:26:29,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:26:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:26:31,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:36,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:39,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:26:39,121 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:26:39,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:26:39,122 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:26:39,220 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:26:41,221 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:26:42,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:44,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:26:44,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:26:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:26:47,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:52,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:57,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:26:58,264 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:26:59,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:26:59,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:26:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:27:02,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:07,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:12,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:14,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:27:14,559 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:27:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:27:17,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:22,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:27,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:28,265 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:27:29,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:27:29,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:27:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:27:33,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:38,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:43,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:44,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:27:44,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:27:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:27:48,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:52,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:27:52,993 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:27:52,994 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:27:52,995 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:27:53,267 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:27:54,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:55,268 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:27:58,266 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:27:59,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:27:59,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:27:59,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:27:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:28:04,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:09,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:14,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:28:14,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:28:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:28:14,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:19,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:24,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:28,267 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:28:29,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:28:29,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:28:29,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:28:30,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:35,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:40,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:44,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:28:44,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:28:44,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:28:45,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:50,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:55,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:28:58,268 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:28:59,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:28:59,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:28:59,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:29:00,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:05,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:06,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:29:06,806 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:29:06,806 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:29:06,808 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:29:07,314 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:29:09,315 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:29:10,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:14,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:29:14,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:29:14,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:29:16,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:21,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:26,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:28,269 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:29:29,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:29:29,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:29:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:29:31,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:36,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:41,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:44,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:29:44,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:29:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:29:47,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:52,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:57,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:29:58,270 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:29:59,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:29:59,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:29:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:30:03,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:08,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:13,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:14,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:30:14,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:30:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:30:18,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:21,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:30:21,009 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:30:21,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:30:21,011 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:30:21,362 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:30:23,363 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:30:24,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:28,271 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:30:29,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:29,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:30:29,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:30:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:30:34,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:39,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:44,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:30:44,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:30:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:30:45,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:50,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:55,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:30:58,272 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:30:59,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:30:59,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:30:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:31:01,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:06,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:11,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:14,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:31:14,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:31:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:31:16,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:21,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:26,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:28,273 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:31:29,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:31:29,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:31:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:31:31,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:35,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:31:35,082 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:31:35,082 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:31:35,083 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:31:35,409 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:31:37,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:37,410 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:31:42,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:44,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:31:44,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:31:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:31:47,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:52,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:57,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:31:58,274 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:31:59,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:31:59,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:31:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:32:02,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:07,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:12,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:14,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:32:14,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:32:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:32:18,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:23,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:28,275 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:32:29,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:29,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:32:29,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:32:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:32:34,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:39,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:44,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:32:44,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:32:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:32:44,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:49,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:32:49,623 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:32:49,624 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:32:49,626 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:32:50,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:50,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:32:50,463 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:32:51,464 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:32:53,465 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:32:55,466 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:32:55,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:32:58,276 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:32:59,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:32:59,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:32:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:33:00,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:05,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:10,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:14,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:33:14,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:33:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:33:15,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:20,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:25,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:28,277 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:33:29,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:33:29,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:33:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:33:30,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:35,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:40,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:44,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:33:44,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:33:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:33:46,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:51,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:56,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:33:58,278 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:33:59,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:33:59,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:33:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:34:02,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:07,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:34:07,663 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:34:07,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:34:07,664 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:34:08,510 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:34:08,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:09,511 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:34:13,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:14,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:34:14,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:34:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:34:18,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:23,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:28,279 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:34:29,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:29,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:34:29,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:34:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:34:34,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:39,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:44,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:34:44,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:34:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:34:45,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:50,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:55,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:34:58,280 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:34:59,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:34:59,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:34:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:35:00,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:05,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:10,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:14,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:35:14,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:35:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:35:16,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:21,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:35:21,324 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:35:21,325 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:35:21,326 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:35:21,555 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:35:22,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:23,557 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:35:27,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:28,281 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:35:29,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:35:29,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:35:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:35:32,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:37,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:42,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:44,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:35:44,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:35:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:35:48,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:53,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:58,282 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:35:59,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:35:59,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:35:59,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:35:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:36:04,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:09,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:14,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:36:14,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:36:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:36:15,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:20,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:25,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:28,283 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:36:29,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:36:29,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:36:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:36:31,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:35,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:36:35,210 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:36:35,210 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:36:35,212 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:36:35,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:36:37,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:37,604 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:36:42,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:44,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:36:44,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:36:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:36:47,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:52,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:57,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:36:58,284 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:36:59,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:36:59,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:36:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:37:03,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:08,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:13,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:14,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:37:14,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:37:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:37:18,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:23,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:28,285 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:37:29,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:29,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:37:29,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:37:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:37:34,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:39,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:44,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:37:44,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:37:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:37:45,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:49,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:37:49,078 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:37:49,078 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:37:49,079 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:37:49,647 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:37:49,647 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:37:51,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:56,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:37:58,286 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:37:59,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:37:59,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:37:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:38:01,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:06,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:11,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:14,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:38:14,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:38:14,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:38:17,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:22,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:27,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:28,287 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:38:29,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:38:29,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:38:29,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:38:32,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:37,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:42,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:44,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:38:44,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:38:44,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:38:47,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:52,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:57,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:38:58,288 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:38:59,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:38:59,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:38:59,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:39:02,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:39:02,943 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:39:02,944 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:39:02,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:02,945 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:39:03,693 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:39:03,693 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:39:07,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:12,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:14,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:39:14,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:39:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:39:18,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:23,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:28,289 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:39:29,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:29,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:39:29,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:39:29,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:39:34,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:39,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:44,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:39:44,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:39:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:39:45,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:50,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:55,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:39:58,290 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:39:59,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:39:59,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:39:59,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:40:01,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:06,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:11,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:14,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:40:14,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:40:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:40:16,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:17,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:40:17,220 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:40:17,220 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:40:17,221 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:40:17,740 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:40:18,741 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:40:22,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:27,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:28,290 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:40:29,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:40:29,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:40:29,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:40:32,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:37,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:42,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:44,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:40:44,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:40:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:40:48,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:53,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:58,291 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:40:59,292 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:40:59,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:40:59,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:40:59,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:41:04,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:09,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:14,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:41:14,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:41:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:41:15,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:20,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:25,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:28,292 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:41:29,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:41:29,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:41:29,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:41:31,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:41:31,244 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:41:31,244 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:41:31,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:31,246 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:41:31,788 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:41:32,789 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:41:36,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:41,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:44,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:41:44,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:41:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:41:46,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:51,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:56,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:41:58,293 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:41:59,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:41:59,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:41:59,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:42:02,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:07,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:12,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:14,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:42:14,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:42:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:42:18,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:23,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:28,294 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:42:29,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:29,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:42:29,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:42:29,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:42:34,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:39,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:44,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:42:44,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:42:44,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:42:44,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:45,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:42:45,128 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:42:45,128 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:42:45,130 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:42:45,834 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:42:46,835 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:42:50,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:55,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:42:58,295 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:42:59,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:42:59,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:42:59,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:43:00,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:05,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:10,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:14,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:43:14,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:43:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:43:15,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:20,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:25,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:28,296 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:43:29,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:43:29,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:43:29,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:43:31,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:36,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:41,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:44,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:43:44,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:43:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:43:47,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:52,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:57,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:43:58,297 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:43:59,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:43:59,098 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:43:59,098 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:43:59,099 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:43:59,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:43:59,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:43:59,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:43:59,881 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:44:00,882 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:44:02,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:07,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:12,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:14,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:44:14,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:44:14,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:44:17,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:22,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:27,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:28,298 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:44:29,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:44:29,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:44:29,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:44:32,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:37,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:42,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:44,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:44:44,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:44:44,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:44:48,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:53,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:58,300 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:44:59,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:44:59,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:44:59,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:44:59,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:45:04,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:09,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:13,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:45:13,323 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:45:13,323 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:45:13,325 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:45:13,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:45:13,927 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:45:14,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:45:14,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:45:14,589 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:45:14,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:14,927 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:45:16,929 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:45:17,930 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:45:18,931 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:45:20,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:25,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:28,300 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:45:29,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:45:29,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:45:29,580 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:45:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:35,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:40,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:44,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:45:44,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:45:44,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:45:46,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:51,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:56,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:45:58,301 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:45:59,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:45:59,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:45:59,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:46:02,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:07,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:12,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:14,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:46:14,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:46:14,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:46:17,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:22,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:27,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:28,302 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:46:29,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:46:29,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:46:29,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:46:31,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:46:31,257 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:46:31,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:46:31,259 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:46:31,973 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:46:32,974 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:46:33,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:38,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:43,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:44,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:46:44,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:46:44,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:46:48,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:53,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:58,303 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:46:59,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:46:59,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:46:59,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:46:59,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:47:04,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:09,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:14,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:47:14,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:47:14,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:47:15,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:20,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:25,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:28,304 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:47:29,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:47:29,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:47:29,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:47:30,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:35,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:40,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:44,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:47:44,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:47:44,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:47:45,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:47:45,193 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:47:45,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:47:45,195 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:47:46,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:47:46,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:47,020 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:47:51,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:56,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:47:58,304 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:47:59,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:47:59,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:47:59,581 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:48:01,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:06,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:11,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:14,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:48:14,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:48:14,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:48:17,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:22,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:27,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:28,306 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:48:29,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:48:29,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:48:29,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:48:32,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:37,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:42,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:44,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:48:44,581 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:48:44,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:48:48,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:53,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:58,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:48:58,306 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:48:59,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:48:59,455 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:48:59,455 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:48:59,457 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:48:59,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:48:59,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:48:59,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:49:00,067 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:49:01,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:49:03,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:08,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:13,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:14,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:49:14,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:49:14,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:49:18,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:23,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:28,309 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:49:29,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:29,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:49:29,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:49:29,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:49:34,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:39,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:44,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:49:44,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:49:44,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:49:44,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:49,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:54,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:49:58,310 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:49:59,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:49:59,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:49:59,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:50:00,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:05,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:10,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:13,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:50:13,552 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:50:13,553 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:50:13,554 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:50:14,112 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:50:14,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:50:14,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:50:14,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:50:15,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:50:15,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:20,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:25,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:28,311 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:50:29,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:50:29,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:50:29,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:50:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:35,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:40,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:44,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:50:44,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:50:44,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:50:46,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:51,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:56,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:50:58,312 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:50:59,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:50:59,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:50:59,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:51:01,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:06,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:11,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:14,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:51:14,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:51:14,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:51:16,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:21,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:26,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:27,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:51:27,488 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:51:27,488 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:51:27,490 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:51:28,157 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:51:28,313 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:51:29,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:51:29,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:51:29,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:51:29,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:51:32,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:37,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:42,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:44,575 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:51:44,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:51:44,583 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:51:47,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:52,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:57,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:51:58,315 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:51:59,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:51:59,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:51:59,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:52:02,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:07,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:12,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:14,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:52:14,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:52:14,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:52:18,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:23,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:28,315 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:52:29,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:29,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:52:29,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:52:29,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:52:34,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:39,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:41,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:52:41,240 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:52:41,241 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:52:41,242 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:52:42,205 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:52:43,206 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:52:44,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:52:44,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:52:44,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:52:45,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:50,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:55,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:52:58,316 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:52:59,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:52:59,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:52:59,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:53:01,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:06,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:11,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:14,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:53:14,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:53:14,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:53:16,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:21,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:26,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:28,317 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:53:29,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:53:29,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:53:29,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:53:32,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:37,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:42,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:44,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:53:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:53:44,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:53:47,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:52,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:55,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:53:55,035 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:53:55,036 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:53:55,037 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:53:55,250 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:53:57,251 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:53:58,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:53:58,318 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:53:59,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:53:59,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:53:59,618 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:54:03,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:08,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:13,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:14,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:54:14,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:54:14,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:54:19,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:24,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:28,319 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:54:29,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:54:29,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:54:29,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:54:30,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:35,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:40,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:44,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:54:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:54:44,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:54:46,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:51,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:56,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:54:58,320 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:54:59,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:54:59,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:54:59,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:55:01,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:06,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:08,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:55:08,915 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:55:08,915 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:55:08,916 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:55:09,295 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:55:11,297 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:55:11,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:14,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:55:14,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:55:14,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:55:17,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:22,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:27,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:28,321 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:55:29,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:55:29,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:55:29,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:55:32,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:37,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:42,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:44,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:55:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:55:44,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:55:48,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:53,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:58,322 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:55:59,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:55:59,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:55:59,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:55:59,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:56:04,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:09,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:14,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:56:14,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:56:14,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:56:15,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:20,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:22,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:56:22,879 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:56:22,879 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:56:22,880 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:56:23,340 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:56:25,342 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:56:25,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:28,323 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:56:29,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:56:29,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:56:29,619 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:56:31,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:36,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:41,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:56:44,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:56:44,620 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:56:46,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:51,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:56,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:56:58,324 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:56:59,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:56:59,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:56:59,620 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:57:02,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:07,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:12,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:14,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:57:14,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:57:14,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:57:17,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:22,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:27,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:28,325 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:57:29,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:57:29,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:57:29,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:57:32,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:37,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:57:37,376 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:57:37,377 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:57:37,378 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:57:37,389 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:57:37,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:37,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:57:39,391 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:57:41,392 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:57:43,393 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:57:43,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:57:44,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:57:44,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:57:48,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:53,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:58,326 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:57:59,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:57:59,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:57:59,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:57:59,620 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:58:04,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:09,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:14,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:58:14,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:58:14,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:58:14,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:19,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:24,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:28,328 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:58:29,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:58:29,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:58:29,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:58:30,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:35,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:40,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:44,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:58:44,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:58:44,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:58:45,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:50,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:55,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 23:58:55,517 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-12 23:58:55,517 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-12 23:58:55,519 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 23:58:56,438 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-12 23:58:56,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:58:57,439 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-12 23:58:58,327 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:58:59,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:58:59,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:58:59,622 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:59:01,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:06,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:11,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:14,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:59:14,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:59:14,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:59:16,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:21,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:26,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:28,330 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:59:29,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:59:29,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:59:29,622 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:59:31,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:36,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:41,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:44,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:59:44,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:59:44,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-12 23:59:46,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:51,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:56,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 23:59:58,330 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-12 23:59:59,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 23:59:59,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 23:59:59,622 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:00:02,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:07,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:09,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:00:09,300 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:00:09,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:00:09,302 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:00:09,484 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:00:11,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:00:13,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:14,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:00:14,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:00:14,622 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:00:18,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:23,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:28,330 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:00:29,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:29,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:00:29,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:00:29,622 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:00:34,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:39,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:44,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:00:44,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:00:44,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:00:44,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:49,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:54,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:00:58,331 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:00:59,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:00:59,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:00:59,656 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:01:00,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:05,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:10,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:14,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:01:14,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:01:14,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:01:15,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:20,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:23,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:01:23,301 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:01:23,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:01:23,302 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:01:23,531 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:01:25,532 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:01:26,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:28,332 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:01:29,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:01:29,656 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:01:29,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:01:31,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:36,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:41,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:44,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:01:44,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:01:44,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:01:47,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:52,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:57,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:01:58,334 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:01:59,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:01:59,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:01:59,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:02:02,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:07,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:12,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:14,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:02:14,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:02:14,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:02:18,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:23,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:28,335 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:02:29,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:29,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:02:29,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:02:29,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:02:34,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:37,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:02:37,306 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:02:37,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:02:37,308 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:02:37,578 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:02:39,580 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:02:40,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:44,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:02:44,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:02:44,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:02:45,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:50,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:55,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:02:58,336 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:02:59,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:02:59,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:02:59,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:03:00,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:05,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:10,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:14,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:03:14,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:03:14,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:03:16,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:21,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:26,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:28,336 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:03:29,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:03:29,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:03:29,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:03:31,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:36,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:41,873 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:44,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:03:44,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:03:44,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:03:47,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:51,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:03:51,246 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:03:51,247 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:03:51,248 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:03:51,625 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:03:53,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:53,626 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:03:58,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:03:58,337 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:03:59,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:03:59,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:03:59,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:04:03,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:08,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:13,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:14,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:04:14,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:04:14,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:04:18,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:23,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:28,338 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:04:29,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:29,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:04:29,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:04:29,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:04:34,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:39,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:44,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:04:44,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:04:44,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:04:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:50,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:55,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:04:58,339 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:04:59,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:04:59,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:04:59,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:05:01,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:05,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:05:05,437 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:05:05,438 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:05:05,439 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:05:05,674 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:05:07,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:07,676 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:05:12,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:14,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:05:14,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:05:14,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:05:17,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:22,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:27,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:28,340 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:05:29,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:05:29,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:05:29,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:05:32,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:37,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:42,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:44,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:05:44,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:05:44,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:05:48,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:53,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:58,341 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:05:59,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:05:59,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:05:59,659 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:05:59,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:06:04,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:09,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:14,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:06:14,660 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:06:14,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:06:14,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:19,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:06:19,780 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:06:19,780 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:06:19,781 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:06:20,724 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:06:20,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:21,725 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:06:25,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:28,342 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:06:29,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:06:29,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:06:29,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:06:30,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:35,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:40,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:44,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:06:44,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:06:44,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:06:46,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:51,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:56,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:06:58,343 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:06:59,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:06:59,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:06:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:07:01,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:06,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:11,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:14,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:07:14,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:07:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:07:16,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:21,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:26,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:28,346 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:07:29,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:07:29,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:07:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:07:31,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:33,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:07:33,785 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:07:33,785 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:07:33,786 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:07:34,771 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:07:35,771 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:07:37,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:42,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:44,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:07:44,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:07:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:07:47,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:52,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:57,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:07:58,345 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:07:59,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:07:59,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:07:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:08:02,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:07,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:12,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:14,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:08:14,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:08:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:08:18,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:23,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:28,347 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:08:29,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:29,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:08:29,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:08:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:08:34,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:39,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:44,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:08:44,661 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:08:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:08:45,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:47,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:08:47,606 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:08:47,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:08:47,608 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:08:47,815 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:08:49,816 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:08:51,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:56,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:08:58,347 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:08:59,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:08:59,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:08:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:09:01,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:06,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:11,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:14,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:09:14,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:09:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:09:16,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:21,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:26,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:28,348 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:09:29,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:09:29,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:09:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:09:31,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:36,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:41,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:44,661 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:09:44,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:09:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:09:46,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:51,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:56,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:09:58,349 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:09:59,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:09:59,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:09:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:10:01,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:10:01,321 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:10:01,321 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:10:01,322 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:10:01,861 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:10:01,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:10:02,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:03,862 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:10:05,864 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:10:07,865 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:10:08,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:13,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:14,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:10:14,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:10:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:10:18,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:23,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:28,350 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:10:29,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:29,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:10:29,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:10:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:10:34,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:39,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:44,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:10:44,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:10:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:10:44,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:49,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:54,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:10:58,351 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:10:59,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:10:59,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:10:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:11:00,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:05,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:10,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:14,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:11:14,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:11:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:11:16,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:19,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:11:19,173 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:11:19,174 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:11:19,175 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:11:19,910 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:11:19,911 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:11:22,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:27,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:28,352 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:11:29,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:11:29,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:11:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:11:32,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:37,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:42,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:44,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:11:44,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:11:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:11:47,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:52,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:57,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:11:58,353 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:11:59,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:11:59,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:11:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:12:03,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:08,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:13,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:14,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:12:14,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:12:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:12:18,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:23,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:28,354 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:12:29,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:29,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:12:29,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:12:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:12:33,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:12:33,315 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:12:33,316 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:12:33,317 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:12:33,956 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:12:33,956 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:12:34,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:39,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:44,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:44,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:12:44,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:12:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:12:49,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:54,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:12:58,355 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:12:59,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:12:59,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:12:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:12:59,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:04,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:09,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:14,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:13:14,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:13:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:13:14,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:19,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:24,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:28,356 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:13:29,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:13:29,663 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:13:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:13:29,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:34,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:39,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:44,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:13:44,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:13:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:13:45,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:47,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:13:47,587 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:13:47,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:13:47,589 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:13:48,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:13:50,007 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:13:51,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:56,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:13:58,357 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:13:59,663 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:13:59,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:13:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:14:01,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:06,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:11,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:14,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:14:14,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:14:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:14:17,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:22,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:27,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:28,358 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:14:29,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:14:29,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:14:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:14:33,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:38,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:43,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:44,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:14:44,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:14:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:14:48,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:53,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:58,360 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:14:59,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:14:59,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:14:59,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:14:59,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:15:01,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:15:01,719 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:15:01,719 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:15:01,721 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:15:02,053 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:15:04,055 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:15:04,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:09,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:14,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:15:14,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:15:14,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:15:14,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:19,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:24,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:28,360 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:15:29,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:15:29,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:15:29,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:15:29,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:34,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:39,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:44,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:15:44,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:15:44,704 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:15:45,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:50,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:55,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:15:58,362 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:15:59,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:15:59,664 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:15:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:16:01,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:06,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:11,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:14,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:16:14,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:16:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:16:15,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:16:15,522 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:16:15,523 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:16:15,524 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:16:16,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:16:16,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:16:17,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:22,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:27,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:28,362 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:16:29,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:16:29,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:16:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:16:32,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:37,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:42,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:44,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:16:44,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:16:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:16:48,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:53,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:58,363 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:16:59,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:16:59,664 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:16:59,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:16:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:17:04,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:09,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:14,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:17:14,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:17:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:17:14,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:19,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:24,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:28,364 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:17:29,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:17:29,257 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:17:29,257 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:17:29,259 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:17:29,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:17:29,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:17:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:17:30,147 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:17:30,147 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:17:30,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:35,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:40,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:44,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:17:44,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:17:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:17:46,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:51,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:56,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:17:58,365 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:17:59,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:17:59,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:17:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:18:01,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:06,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:11,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:14,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:18:14,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:18:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:18:16,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:21,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:26,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:28,366 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:18:29,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:18:29,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:18:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:18:32,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:37,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:42,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:43,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:18:43,172 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:18:43,173 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:18:43,174 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:18:43,192 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:18:44,193 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:18:44,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:18:44,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:18:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:18:48,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:53,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:58,367 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:18:59,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:18:59,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:18:59,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:18:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:19:04,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:09,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:14,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:19:14,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:19:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:19:14,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:19,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:24,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:28,368 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:19:29,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:19:29,665 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:19:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:19:30,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:35,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:40,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:44,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:19:44,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:19:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:19:46,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:51,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:56,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:19:57,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:19:57,012 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:19:57,012 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:19:57,014 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:19:57,239 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:19:58,240 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:19:58,369 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:19:59,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:19:59,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:19:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:20:02,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:07,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:12,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:14,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:20:14,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:20:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:20:17,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:22,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:27,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:28,370 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:20:29,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:20:29,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:20:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:20:33,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:38,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:43,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:44,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:20:44,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:20:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:20:48,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:53,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:58,371 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:20:59,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:20:59,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:20:59,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:20:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:21:04,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:09,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:10,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:21:10,969 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:21:10,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:21:10,971 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:21:11,286 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:21:12,287 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:21:14,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:21:14,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:21:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:21:15,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:20,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:25,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:28,372 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:21:29,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:21:29,666 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:21:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:21:30,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:35,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:40,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:44,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:21:44,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:21:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:21:46,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:51,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:56,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:21:58,373 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:21:59,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:21:59,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:21:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:22:01,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:06,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:11,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:14,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:22:14,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:22:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:22:17,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:22,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:25,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:22:25,417 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:22:25,417 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:22:25,418 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:22:25,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:22:26,336 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:22:26,337 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:22:28,338 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:22:28,374 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:22:28,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:29,666 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:22:29,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:22:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:22:30,339 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:22:33,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:38,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:43,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:44,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:22:44,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:22:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:22:49,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:54,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:22:58,375 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:22:59,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:22:59,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:22:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:22:59,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:04,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:09,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:14,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:23:14,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:23:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:23:14,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:19,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:24,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:28,376 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:23:29,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:23:29,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:23:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:23:30,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:35,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:40,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:43,626 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:23:43,628 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:23:43,628 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:23:43,629 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:23:44,386 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:23:44,387 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:23:44,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:23:44,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:23:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:23:46,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:51,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:56,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:23:58,377 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:23:59,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:23:59,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:23:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:24:01,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:06,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:11,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:14,667 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:24:14,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:24:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:24:16,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:21,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:26,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:28,378 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:24:29,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:24:29,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:24:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:24:31,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:36,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:41,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:44,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:24:44,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:24:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:24:47,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:52,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:57,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:24:57,443 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:24:57,444 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:24:57,445 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:24:58,379 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:24:58,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:24:58,434 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:24:59,434 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:24:59,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:24:59,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:24:59,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:25:03,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:08,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:13,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:14,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:25:14,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:25:14,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:25:18,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:23,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:28,380 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:25:29,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:29,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:25:29,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:25:29,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:25:34,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:39,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:44,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:25:44,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:25:44,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:25:44,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:49,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:54,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:25:58,381 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:25:59,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:25:59,668 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:25:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:26:00,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:05,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:10,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:26:11,371 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:26:11,372 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:26:11,373 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:26:11,479 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:26:13,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:26:14,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:26:14,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:26:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:26:16,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:21,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:26,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:28,383 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:26:29,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:26:29,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:26:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:26:31,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:36,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:41,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:44,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:26:44,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:26:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:26:47,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:52,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:57,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:26:58,384 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:26:59,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:26:59,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:26:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:27:02,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:07,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:12,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:14,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:27:14,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:27:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:27:17,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:22,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:25,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:27:25,190 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:27:25,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:27:25,192 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:27:25,524 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:27:27,526 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:27:28,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:28,384 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:27:29,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:27:29,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:27:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:27:33,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:38,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:43,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:44,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:27:44,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:27:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:27:49,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:54,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:27:58,385 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:27:59,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:27:59,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:27:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:27:59,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:04,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:09,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:14,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:28:14,669 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:28:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:28:15,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:20,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:25,885 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:28,386 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:28:29,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:28:29,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:28:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:28:31,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:36,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:38,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:28:38,868 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:28:38,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:28:38,869 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:28:39,570 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:28:41,571 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:28:41,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:44,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:28:44,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:28:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:28:46,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:51,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:56,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:28:58,387 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:28:59,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:28:59,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:28:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:29:02,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:07,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:12,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:14,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:29:14,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:29:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:29:17,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:22,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:27,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:28,388 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:29:29,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:29:29,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:29:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:29:33,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:38,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:43,924 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:44,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:29:44,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:29:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:29:49,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:52,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:29:52,743 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:29:52,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:29:52,744 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:29:53,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:29:55,619 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:29:55,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:29:58,390 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:29:59,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:29:59,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:29:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:30:00,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:05,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:10,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:14,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:30:14,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:30:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:30:15,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:20,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:25,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:28,390 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:30:29,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:30:29,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:30:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:30:31,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:36,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:41,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:44,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:30:44,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:30:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:30:47,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:52,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:57,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:30:58,392 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:30:59,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:30:59,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:30:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:31:02,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:07,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:31:07,283 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:31:07,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:31:07,285 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:31:07,667 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:31:08,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:09,668 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:31:13,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:14,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:31:14,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:31:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:31:18,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:23,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:28,392 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:31:29,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:29,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:31:29,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:31:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:31:34,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:39,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:44,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:31:44,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:31:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:31:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:50,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:55,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:31:58,393 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:31:59,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:31:59,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:31:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:32:00,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:05,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:10,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:14,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:32:14,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:32:14,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:32:16,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:21,135 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:32:21,137 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:32:21,137 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:32:21,139 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:32:21,717 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:32:22,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:23,718 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:32:27,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:28,394 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:32:29,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:32:29,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:32:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:32:32,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:37,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:42,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:44,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:32:44,672 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:32:44,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:32:47,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:52,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:57,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:32:58,395 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:32:59,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:32:59,672 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:32:59,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:33:03,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:08,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:13,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:14,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:33:14,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:33:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:33:19,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:24,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:28,396 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:33:29,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:33:29,676 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:33:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:33:29,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:34,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:35,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:33:35,045 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:33:35,046 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:33:35,047 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:33:35,764 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:33:37,765 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:33:40,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:44,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:33:44,676 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:33:44,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:33:45,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:50,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:55,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:33:58,397 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:33:59,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:33:59,676 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:33:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:34:00,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:05,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:10,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:14,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:34:14,676 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:34:14,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:34:16,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:21,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:26,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:28,398 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:34:29,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:34:29,676 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:34:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:34:31,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:36,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:41,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:44,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:34:44,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:34:44,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:34:47,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:49,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:34:49,064 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:34:49,064 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:34:49,066 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:34:49,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:34:49,811 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:34:51,813 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:34:53,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:53,814 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:34:55,815 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:34:58,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:34:58,398 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:34:59,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:34:59,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:34:59,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:35:03,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:08,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:13,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:14,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:35:14,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:35:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:35:19,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:24,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:28,399 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:35:29,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:35:29,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:35:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:35:29,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:34,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:39,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:44,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:35:44,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:35:44,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:35:44,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:49,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:54,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:35:58,400 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:35:59,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:35:59,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:35:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:36:00,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:05,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:06,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:36:06,918 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:36:06,919 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:36:06,920 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:36:07,861 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:36:07,861 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:36:10,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:14,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:36:14,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:36:14,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:36:16,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:21,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:26,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:28,401 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:36:29,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:36:29,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:36:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:36:31,897 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:36,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:41,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:44,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:36:44,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:36:44,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:36:46,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:51,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:56,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:36:58,403 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:36:59,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:36:59,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:36:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:37:02,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:07,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:12,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:14,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:37:14,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:37:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:37:17,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:20,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:37:20,758 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:37:20,759 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:37:20,760 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:37:20,908 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:37:21,909 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:37:23,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:28,403 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:37:29,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:29,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:37:29,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:37:29,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:37:34,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:39,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:44,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:37:44,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:37:44,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:37:45,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:50,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:55,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:37:58,404 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:37:59,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:37:59,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:37:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:38:00,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:05,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:10,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:14,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:38:14,678 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:38:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:38:16,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:21,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:26,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:28,405 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:38:29,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:38:29,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:38:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:38:31,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:34,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:38:34,714 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:38:34,715 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:38:34,716 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:38:35,002 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:38:36,002 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:38:37,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:42,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:44,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:38:44,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:38:44,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:38:47,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:52,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:57,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:38:58,406 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:38:59,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:38:59,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:38:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:39:02,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:07,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:12,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:14,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:39:14,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:39:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:39:18,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:23,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:28,407 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:39:29,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:29,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:39:29,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:39:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:39:34,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:39,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:44,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:39:44,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:39:44,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:39:44,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:49,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:39:49,145 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:39:49,145 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:39:49,146 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:39:50,051 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:39:50,052 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:39:50,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:55,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:39:58,408 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:39:59,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:39:59,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:39:59,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:40:00,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:05,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:10,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:14,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:40:14,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:40:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:40:16,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:21,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:26,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:28,409 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:40:29,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:40:29,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:40:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:40:31,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:36,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:41,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:44,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:40:44,679 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:40:44,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:40:46,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:51,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:56,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:40:58,410 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:40:59,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:40:59,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:40:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:41:01,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:02,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:41:02,900 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:41:02,900 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:41:02,902 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:41:03,098 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:41:04,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:41:07,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:12,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:14,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:41:14,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:41:14,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:41:18,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:23,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:28,411 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:41:29,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:29,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:41:29,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:41:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:41:34,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:39,887 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:44,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:41:44,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:41:44,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:41:44,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:49,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:54,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:41:58,412 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:41:59,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:41:59,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:41:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:42:00,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:05,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:10,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:14,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:42:14,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:42:14,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:42:16,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:16,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:42:16,912 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:42:16,912 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:42:16,913 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:42:17,146 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:42:18,147 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:42:21,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:26,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:28,412 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:42:29,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:42:29,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:42:29,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:42:32,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:37,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:42,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:44,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:42:44,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:42:44,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:42:48,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:53,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:58,414 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:42:59,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:42:59,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:42:59,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:42:59,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:43:04,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:09,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:14,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:43:14,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:43:14,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:43:14,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:19,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:24,912 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:28,415 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:43:29,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:43:29,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:43:29,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:43:29,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:30,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:43:30,621 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:43:30,621 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:43:30,622 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:43:31,193 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:43:32,193 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:43:35,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:40,662 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:44,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:43:44,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:43:44,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:43:45,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:50,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:55,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:43:58,415 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:43:59,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:43:59,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:43:59,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:44:00,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:05,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:10,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:14,680 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:44:14,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:44:14,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:44:16,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:21,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:26,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:28,416 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:44:29,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:44:29,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:44:29,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:44:31,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:36,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:41,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:44,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:44:44,300 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:44:44,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:44:44,302 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:44:44,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:44:44,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:44:44,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:44:45,241 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:44:46,242 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:44:46,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:51,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:56,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:44:58,417 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:44:59,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:44:59,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:44:59,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:45:02,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:07,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:12,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:14,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:45:14,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:45:14,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:45:17,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:22,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:27,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:28,419 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:45:29,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:45:29,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:45:29,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:45:33,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:38,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:43,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:44,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:45:44,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:45:44,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:45:49,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:54,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:45:58,128 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:45:58,130 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:45:58,130 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:45:58,131 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:45:58,288 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:45:58,419 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:45:59,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:45:59,681 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:45:59,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:45:59,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:00,289 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:46:04,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:09,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:14,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:46:14,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:46:14,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:46:14,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:19,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:24,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:28,420 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:46:29,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:46:29,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:46:29,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:46:29,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:34,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:39,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:44,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:46:44,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:46:44,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:46:45,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:50,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:55,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:46:58,422 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:46:59,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:46:59,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:46:59,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:47:00,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:05,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:10,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:11,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:47:11,958 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:47:11,958 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:47:11,960 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:47:12,335 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:47:12,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:47:14,336 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:47:14,940 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:47:14,947 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:47:14,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:47:16,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:16,337 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:47:18,339 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:47:21,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:26,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:28,422 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:47:29,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:47:29,902 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:47:29,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:47:32,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:37,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:42,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:44,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:47:44,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:47:44,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:47:48,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:53,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:58,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:47:58,423 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:47:59,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:47:59,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:47:59,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:48:03,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:08,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:13,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:14,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:48:14,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:48:14,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:48:18,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:23,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:28,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:28,424 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:48:29,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:48:29,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:48:29,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:48:30,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:48:30,427 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:48:30,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:48:30,428 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:48:31,389 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:48:32,390 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:48:33,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:38,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:43,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:44,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:48:44,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:48:44,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:48:49,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:54,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:58,425 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:48:59,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:48:59,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:48:59,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:48:59,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:49:05,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:10,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:14,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:49:14,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:49:14,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:49:15,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:20,138 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:25,139 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:28,426 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:49:29,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:49:29,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:49:29,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:49:30,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:35,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:40,175 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:44,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:49:44,439 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:49:44,439 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:49:44,441 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:49:44,441 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:49:44,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:49:44,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:49:44,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:49:46,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:46,442 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:49:51,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:56,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:49:58,427 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:49:59,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:49:59,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:49:59,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:50:01,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:06,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:11,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:14,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:50:14,903 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:50:14,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:50:16,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:21,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:26,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:28,429 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:50:29,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:50:29,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:50:29,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:50:32,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:37,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:42,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:44,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:50:44,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:50:44,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:50:47,180 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:52,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:57,181 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:50:58,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:50:58,192 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:50:58,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:50:58,193 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:50:58,429 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:50:58,488 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:50:59,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:50:59,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:50:59,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:51:00,490 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:51:02,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:07,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:12,187 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:14,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:51:14,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:51:14,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:51:18,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:23,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:28,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:28,430 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:51:29,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:51:29,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:51:29,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:51:33,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:38,172 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:43,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:44,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:51:44,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:51:44,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:51:49,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:54,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:58,431 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:51:59,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:51:59,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:51:59,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:51:59,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:52:05,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:10,130 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:12,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:52:12,236 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:52:12,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:52:12,238 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:52:12,535 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:52:14,536 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:52:14,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:52:14,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:52:14,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:52:16,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:21,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:26,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:28,433 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:52:29,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:52:29,904 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:52:29,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:52:31,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:36,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:41,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:44,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:52:44,905 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:52:44,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:52:47,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:52,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:57,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:52:58,432 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:52:59,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:52:59,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:52:59,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:53:02,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:07,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:12,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:14,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:53:14,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:53:14,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:53:18,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:23,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:26,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:53:26,332 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:53:26,332 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:53:26,335 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:53:26,579 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:53:28,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:28,433 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:53:28,580 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:53:29,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:53:29,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:53:29,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:53:34,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:39,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:44,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:44,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:53:44,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:53:44,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:53:49,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:54,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:58,435 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:53:59,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:53:59,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:53:59,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:53:59,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:54:05,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:10,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:14,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:54:14,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:54:14,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:54:16,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:21,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:26,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:28,435 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:54:29,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:54:29,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:54:29,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:54:31,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:36,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:40,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:54:40,341 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:54:40,341 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:54:40,342 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:54:40,623 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:54:41,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:42,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:54:44,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:54:44,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:54:44,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:54:47,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:52,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:57,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:54:58,436 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:54:59,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:54:59,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:54:59,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:55:03,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:08,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:13,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:14,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:55:14,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:55:14,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:55:18,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:23,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:28,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:28,437 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:55:29,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:55:29,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:55:29,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:55:33,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:38,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:43,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:44,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:55:44,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:55:44,969 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:55:49,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:54,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:55,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:55:55,051 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:55:55,051 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:55:55,053 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:55:55,675 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:55:56,676 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:55:58,438 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:55:59,439 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:55:59,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:55:59,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:55:59,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:56:05,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:10,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:14,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:56:14,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:56:14,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:56:16,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:21,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:26,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:28,439 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:56:29,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:56:29,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:56:29,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:56:31,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:36,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:41,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:44,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:56:44,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:56:44,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:56:46,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:51,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:56,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:56:58,440 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:56:59,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:56:59,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:56:59,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:57:01,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:06,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:09,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:57:09,327 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:57:09,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:57:09,329 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:57:09,730 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:57:10,731 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:57:11,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:14,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:57:14,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:57:14,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:57:17,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:22,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:27,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:28,440 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:57:29,969 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:57:29,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:57:29,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:57:32,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:37,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:42,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:44,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:57:44,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:57:44,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:57:48,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:53,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:58,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:57:58,441 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:57:59,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:57:59,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:57:59,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:58:04,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:09,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:14,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:14,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:58:14,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:58:14,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:58:19,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:23,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:58:23,326 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:58:23,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:58:23,328 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:58:23,777 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:58:24,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:24,777 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:58:28,442 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:58:29,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:29,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:58:29,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:58:29,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:58:35,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:40,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:44,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:58:44,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:58:44,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:58:45,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:50,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:55,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:58:58,443 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:58:59,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:58:59,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:58:59,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:59:01,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:06,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:11,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:14,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:59:14,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:59:14,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:59:16,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:21,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:26,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:28,445 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:59:29,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:59:29,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:59:29,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:59:33,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:37,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:59:37,354 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 00:59:37,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 00:59:37,356 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 00:59:37,822 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 00:59:37,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 00:59:38,823 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:59:38,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:40,825 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:59:42,826 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 00:59:44,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:44,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:59:44,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:59:44,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 00:59:50,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:55,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 00:59:58,446 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 00:59:59,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 00:59:59,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 00:59:59,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:00:01,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:06,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:11,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:14,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:00:14,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:00:14,971 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:00:16,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:21,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:26,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:28,447 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:00:29,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:00:29,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:00:29,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:00:31,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:36,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:41,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:44,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:00:44,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:00:44,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:00:46,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:51,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:55,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:00:55,878 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:00:55,879 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:00:55,880 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:00:56,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:00:56,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:00:56,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:00:58,447 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:00:59,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:00:59,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:00:59,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:01:02,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:07,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:12,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:14,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:01:14,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:01:14,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:01:17,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:22,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:27,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:28,448 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:01:29,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:01:29,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:01:29,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:01:33,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:38,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:43,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:44,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:01:44,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:01:44,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:01:48,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:53,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:58,255 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:01:58,449 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:01:59,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:01:59,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:01:59,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:02:04,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:09,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:09,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:02:09,857 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:02:09,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:02:09,859 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:02:09,915 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:02:10,915 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:02:14,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:14,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:02:14,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:02:14,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:02:20,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:25,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:28,450 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:02:30,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:02:30,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:02:30,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:02:31,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:36,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:41,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:45,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:02:45,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:02:45,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:02:46,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:51,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:56,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:02:58,451 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:03:00,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:03:00,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:03:00,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:03:02,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:07,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:12,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:15,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:03:15,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:03:15,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:03:17,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:22,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:23,675 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:03:23,677 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:03:23,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:03:23,678 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:03:23,959 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:03:24,959 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:03:27,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:28,452 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:03:30,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:03:30,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:03:30,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:03:33,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:38,251 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:43,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:45,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:03:45,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:03:45,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:03:49,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:54,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:03:58,453 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:03:59,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:00,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:04:00,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:04:00,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:04:05,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:10,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:15,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:04:15,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:04:15,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:04:16,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:21,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:26,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:28,454 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:04:30,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:04:30,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:04:30,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:04:31,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:36,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:38,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:04:38,128 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:04:38,128 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:04:38,129 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:04:39,005 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:04:40,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:04:42,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:45,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:04:45,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:04:45,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:04:47,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:52,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:57,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:04:58,455 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:05:00,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:05:00,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:05:00,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:05:03,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:08,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:13,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:15,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:05:15,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:05:15,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:05:18,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:23,240 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:28,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:28,458 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:05:30,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:05:30,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:05:30,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:05:33,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:38,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:43,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:45,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:05:45,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:05:45,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:05:49,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:52,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:05:52,191 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:05:52,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:05:52,192 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:05:53,050 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:05:54,051 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:05:55,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:05:58,459 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:06:00,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:06:00,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:06:00,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:06:00,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:05,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:10,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:15,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:06:15,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:06:15,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:06:16,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:21,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:26,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:28,460 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:06:30,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:06:30,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:06:30,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:06:31,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:36,253 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:41,254 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:45,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:06:45,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:06:45,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:06:46,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:51,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:56,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:06:58,461 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:07:00,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:07:00,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:07:00,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:07:02,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:06,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:07:06,131 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:07:06,131 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:07:06,132 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:07:07,094 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:07:08,095 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:07:08,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:13,174 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:15,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:07:15,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:07:15,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:07:18,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:23,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:28,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:28,462 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:07:30,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:07:30,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:07:30,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:07:34,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:39,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:44,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:45,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:07:45,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:07:45,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:07:49,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:54,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:07:58,463 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:07:59,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:00,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:08:00,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:08:00,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:08:05,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:10,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:15,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:08:15,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:08:15,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:08:16,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:20,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:08:20,054 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:08:20,054 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:08:20,055 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:08:20,140 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:08:22,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:22,142 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:08:27,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:28,464 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:08:30,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:08:30,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:08:30,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:08:32,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:37,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:42,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:45,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:08:45,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:08:45,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:08:48,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:53,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:58,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:08:58,465 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:09:00,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:09:00,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:09:00,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:09:04,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:09,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:14,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:15,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:09:15,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:09:15,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:09:20,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:25,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:28,466 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:09:30,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:09:30,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:09:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:09:30,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:34,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:09:34,185 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:09:34,185 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:09:34,186 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:09:35,187 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:09:36,188 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:09:36,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:41,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:45,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:09:45,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:09:45,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:09:47,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:52,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:57,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:09:58,467 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:10:00,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:10:00,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:10:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:10:02,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:07,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:12,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:15,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:10:15,024 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:10:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:10:17,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:22,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:27,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:28,467 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:10:30,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:10:30,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:10:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:10:33,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:38,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:43,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:45,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:10:45,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:10:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:10:48,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:10:48,323 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:10:48,323 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:10:48,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:48,324 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:10:49,232 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:10:50,232 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:10:53,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:58,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:10:58,468 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:11:00,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:11:00,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:11:00,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:11:04,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:09,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:14,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:15,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:11:15,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:11:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:11:20,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:25,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:28,469 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:11:30,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:11:30,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:11:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:11:30,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:35,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:40,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:45,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:11:45,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:11:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:11:45,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:50,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:55,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:11:58,470 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:12:00,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:12:00,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:12:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:12:01,224 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:02,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:12:02,496 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:12:02,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:12:02,497 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:12:03,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:12:03,277 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:12:04,278 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:12:06,279 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:12:06,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:08,281 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:12:11,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:15,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:12:15,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:12:15,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:12:17,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:22,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:27,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:28,471 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:12:30,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:12:30,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:12:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:12:33,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:38,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:43,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:45,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:12:45,025 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:12:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:12:48,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:53,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:58,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:12:58,472 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:13:00,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:13:00,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:13:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:13:03,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:08,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:13,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:15,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:13:15,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:13:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:13:18,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:21,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:13:21,423 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:13:21,423 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:13:21,424 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:13:22,328 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:13:23,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:24,329 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:13:28,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:28,473 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:13:30,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:13:30,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:13:30,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:13:34,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:39,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:44,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:45,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:13:45,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:13:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:13:50,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:55,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:13:58,474 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:14:00,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:14:00,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:14:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:14:01,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:06,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:11,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:15,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:14:15,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:14:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:14:16,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:21,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:26,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:28,475 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:14:30,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:14:30,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:14:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:14:32,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:35,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:14:35,606 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:14:35,606 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:14:35,608 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:14:36,372 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:14:37,649 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:38,373 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:14:42,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:45,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:14:45,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:14:45,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:14:48,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:53,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:58,202 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:14:58,476 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:15:00,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:15:00,026 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:15:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:15:03,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:08,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:13,243 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:15,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:15:15,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:15:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:15:19,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:24,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:28,479 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:15:29,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:30,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:15:30,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:15:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:15:35,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:40,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:45,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:15:45,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:15:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:15:45,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:49,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:15:49,606 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:15:49,606 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:15:49,607 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:15:50,417 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:15:50,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:52,418 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:15:55,646 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:15:58,478 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:16:00,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:16:00,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:16:00,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:16:01,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:06,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:11,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:15,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:16:15,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:16:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:16:16,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:21,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:26,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:28,480 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:16:30,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:16:30,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:16:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:16:31,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:36,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:41,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:45,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:16:45,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:16:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:16:47,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:52,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:57,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:16:58,480 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:17:00,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:17:00,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:17:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:17:02,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:03,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:17:03,811 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:17:03,812 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:17:03,813 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:17:04,462 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:17:06,463 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:17:07,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:12,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:15,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:17:15,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:17:15,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:17:18,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:23,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:28,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:28,481 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:17:30,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:17:30,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:17:30,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:17:33,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:38,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:43,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:45,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:17:45,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:17:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:17:49,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:54,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:17:58,482 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:17:59,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:00,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:18:00,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:18:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:18:05,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:10,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:15,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:18:15,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:18:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:18:15,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:17,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:18:17,671 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:18:17,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:18:17,672 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:18:18,507 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:18:20,508 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:18:20,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:25,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:28,483 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:18:30,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:18:30,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:18:30,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:18:31,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:36,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:41,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:45,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:18:45,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:18:45,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:18:47,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:52,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:57,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:18:58,484 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:19:00,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:19:00,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:19:00,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:19:03,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:08,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:13,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:15,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:19:15,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:19:15,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:19:18,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:23,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:28,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:28,485 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:19:30,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:19:30,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:19:30,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:19:31,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:19:31,657 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:19:31,657 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:19:31,659 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:19:32,552 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:19:32,552 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:19:33,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:38,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:43,699 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:45,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:19:45,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:19:45,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:19:49,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:54,210 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:19:58,486 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:19:59,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:00,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:20:00,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:20:00,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:20:05,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:10,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:15,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:20:15,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:20:15,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:20:16,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:21,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:26,218 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:28,487 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:20:30,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:20:30,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:20:30,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:20:31,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:36,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:41,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:45,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:20:45,029 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:20:45,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:20:46,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:20:46,248 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:20:46,248 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:20:46,250 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:20:46,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:46,599 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:20:48,601 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:20:51,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:56,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:20:58,488 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:21:00,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:21:00,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:21:00,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:21:01,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:06,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:11,312 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:15,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:21:15,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:21:15,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:21:16,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:21,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:26,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:28,489 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:21:30,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:21:30,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:21:30,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:21:32,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:37,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:42,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:45,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:21:45,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:21:45,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:21:47,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:52,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:57,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:21:58,490 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:22:00,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:22:00,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:22:00,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:22:00,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:22:00,766 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:22:00,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:22:00,768 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:22:01,645 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:22:02,646 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:22:02,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:07,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:12,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:15,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:22:15,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:22:15,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:22:18,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:23,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:28,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:28,491 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:22:30,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:22:30,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:22:30,090 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:22:33,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:38,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:43,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:45,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:22:45,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:22:45,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:22:49,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:54,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:22:58,492 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:22:59,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:00,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:23:00,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:23:00,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:23:05,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:10,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:14,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:23:14,733 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:23:14,733 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:23:14,735 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:23:15,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:23:15,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:23:15,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:23:15,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:23:16,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:16,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:23:21,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:26,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:28,492 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:23:30,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:23:30,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:23:30,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:23:32,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:37,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:42,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:45,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:23:45,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:23:45,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:23:47,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:52,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:57,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:23:58,493 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:24:00,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:24:00,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:24:00,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:24:02,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:07,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:12,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:15,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:24:15,091 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:24:15,132 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:24:18,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:23,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:28,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:28,494 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:24:28,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:24:28,765 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:24:28,765 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:24:28,767 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:24:29,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:24:29,737 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:24:30,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:24:30,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:24:30,408 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:24:30,738 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:24:32,739 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:24:33,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:34,740 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:24:38,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:43,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:45,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:24:45,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:24:45,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:24:49,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:54,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:24:58,495 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:24:59,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:00,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:25:00,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:25:00,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:25:05,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:10,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:15,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:25:15,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:25:15,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:25:16,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:21,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:26,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:28,496 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:25:30,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:25:30,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:25:30,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:25:32,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:37,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:42,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:45,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:25:45,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:25:45,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:25:46,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:25:46,879 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:25:46,880 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:25:46,881 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:25:47,785 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:25:47,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:48,785 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:25:52,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:57,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:25:58,497 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:26:00,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:26:00,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:26:00,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:26:03,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:08,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:13,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:15,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:26:15,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:26:15,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:26:18,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:23,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:28,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:28,498 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:26:30,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:26:30,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:26:30,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:26:34,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:39,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:44,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:45,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:26:45,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:26:45,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:26:49,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:54,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:26:58,499 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:26:59,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:00,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:27:00,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:27:00,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:27:00,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:27:00,821 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:27:00,821 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:27:00,823 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:27:00,830 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:27:02,832 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:27:04,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:09,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:14,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:15,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:27:15,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:27:15,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:27:20,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:25,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:28,500 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:27:30,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:27:30,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:27:30,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:27:31,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:36,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:41,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:45,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:27:45,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:27:45,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:27:46,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:51,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:56,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:27:58,501 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:28:00,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:28:00,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:28:00,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:28:02,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:07,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:12,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:14,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:28:14,933 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:28:14,934 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:28:14,936 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:28:15,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:28:15,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:28:15,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:28:15,877 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:28:16,878 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:28:18,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:23,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:28,299 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:28,502 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:28:30,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:28:30,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:28:30,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:28:33,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:38,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:43,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:45,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:28:45,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:28:45,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:28:48,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:53,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:58,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:28:58,503 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:29:00,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:29:00,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:29:00,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:29:04,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:09,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:14,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:15,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:29:15,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:29:15,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:29:19,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:24,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:28,504 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:29:29,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:29:29,398 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:29:29,399 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:29:29,400 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:29:29,926 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:29:30,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:30,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:29:30,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:29:30,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:29:30,927 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:29:35,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:40,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:45,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:29:45,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:29:45,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:29:45,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:50,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:55,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:29:58,504 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:30:00,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:30:00,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:30:00,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:30:00,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:05,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:10,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:15,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:30:15,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:30:15,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:30:16,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:21,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:26,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:28,505 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:30:30,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:30:30,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:30:30,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:30:32,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:37,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:42,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:43,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:30:43,594 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:30:43,594 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:30:43,595 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:30:43,973 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:30:44,973 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:30:45,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:30:45,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:30:45,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:30:47,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:52,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:57,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:30:58,506 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:31:00,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:31:00,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:31:00,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:31:03,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:08,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:13,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:15,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:31:15,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:31:15,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:31:19,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:24,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:28,507 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:31:29,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:30,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:31:30,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:31:30,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:31:35,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:40,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:45,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:31:45,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:31:45,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:31:45,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:50,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:55,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:31:57,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:31:57,615 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:31:57,615 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:31:57,617 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:31:58,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:31:58,508 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:31:59,020 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:32:00,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:32:00,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:32:00,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:32:00,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:05,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:10,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:15,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:32:15,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:32:15,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:32:16,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:21,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:26,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:28,509 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:32:30,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:32:30,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:32:30,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:32:32,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:37,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:42,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:45,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:32:45,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:32:45,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:32:48,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:53,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:58,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:32:58,510 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:33:00,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:33:00,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:33:00,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:33:03,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:08,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:11,692 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:33:11,694 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:33:11,695 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:33:11,696 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:33:12,064 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:33:13,065 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:33:13,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:15,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:33:15,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:33:15,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:33:19,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:24,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:28,511 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:33:29,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:30,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:33:30,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:33:30,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:33:35,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:40,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:45,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:33:45,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:33:45,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:33:46,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:51,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:56,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:33:58,512 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:34:00,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:34:00,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:34:00,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:34:01,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:06,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:11,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:15,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:34:15,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:34:15,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:34:17,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:22,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:25,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:34:25,734 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:34:25,734 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:34:25,736 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:34:26,112 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:34:27,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:34:27,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:28,512 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:34:30,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:34:30,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:34:30,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:34:33,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:38,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:43,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:45,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:34:45,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:34:45,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:34:49,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:54,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:34:58,513 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:34:59,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:00,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:35:00,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:35:00,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:35:05,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:10,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:15,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:35:15,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:35:15,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:35:16,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:21,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:26,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:28,514 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:35:30,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:35:30,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:35:30,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:35:32,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:37,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:39,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:35:39,733 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:35:39,734 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:35:39,735 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:35:40,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:35:41,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:35:42,735 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:45,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:35:45,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:35:45,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:35:48,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:53,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:58,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:35:58,515 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:36:00,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:36:00,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:36:00,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:36:04,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:09,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:14,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:15,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:36:15,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:36:15,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:36:19,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:24,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:28,516 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:36:29,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:30,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:36:30,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:36:30,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:36:35,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:40,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:45,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:36:45,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:36:45,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:36:45,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:50,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:53,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:36:53,540 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:36:53,540 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:36:53,542 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:36:54,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:36:54,204 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:36:55,204 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:36:55,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:36:57,206 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:36:58,517 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:36:59,207 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:37:00,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:37:00,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:37:00,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:37:01,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:06,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:11,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:15,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:37:15,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:37:15,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:37:16,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:21,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:26,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:28,518 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:37:30,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:37:30,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:37:30,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:37:32,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:37,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:42,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:45,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:37:45,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:37:45,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:37:47,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:52,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:57,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:37:58,520 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:38:00,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:38:00,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:38:00,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:38:03,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:08,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:12,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:38:12,464 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:38:12,465 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:38:12,466 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:38:13,271 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:38:13,271 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:38:13,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:15,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:38:15,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:38:15,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:38:19,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:24,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:28,520 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:38:29,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:30,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:38:30,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:38:30,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:38:35,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:40,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:45,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:38:45,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:38:45,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:38:45,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:50,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:55,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:38:58,521 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:39:00,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:39:00,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:39:00,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:39:01,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:06,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:11,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:15,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:39:15,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:39:15,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:39:17,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:22,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:26,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:39:26,404 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:39:26,404 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:39:26,406 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:39:27,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:39:27,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:39:27,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:28,521 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:39:30,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:39:30,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:39:30,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:39:33,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:38,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:43,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:45,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:39:45,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:39:45,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:39:48,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:53,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:58,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:39:58,522 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:40:00,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:40:00,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:40:00,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:40:04,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:09,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:14,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:15,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:40:15,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:40:15,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:40:20,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:25,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:28,523 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:40:30,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:40:30,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:40:30,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:40:30,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:35,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:40,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:40,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:40:40,373 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:40:40,374 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:40:40,375 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:40:40,390 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:40:41,391 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:40:45,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:40:45,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:40:45,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:40:46,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:51,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:56,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:40:58,524 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:41:00,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:41:00,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:41:00,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:41:02,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:07,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:12,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:15,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:41:15,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:41:15,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:41:17,334 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:22,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:27,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:28,525 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:41:30,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:41:30,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:41:30,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:41:32,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:37,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:42,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:45,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:41:45,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:41:45,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:41:48,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:53,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:41:54,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:41:54,537 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:41:54,538 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:41:54,539 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:41:55,437 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:41:55,437 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:41:58,526 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:41:58,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:00,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:42:00,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:42:00,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:42:04,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:09,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:14,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:15,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:42:15,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:42:15,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:42:20,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:25,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:28,527 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:42:30,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:42:30,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:42:30,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:42:31,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:36,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:41,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:45,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:42:45,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:42:45,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:42:47,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:52,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:57,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:42:58,528 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:43:00,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:43:00,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:43:00,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:43:03,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:08,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:08,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:43:08,522 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:43:08,523 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:43:08,524 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:43:09,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:43:10,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:43:13,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:15,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:43:15,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:43:15,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:43:19,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:24,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:28,529 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:43:29,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:30,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:43:30,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:43:30,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:43:35,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:40,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:45,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:43:45,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:43:45,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:43:46,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:51,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:56,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:43:58,530 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:44:00,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:44:00,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:44:00,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:44:01,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:06,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:11,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:15,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:44:15,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:44:15,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:44:17,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:22,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:22,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:44:22,585 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:44:22,586 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:44:22,587 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:44:23,525 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:44:24,526 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:44:27,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:28,531 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:44:30,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:44:30,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:44:30,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:44:33,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:38,310 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:43,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:45,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:44:45,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:44:45,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:44:48,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:53,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:58,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:44:58,532 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:45:00,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:45:00,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:45:00,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:45:03,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:08,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:13,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:15,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:45:15,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:45:15,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:45:19,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:24,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:28,533 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:45:29,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:30,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:45:30,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:45:30,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:45:35,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:37,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:45:37,033 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:45:37,033 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:45:37,034 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:45:37,573 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:45:38,573 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:45:41,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:45,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:45:45,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:45:45,125 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:45:46,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:51,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:56,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:45:58,535 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:46:00,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:46:00,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:46:00,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:46:01,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:06,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:11,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:15,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:46:15,125 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:46:15,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:46:17,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:22,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:27,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:28,535 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:46:30,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:46:30,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:46:30,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:46:32,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:37,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:42,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:45,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:46:45,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:46:45,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:46:48,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:51,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:46:51,392 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:46:51,392 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:46:51,395 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:46:51,619 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:46:52,620 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:46:53,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:58,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:46:58,535 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:47:00,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:47:00,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:47:00,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:47:03,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:08,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:13,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:15,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:47:15,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:47:15,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:47:19,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:24,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:28,537 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:47:29,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:30,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:47:30,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:47:30,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:47:35,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:40,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:45,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:47:45,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:47:45,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:47:46,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:51,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:56,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:47:58,537 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:48:00,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:48:00,126 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:48:00,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:48:02,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:05,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:48:05,328 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:48:05,328 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:48:05,329 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:48:05,665 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:48:06,665 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:48:07,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:12,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:15,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:48:15,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:48:15,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:48:17,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:22,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:27,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:28,538 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:48:30,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:48:30,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:48:30,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:48:33,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:38,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:43,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:45,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:48:45,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:48:45,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:48:48,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:53,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:58,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:48:58,539 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:49:00,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:49:00,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:49:00,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:49:03,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:08,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:13,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:15,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:49:15,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:49:15,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:49:19,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:49:19,370 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:49:19,371 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:49:19,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:19,372 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:49:19,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:49:19,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:49:20,709 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:49:22,710 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:49:24,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:24,711 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:49:28,540 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:49:29,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:30,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:49:30,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:49:30,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:49:35,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:40,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:45,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:49:45,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:49:45,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:49:45,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:50,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:55,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:49:58,541 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:50:00,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:50:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:50:00,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:50:01,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:06,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:11,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:15,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:50:15,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:50:15,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:50:16,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:21,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:26,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:28,542 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:50:30,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:50:30,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:50:30,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:50:32,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:37,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:50:37,365 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:50:37,365 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:50:37,366 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:50:37,755 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:50:38,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:38,755 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:50:43,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:45,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:50:45,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:50:45,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:50:48,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:53,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:58,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:50:58,542 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:51:00,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:51:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:51:00,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:51:04,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:09,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:14,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:15,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:51:15,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:51:15,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:51:20,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:25,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:28,543 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:51:30,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:51:30,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:51:30,151 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:51:30,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:35,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:40,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:45,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:51:45,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:51:45,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:51:45,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:50,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:51,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:51:51,253 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:51:51,254 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:51:51,255 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:51:51,799 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:51:52,800 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:51:56,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:51:58,544 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:52:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:52:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:52:00,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:52:01,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:06,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:11,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:15,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:52:15,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:52:15,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:52:17,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:22,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:27,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:28,545 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:52:30,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:52:30,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:52:30,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:52:32,402 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:37,403 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:42,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:45,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:52:45,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:52:45,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:52:48,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:53,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:58,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:52:58,546 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:53:00,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:53:00,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:53:00,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:53:04,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:05,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:53:05,283 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:53:05,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:53:05,285 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:53:05,844 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:53:06,845 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:53:10,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:15,151 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:53:15,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:53:15,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:53:15,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:20,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:25,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:28,547 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:53:30,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:53:30,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:53:30,152 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:53:31,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:36,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:41,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:45,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:53:45,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:53:45,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:53:46,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:51,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:56,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:53:58,548 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:54:00,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:54:00,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:54:00,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:54:02,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:07,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:12,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:15,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:54:15,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:54:15,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:54:17,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:19,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:54:19,841 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:54:19,841 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:54:19,843 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:54:19,894 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:54:20,895 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:54:22,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:27,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:28,549 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:54:30,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:54:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:54:30,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:54:33,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:38,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:43,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:45,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:54:45,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:54:45,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:54:49,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:54,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:54:58,550 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:54:59,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:00,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:55:00,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:55:00,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:55:05,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:10,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:15,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:55:15,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:55:15,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:55:15,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:20,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:25,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:28,551 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:55:30,152 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:55:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:55:30,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:55:31,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:33,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:55:33,704 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:55:33,704 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:55:33,705 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:55:33,941 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:55:34,942 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:55:36,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:41,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:45,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:55:45,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:55:45,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:55:47,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:52,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:57,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:55:58,552 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:56:00,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:56:00,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:56:00,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:56:02,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:07,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:12,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:15,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:56:15,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:56:15,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:56:18,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:23,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:28,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:28,553 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:56:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:56:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:56:30,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:56:34,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:39,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:44,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:45,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:56:45,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:56:45,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:56:47,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:56:47,712 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:56:47,713 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:56:47,714 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:56:47,987 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:56:48,988 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:56:49,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:54,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:56:58,554 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:57:00,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:57:00,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:57:00,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:57:00,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:05,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:10,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:15,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:57:15,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:57:15,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:57:16,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:21,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:26,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:28,555 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:57:30,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:57:30,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:57:30,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:57:31,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:36,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:41,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:45,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:57:45,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:57:45,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:57:46,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:51,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:56,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:57:58,555 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:58:00,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:58:00,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:58:00,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:58:01,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:58:01,803 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:58:01,804 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:58:01,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:01,806 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:58:02,033 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:58:03,034 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:58:06,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:11,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:15,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:58:15,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:58:15,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:58:17,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:22,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:27,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:28,556 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:58:30,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:58:30,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:58:30,154 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:58:33,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:38,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:43,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:45,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:58:45,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:58:45,155 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:58:48,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:53,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:58,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:58:58,557 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:59:00,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:59:00,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:59:00,155 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:59:04,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:09,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:14,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:15,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:59:15,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:59:15,155 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:59:15,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 01:59:15,779 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 01:59:15,779 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 01:59:15,780 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 01:59:16,078 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 01:59:17,079 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 01:59:19,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:24,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:28,558 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 01:59:30,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:59:30,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:59:30,155 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:59:30,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:35,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:40,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:45,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 01:59:45,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 01:59:45,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 01:59:46,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:51,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:56,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 01:59:58,559 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:00:00,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:00:00,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:00:00,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:00:01,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:06,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:11,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:15,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:00:15,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:00:15,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:00:17,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:22,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:27,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:28,560 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:00:29,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:00:29,783 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:00:29,783 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:00:29,784 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:00:30,122 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:00:30,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:00:30,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:00:30,155 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:00:31,123 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:00:33,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:38,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:43,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:45,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:00:45,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:00:45,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:00:48,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:53,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:58,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:00:58,561 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:01:00,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:01:00,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:01:00,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:01:04,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:09,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:14,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:15,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:01:15,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:01:15,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:01:19,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:24,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:28,563 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:01:29,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:30,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:01:30,156 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:01:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:01:35,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:40,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:43,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:01:43,960 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:01:43,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:01:43,962 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:01:44,168 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:01:44,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:01:45,169 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:01:45,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:01:45,204 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:01:45,244 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:01:45,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:47,170 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:01:48,171 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:01:49,171 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:01:51,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:56,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:01:58,563 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:02:00,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:02:00,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:02:00,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:02:01,441 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:06,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:11,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:15,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:02:15,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:02:15,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:02:16,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:21,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:26,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:28,564 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:02:30,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:02:30,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:02:30,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:02:32,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:37,384 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:42,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:45,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:02:45,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:02:45,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:02:47,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:52,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:57,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:02:58,564 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:03:00,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:03:00,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:03:00,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:03:02,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:03:02,554 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:03:02,555 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:03:02,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:02,556 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:03:03,219 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:03:05,220 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:03:07,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:12,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:15,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:03:15,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:03:15,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:03:18,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:23,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:28,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:28,565 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:03:30,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:03:30,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:03:30,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:03:34,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:39,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:44,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:45,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:03:45,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:03:45,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:03:49,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:54,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:03:58,567 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:03:59,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:00,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:04:00,167 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:04:00,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:04:05,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:10,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:15,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:04:15,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:04:15,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:04:15,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:16,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:04:16,588 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:04:16,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:04:16,589 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:04:17,265 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:04:19,266 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:04:20,630 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:25,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:28,567 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:04:30,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:04:30,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:04:30,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:04:31,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:36,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:41,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:45,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:04:45,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:04:45,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:04:47,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:52,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:57,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:04:58,568 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:05:00,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:05:00,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:05:00,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:05:02,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:07,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:15,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:05:15,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:05:15,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:05:18,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:23,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:28,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:28,569 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:05:30,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:05:30,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:05:30,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:05:30,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:05:30,382 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:05:30,382 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:05:30,384 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:05:31,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:05:31,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:05:34,385 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:39,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:44,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:45,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:05:45,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:05:45,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:05:49,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:54,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:05:58,570 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:05:59,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:00,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:06:00,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:06:00,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:06:05,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:10,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:15,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:06:15,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:06:15,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:06:15,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:20,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:25,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:28,571 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:06:30,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:06:30,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:06:30,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:06:31,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:36,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:41,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:44,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:06:44,403 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:06:44,403 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:06:44,404 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:06:45,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:06:45,168 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:06:45,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:06:45,351 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:06:45,351 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:06:47,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:52,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:57,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:06:58,572 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:07:00,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:07:00,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:07:00,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:07:02,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:07,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:12,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:15,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:07:15,169 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:07:15,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:07:18,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:23,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:28,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:28,573 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:07:30,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:07:30,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:07:30,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:07:34,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:39,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:44,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:45,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:07:45,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:07:45,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:07:49,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:54,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:07:58,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:07:58,380 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:07:58,380 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:07:58,382 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:07:58,393 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:07:58,574 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:07:59,394 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:07:59,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:00,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:08:00,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:08:00,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:08:05,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:10,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:15,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:08:15,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:08:15,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:08:15,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:20,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:25,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:28,576 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:08:30,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:08:30,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:08:30,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:08:31,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:36,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:41,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:45,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:08:45,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:08:45,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:08:47,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:52,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:57,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:08:58,575 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:09:00,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:09:00,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:09:00,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:09:03,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:08,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:12,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:09:12,206 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:09:12,207 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:09:12,209 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:09:12,437 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:09:13,438 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:09:14,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:15,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:09:15,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:09:15,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:09:19,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:24,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:28,576 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:09:29,577 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:30,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:09:30,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:09:30,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:09:35,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:40,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:45,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:09:45,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:09:45,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:09:46,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:51,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:56,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:09:58,578 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:10:00,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:10:00,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:10:00,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:10:02,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:07,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:12,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:15,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:10:15,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:10:15,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:10:17,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:22,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:26,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:10:26,790 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:10:26,791 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:10:26,792 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:10:27,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:10:27,485 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:10:27,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:28,578 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:10:30,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:10:30,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:10:30,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:10:33,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:38,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:43,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:45,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:10:45,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:10:45,192 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:10:49,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:54,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:10:58,579 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:10:59,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:00,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:11:00,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:11:00,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:11:05,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:10,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:15,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:11:15,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:11:15,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:11:15,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:20,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:25,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:28,580 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:11:30,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:11:30,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:11:30,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:11:31,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:36,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:41,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:11:41,022 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:11:41,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:11:41,024 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:11:41,529 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:11:41,529 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:11:42,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:45,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:11:45,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:11:45,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:11:47,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:52,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:57,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:11:58,581 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:12:00,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:12:00,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:12:00,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:12:03,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:08,388 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:13,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:15,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:12:15,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:12:15,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:12:19,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:24,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:28,582 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:12:29,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:30,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:12:30,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:12:30,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:12:35,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:40,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:45,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:12:45,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:12:45,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:12:46,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:51,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:54,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:12:54,879 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:12:54,879 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:12:54,881 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:12:55,572 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:12:55,573 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:12:56,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:12:58,583 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:13:00,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:13:00,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:13:00,193 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:13:02,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:07,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:12,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:15,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:13:15,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:13:15,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:13:18,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:23,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:28,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:28,583 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:13:30,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:13:30,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:13:30,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:13:33,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:38,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:43,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:45,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:13:45,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:13:45,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:13:49,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:54,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:13:58,585 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:13:59,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:00,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:14:00,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:14:00,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:14:05,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:08,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:14:08,811 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:14:08,811 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:14:08,813 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:14:09,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:14:09,616 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:14:09,616 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:14:10,657 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:11,617 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:14:12,618 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:14:13,619 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:14:15,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:14:15,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:14:15,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:14:16,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:21,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:26,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:28,585 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:14:30,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:14:30,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:14:30,194 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:14:32,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:37,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:42,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:45,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:14:45,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:14:45,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:14:47,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:52,432 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:57,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:14:58,586 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:15:00,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:15:00,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:15:00,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:15:02,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:07,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:12,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:15,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:15:15,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:15:15,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:15:18,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:23,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:26,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:15:26,961 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:15:26,961 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:15:26,962 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:15:27,662 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:15:27,663 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:15:28,588 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:15:28,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:30,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:15:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:15:30,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:15:34,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:39,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:44,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:45,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:15:45,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:15:45,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:15:49,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:54,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:15:58,588 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:15:59,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:00,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:16:00,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:16:00,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:16:05,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:10,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:15,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:16:15,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:16:15,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:16:15,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:20,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:25,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:28,619 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:16:30,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:16:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:16:30,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:16:31,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:36,446 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:41,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:16:41,053 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:16:41,053 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:16:41,055 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:16:41,708 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:16:41,709 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:16:42,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:45,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:16:45,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:16:45,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:16:47,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:52,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:57,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:16:58,620 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:17:00,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:17:00,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:17:00,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:17:03,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:08,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:13,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:15,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:17:15,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:17:15,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:17:18,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:23,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:28,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:28,621 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:17:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:17:30,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:17:30,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:17:34,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:39,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:44,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:45,195 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:17:45,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:17:45,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:17:49,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:54,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:17:55,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:17:55,040 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:17:55,041 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:17:55,042 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:17:55,753 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:17:55,753 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:17:58,622 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:17:59,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:00,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:18:00,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:18:00,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:18:05,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:10,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:15,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:18:15,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:18:15,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:18:15,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:20,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:25,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:28,623 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:18:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:18:30,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:18:30,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:18:31,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:36,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:41,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:45,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:18:45,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:18:45,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:18:47,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:52,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:57,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:18:58,624 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:19:00,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:19:00,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:19:00,197 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:19:03,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:08,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:09,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:19:09,594 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:19:09,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:19:09,596 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:19:09,800 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:19:11,803 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:19:13,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:15,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:19:15,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:19:15,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:19:19,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:24,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:28,624 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:19:29,625 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:19:30,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:19:30,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:19:35,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:40,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:45,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:19:45,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:19:45,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:19:45,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:50,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:55,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:19:58,625 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:20:00,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:20:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:20:00,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:20:01,398 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:06,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:11,399 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:15,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:20:15,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:20:15,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:20:17,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:22,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:23,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:20:23,530 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:20:23,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:20:23,531 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:20:23,848 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:20:23,848 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:20:27,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:28,626 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:20:30,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:20:30,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:20:30,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:20:33,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:38,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:43,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:45,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:20:45,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:20:45,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:20:49,386 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:54,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:20:58,627 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:20:59,628 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:00,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:21:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:21:00,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:21:05,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:10,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:15,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:21:15,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:21:15,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:21:16,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:21,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:26,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:28,628 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:21:30,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:21:30,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:21:30,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:21:31,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:36,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:37,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:21:37,760 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:21:37,761 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:21:37,762 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:21:37,897 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:21:40,899 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:21:41,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:45,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:21:45,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:21:45,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:21:47,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:52,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:57,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:21:58,629 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:22:00,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:22:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:22:00,198 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:22:02,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:07,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:12,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:15,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:22:15,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:22:15,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:22:18,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:23,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:28,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:28,630 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:22:30,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:22:30,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:22:30,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:22:33,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:38,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:43,397 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:45,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:22:45,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:22:45,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:22:48,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:51,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:22:51,847 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:22:51,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:22:51,849 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:22:51,943 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:22:53,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:22:54,945 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:22:58,631 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:22:59,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:23:00,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:23:00,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:23:05,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:10,465 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:15,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:23:15,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:23:15,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:23:16,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:21,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:26,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:28,632 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:23:30,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:23:30,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:23:30,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:23:32,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:37,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:42,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:45,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:23:45,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:23:45,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:23:48,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:53,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:58,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:23:58,633 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:24:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:24:00,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:24:00,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:24:03,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:05,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:24:05,780 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:24:05,781 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:24:05,782 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:24:05,990 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:24:06,991 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:24:08,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:13,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:15,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:24:15,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:24:15,199 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:24:19,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:24,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:28,634 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:24:29,634 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:30,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:24:30,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:24:30,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:24:35,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:40,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:45,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:24:45,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:24:45,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:24:46,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:51,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:56,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:24:58,635 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:25:00,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:25:00,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:25:00,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:25:01,423 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:06,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:11,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:15,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:25:15,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:25:15,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:25:17,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:19,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:25:19,880 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:25:19,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:25:19,882 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:25:20,039 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:25:21,039 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:25:22,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:27,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:28,635 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:25:30,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:25:30,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:25:30,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:25:33,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:38,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:43,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:45,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:25:45,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:25:45,200 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:25:48,453 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:53,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:58,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:25:58,636 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:26:00,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:26:00,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:26:00,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:26:03,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:08,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:13,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:15,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:26:15,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:26:15,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:26:19,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:24,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:28,637 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:26:29,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:30,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:26:30,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:26:30,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:26:34,003 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:26:34,005 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:26:34,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:26:34,007 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:26:34,085 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:26:34,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:26:35,086 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:26:35,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:37,087 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:26:38,088 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:26:39,089 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:26:41,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:41,090 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:26:45,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:26:45,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:26:45,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:26:46,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:51,417 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:56,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:26:58,639 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:27:00,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:27:00,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:27:00,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:27:02,410 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:07,411 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:12,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:15,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:27:15,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:27:15,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:27:17,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:22,588 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:27,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:28,639 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:27:30,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:27:30,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:27:30,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:27:33,433 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:38,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:43,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:45,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:27:45,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:27:45,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:27:48,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:52,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:27:52,331 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:27:52,331 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:27:52,332 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:27:53,137 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:27:54,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:27:55,138 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:27:58,640 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:27:59,642 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:00,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:28:00,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:28:00,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:28:05,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:10,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:15,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:28:15,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:28:15,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:28:16,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:21,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:26,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:28,641 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:28:30,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:28:30,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:28:30,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:28:32,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:37,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:42,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:45,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:28:45,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:28:45,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:28:48,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:53,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:58,401 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:28:58,642 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:29:00,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:29:00,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:29:00,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:29:03,418 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:06,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:29:06,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:29:06,268 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:29:06,270 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:29:07,183 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:29:09,184 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:29:09,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:14,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:15,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:29:15,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:29:15,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:29:19,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:24,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:28,643 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:29:29,645 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:30,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:29:30,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:29:30,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:29:35,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:40,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:45,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:29:45,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:29:45,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:29:45,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:50,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:55,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:29:58,644 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:30:00,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:30:00,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:30:00,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:30:01,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:06,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:11,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:15,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:30:15,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:30:15,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:30:16,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:20,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:30:20,152 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:30:20,153 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:30:20,154 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:30:20,227 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:30:21,228 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:30:22,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:27,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:28,645 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:30:30,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:30:30,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:30:30,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:30:32,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:37,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:42,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:45,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:30:45,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:30:45,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:30:48,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:53,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:58,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:30:58,646 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:31:00,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:31:00,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:31:00,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:31:03,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:08,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:13,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:15,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:31:15,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:31:15,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:31:19,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:24,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:28,648 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:31:29,650 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:30,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:31:30,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:31:30,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:31:33,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:31:33,998 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:31:33,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:31:34,000 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:31:34,274 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:31:35,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:35,275 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:31:40,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:45,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:45,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:31:45,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:31:45,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:31:50,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:55,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:31:58,648 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:32:00,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:32:00,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:32:00,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:32:01,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:06,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:11,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:15,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:32:15,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:32:15,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:32:17,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:22,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:27,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:28,650 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:32:30,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:32:30,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:32:30,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:32:32,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:37,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:42,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:45,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:32:45,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:32:45,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:32:47,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:48,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:32:48,251 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:32:48,252 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:32:48,253 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:32:48,322 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:32:49,322 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:32:53,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:58,294 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:32:58,650 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:33:00,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:33:00,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:33:00,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:33:03,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:08,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:13,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:15,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:33:15,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:33:15,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:33:19,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:24,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:28,651 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:33:29,652 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:30,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:33:30,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:33:30,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:33:35,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:40,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:45,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:33:45,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:33:45,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:33:45,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:50,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:55,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:33:58,652 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:34:00,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:34:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:34:00,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:34:00,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:02,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:34:02,202 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:34:02,202 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:34:02,203 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:34:02,367 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:34:03,368 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:34:06,204 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:11,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:15,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:34:15,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:34:15,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:34:16,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:21,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:26,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:28,653 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:34:30,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:34:30,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:34:30,270 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:34:32,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:37,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:42,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:45,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:34:45,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:34:45,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:34:47,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:52,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:57,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:34:58,654 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:35:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:35:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:35:00,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:35:02,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:07,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:35:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:35:15,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:35:16,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:35:16,385 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:35:16,385 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:35:16,387 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:35:16,416 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:35:17,417 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:35:18,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:23,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:28,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:28,655 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:35:30,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:35:30,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:35:30,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:35:33,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:38,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:43,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:45,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:35:45,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:35:45,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:35:48,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:53,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:58,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:35:58,656 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:36:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:36:00,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:36:00,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:36:04,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:09,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:14,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:15,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:36:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:36:15,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:36:19,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:24,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:28,657 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:36:29,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:30,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:36:30,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:36:30,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:36:30,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:36:30,526 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:36:30,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:36:30,528 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:36:31,464 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:36:31,464 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:36:35,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:40,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:45,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:36:45,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:36:45,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:36:46,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:51,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:56,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:36:58,658 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:37:00,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:37:00,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:37:00,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:37:01,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:06,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:11,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:37:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:37:15,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:37:16,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:21,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:26,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:28,659 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:37:30,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:37:30,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:37:30,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:37:32,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:37,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:42,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:44,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:37:44,394 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:37:44,394 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:37:44,396 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:37:44,509 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:37:45,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:37:45,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:37:45,271 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:37:45,510 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:37:47,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:52,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:57,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:37:58,660 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:38:00,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:38:00,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:38:00,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:38:02,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:07,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:12,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:38:15,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:38:15,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:38:18,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:23,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:28,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:28,661 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:38:30,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:38:30,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:38:30,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:38:33,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:38,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:43,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:45,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:38:45,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:38:45,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:38:49,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:54,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:38:58,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:38:58,205 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:38:58,206 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:38:58,207 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:38:58,555 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:38:58,662 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:38:58,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:38:59,555 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:38:59,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:00,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:39:00,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:39:00,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:39:01,557 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:39:02,557 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:39:05,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:10,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:15,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:39:15,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:39:15,272 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:39:15,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:20,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:25,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:28,663 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:39:30,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:39:30,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:39:30,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:39:30,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:35,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:40,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:45,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:39:45,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:39:45,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:39:46,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:51,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:56,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:39:58,664 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:40:00,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:40:00,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:40:00,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:40:01,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:06,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:11,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:15,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:40:15,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:40:15,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:40:16,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:16,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:40:16,519 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:40:16,519 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:40:16,521 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:40:16,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:40:17,602 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:40:21,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:26,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:28,664 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:40:30,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:40:30,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:40:30,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:40:31,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:36,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:41,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:45,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:40:45,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:40:45,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:40:47,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:52,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:57,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:40:58,665 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:41:00,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:41:00,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:41:00,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:41:03,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:08,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:13,452 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:15,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:41:15,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:41:15,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:41:18,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:23,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:28,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:28,666 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:41:30,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:41:30,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:41:30,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:41:30,584 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:41:30,587 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:41:30,587 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:41:30,588 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:41:30,646 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:41:31,647 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:41:33,589 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:38,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:43,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:45,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:41:45,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:41:45,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:41:49,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:54,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:41:58,667 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:41:59,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:00,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:42:00,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:42:00,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:42:05,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:10,451 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:15,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:42:15,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:42:15,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:42:15,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:20,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:25,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:28,668 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:42:30,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:42:30,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:42:30,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:42:31,472 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:36,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:41,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:44,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:42:44,767 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:42:44,767 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:42:44,769 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:42:45,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:42:45,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:42:45,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:42:45,691 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:42:45,692 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:42:46,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:51,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:56,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:42:58,669 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:43:00,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:43:00,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:43:00,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:43:01,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:06,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:11,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:15,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:43:15,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:43:15,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:43:17,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:22,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:27,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:28,670 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:43:30,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:43:30,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:43:30,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:43:32,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:37,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:42,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:45,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:43:45,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:43:45,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:43:48,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:53,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:58,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:43:58,671 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:43:59,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:43:59,320 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:43:59,321 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:43:59,322 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:43:59,739 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:44:00,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:44:00,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:44:00,274 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:44:01,740 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:44:03,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:08,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:13,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:15,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:44:15,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:44:15,275 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:44:19,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:24,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:28,672 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:44:29,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:30,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:44:30,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:44:30,275 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:44:35,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:40,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:45,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:44:45,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:44:45,275 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:44:45,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:50,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:55,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:44:58,673 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:45:00,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:45:00,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:45:00,275 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:45:03,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:08,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:13,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:45:13,322 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:45:13,322 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:45:13,324 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:45:13,783 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:45:14,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:15,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:45:15,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:45:15,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:45:15,784 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:45:19,582 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:24,583 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:28,674 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:45:29,674 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:30,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:45:30,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:45:30,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:45:35,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:40,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:45,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:45:45,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:45:45,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:45:46,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:51,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:56,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:45:58,675 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:46:00,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:46:00,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:46:00,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:46:01,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:06,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:11,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:15,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:46:15,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:46:15,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:46:16,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:21,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:26,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:27,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:46:27,150 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:46:27,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:46:27,151 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:46:27,828 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:46:27,829 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:46:28,677 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:46:30,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:46:30,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:46:30,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:46:32,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:37,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:42,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:45,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:46:45,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:46:45,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:46:48,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:53,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:58,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:46:58,676 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:47:00,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:47:00,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:47:00,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:47:04,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:09,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:14,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:15,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:47:15,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:47:15,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:47:19,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:24,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:28,678 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:47:29,679 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:30,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:47:30,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:47:30,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:47:35,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:40,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:41,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:47:41,045 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:47:41,045 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:47:41,047 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:47:41,874 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:47:41,874 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:47:45,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:47:45,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:47:45,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:47:46,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:51,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:56,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:47:58,678 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:48:00,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:48:00,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:48:00,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:48:01,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:06,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:11,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:15,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:48:15,276 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:48:15,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:48:17,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:22,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:27,515 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:28,679 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:48:30,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:48:30,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:48:30,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:48:33,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:38,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:43,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:45,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:48:45,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:48:45,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:48:48,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:53,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:48:54,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:48:54,953 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:48:54,953 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:48:54,954 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:48:55,919 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:48:55,919 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:48:58,680 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:48:58,681 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:00,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:49:00,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:49:00,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:49:04,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:09,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:14,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:15,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:49:15,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:49:15,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:49:20,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:25,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:28,681 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:49:30,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:49:30,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:49:30,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:49:31,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:36,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:41,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:45,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:49:45,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:49:45,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:49:46,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:51,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:56,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:49:58,682 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:50:00,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:50:00,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:50:00,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:50:02,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:07,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:09,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:50:09,062 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:50:09,062 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:50:09,063 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:50:09,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:50:09,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:50:13,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:15,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:50:15,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:50:15,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:50:18,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:23,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:28,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:28,683 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:50:30,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:50:30,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:50:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:50:34,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:39,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:44,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:45,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:50:45,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:50:45,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:50:50,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:55,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:50:58,684 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:51:00,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:51:00,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:51:00,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:51:00,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:05,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:10,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:15,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:51:15,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:51:15,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:51:16,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:21,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:23,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:51:23,085 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:51:23,085 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:51:23,086 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:51:23,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:51:24,008 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:51:24,008 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:51:26,009 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:51:27,010 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:51:27,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:28,011 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:51:28,684 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:51:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:51:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:51:30,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:51:32,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:37,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:42,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:45,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:51:45,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:51:45,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:51:48,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:53,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:58,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:51:58,685 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:52:00,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:52:00,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:52:00,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:52:03,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:08,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:13,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:15,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:52:15,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:52:15,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:52:19,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:24,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:28,686 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:52:29,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:52:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:52:30,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:52:35,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:40,484 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:41,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:52:41,432 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:52:41,432 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:52:41,433 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:52:42,060 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:52:42,061 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:52:45,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:52:45,278 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:52:45,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:52:45,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:50,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:55,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:52:58,687 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:53:00,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:53:00,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:53:00,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:53:01,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:06,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:11,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:15,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:53:15,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:53:15,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:53:17,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:22,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:27,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:28,688 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:53:30,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:53:30,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:53:30,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:53:32,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:37,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:42,497 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:45,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:53:45,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:53:45,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:53:47,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:52,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:55,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:53:55,265 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:53:55,266 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:53:55,268 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:53:56,107 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:53:56,107 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:53:58,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:53:58,689 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:54:00,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:54:00,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:54:00,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:54:03,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:08,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:13,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:15,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:54:15,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:54:15,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:54:18,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:23,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:28,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:28,690 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:54:30,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:54:30,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:54:30,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:54:33,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:38,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:43,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:45,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:54:45,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:54:45,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:54:49,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:54,514 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:54:58,691 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:54:59,693 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:00,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:55:00,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:55:00,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:55:05,489 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:09,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:55:09,306 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:55:09,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:55:09,309 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:55:10,160 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:55:10,161 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:55:11,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:15,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:55:15,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:55:15,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:55:16,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:21,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:26,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:28,692 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:55:30,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:55:30,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:55:30,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:55:32,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:37,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:42,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:45,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:55:45,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:55:45,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:55:47,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:52,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:57,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:55:58,693 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:56:00,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:56:00,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:56:00,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:56:03,489 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:08,490 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:13,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:15,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:56:15,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:56:15,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:56:18,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:23,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:56:23,283 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:56:23,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:56:23,285 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:56:24,207 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:56:24,208 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:56:24,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:28,694 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:56:29,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:30,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:56:30,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:56:30,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:56:35,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:40,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:45,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:56:45,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:56:45,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:56:46,478 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:51,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:56,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:56:58,695 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:57:00,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:57:00,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:57:00,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:57:01,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:06,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:11,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:15,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:57:15,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:57:15,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:57:16,547 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:21,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:26,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:28,695 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:57:30,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:57:30,280 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:57:30,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:57:32,510 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:37,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:57:37,496 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:57:37,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:57:37,498 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:57:38,254 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:57:38,255 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:57:38,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:43,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:45,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:57:45,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:57:45,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:57:49,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:54,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:57:58,696 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:57:59,697 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:00,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:58:00,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:58:00,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:58:05,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:10,477 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:15,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:58:15,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:58:15,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:58:15,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:20,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:25,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:28,697 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:58:30,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:58:30,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:58:30,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:58:30,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:35,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:40,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:45,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:58:45,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:58:45,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:58:46,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:51,343 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 02:58:51,345 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 02:58:51,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 02:58:51,347 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 02:58:52,301 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 02:58:52,301 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 02:58:52,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:57,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:58:58,698 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:59:00,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:59:00,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:59:00,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:59:02,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:07,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:12,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:15,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:59:15,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:59:15,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:59:18,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:23,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:28,518 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:28,699 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:59:30,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:59:30,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:59:30,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:59:33,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:38,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:43,556 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:45,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 02:59:45,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 02:59:45,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 02:59:49,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:54,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 02:59:58,700 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 02:59:59,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:00,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:00:00,281 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:00:00,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:00:05,400 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:00:05,403 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:00:05,403 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:00:05,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:05,405 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:00:06,353 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:00:06,353 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:00:10,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:15,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:00:15,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:00:15,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:00:15,457 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:20,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:25,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:28,704 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:00:30,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:00:30,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:00:30,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:00:30,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:35,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:40,504 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:45,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:00:45,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:00:45,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:00:46,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:51,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:56,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:00:58,702 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:01:00,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:01:00,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:01:00,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:01:01,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:06,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:11,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:15,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:01:15,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:01:15,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:01:17,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:19,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:01:19,913 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:01:19,913 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:01:19,915 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:01:20,401 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:01:21,402 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:01:22,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:27,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:28,703 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:01:30,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:01:30,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:01:30,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:01:33,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:38,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:43,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:45,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:01:45,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:01:45,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:01:48,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:53,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:58,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:01:58,705 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:02:00,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:02:00,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:02:00,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:02:04,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:09,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:14,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:15,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:02:15,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:02:15,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:02:19,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:24,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:28,704 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:02:29,705 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:30,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:02:30,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:02:30,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:02:33,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:02:33,900 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:02:33,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:02:33,902 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:02:34,446 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:02:34,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:35,447 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:02:39,903 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:44,904 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:45,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:02:45,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:02:45,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:02:50,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:55,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:02:58,705 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:03:00,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:03:00,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:03:00,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:03:00,491 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:05,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:10,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:15,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:03:15,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:03:15,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:03:15,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:20,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:25,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:28,706 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:03:30,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:03:30,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:03:30,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:03:30,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:35,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:40,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:45,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:03:45,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:03:45,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:03:46,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:47,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:03:47,953 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:03:47,953 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:03:47,954 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:03:48,491 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:03:48,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:03:49,492 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:03:51,494 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:03:52,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:52,495 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:03:53,496 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:03:57,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:03:58,707 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:04:00,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:04:00,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:04:00,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:04:02,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:07,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:12,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:15,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:04:15,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:04:15,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:04:18,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:23,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:28,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:28,708 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:04:30,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:04:30,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:04:30,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:04:34,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:39,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:44,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:45,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:04:45,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:04:45,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:04:49,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:54,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:04:58,709 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:04:59,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:00,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:05:00,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:05:00,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:05:05,548 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:06,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:05:06,038 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:05:06,039 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:05:06,040 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:05:06,540 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:05:07,541 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:05:11,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:15,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:05:15,283 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:05:15,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:05:16,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:21,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:26,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:28,710 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:05:30,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:05:30,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:05:30,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:05:31,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:36,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:41,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:45,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:05:45,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:05:45,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:05:47,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:52,454 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:57,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:05:58,710 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:06:00,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:06:00,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:06:00,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:06:02,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:07,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:12,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:15,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:06:15,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:06:15,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:06:18,519 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:19,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:06:19,878 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:06:19,878 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:06:19,880 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:06:20,587 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:06:21,587 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:06:23,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:28,711 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:06:29,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:30,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:06:30,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:06:30,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:06:35,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:40,555 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:45,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:06:45,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:06:45,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:06:46,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:51,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:56,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:06:58,712 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:07:00,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:07:00,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:07:00,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:07:01,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:06,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:11,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:15,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:07:15,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:07:15,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:07:17,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:22,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:27,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:28,713 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:07:30,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:07:30,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:07:30,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:07:33,493 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:33,880 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:07:33,883 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:07:33,883 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:07:33,885 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:07:34,631 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:07:35,632 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:07:38,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:43,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:45,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:07:45,284 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:07:45,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:07:49,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:54,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:07:58,714 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:07:59,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:00,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:08:00,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:08:00,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:08:05,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:10,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:15,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:08:15,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:08:15,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:08:15,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:20,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:25,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:28,715 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:08:30,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:08:30,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:08:30,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:08:31,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:36,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:41,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:45,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:08:45,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:08:45,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:08:47,456 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:48,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:08:48,461 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:08:48,461 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:08:48,463 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:08:48,679 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:08:49,680 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:08:52,463 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:57,464 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:08:58,717 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:09:00,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:09:00,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:09:00,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:09:02,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:07,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:12,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:15,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:09:15,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:09:15,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:09:18,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:23,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:28,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:28,717 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:09:30,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:09:30,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:09:30,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:09:34,498 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:39,499 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:44,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:45,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:09:45,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:09:45,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:09:49,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:54,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:09:58,718 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:09:59,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:00,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:10:00,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:10:00,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:10:02,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:10:02,567 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:10:02,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:10:02,569 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:10:02,727 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:10:03,728 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:10:05,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:10,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:15,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:10:15,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:10:15,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:10:16,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:21,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:26,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:28,719 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:10:30,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:10:30,285 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:10:30,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:10:32,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:37,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:42,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:45,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:10:45,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:10:45,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:10:48,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:53,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:58,468 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:10:58,720 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:11:00,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:11:00,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:11:00,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:11:03,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:08,508 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:13,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:15,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:11:15,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:11:15,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:11:16,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:11:16,720 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:11:16,721 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:11:16,722 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:11:16,774 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:11:17,775 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:11:18,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:23,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:28,720 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:11:29,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:30,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:11:30,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:11:30,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:11:35,516 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:40,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:45,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:11:45,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:11:45,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:11:46,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:51,472 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:56,473 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:11:58,721 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:12:00,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:12:00,286 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:12:00,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:12:01,475 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:06,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:11,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:15,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:12:15,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:12:15,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:12:16,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:21,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:26,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:28,722 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:12:30,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:12:30,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:12:30,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:12:30,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:12:30,787 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:12:30,788 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:12:30,789 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:12:30,821 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:12:31,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:31,822 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:12:36,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:41,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:45,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:12:45,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:12:45,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:12:47,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:52,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:57,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:12:58,723 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:13:00,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:13:00,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:13:00,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:13:03,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:08,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:13,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:15,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:13:15,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:13:15,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:13:18,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:23,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:28,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:28,724 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:13:30,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:13:30,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:13:30,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:13:33,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:38,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:43,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:44,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:13:44,777 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:13:44,778 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:13:44,780 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:13:44,867 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:13:45,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:13:45,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:13:45,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:13:45,868 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:13:49,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:54,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:13:58,725 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:13:59,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:00,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:14:00,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:14:00,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:14:05,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:10,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:15,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:14:15,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:14:15,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:14:15,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:20,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:25,546 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:28,726 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:14:30,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:14:30,287 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:14:30,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:14:31,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:36,502 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:41,503 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:45,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:14:45,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:14:45,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:14:46,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:51,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:56,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:14:58,727 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:14:58,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:14:58,757 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:14:58,758 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:14:58,759 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:14:58,913 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:14:59,913 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:15:00,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:15:00,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:15:00,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:15:02,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:07,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:12,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:15,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:15:15,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:15:15,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:15:17,480 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:22,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:27,481 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:28,728 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:15:30,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:15:30,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:15:30,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:15:32,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:37,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:42,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:45,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:15:45,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:15:45,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:15:48,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:53,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:58,545 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:15:58,729 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:16:00,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:16:00,288 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:16:00,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:16:04,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:09,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:12,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:16:12,795 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:16:12,796 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:16:12,797 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:16:12,960 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:16:13,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:16:13,960 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:16:14,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:15,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:16:15,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:16:15,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:16:15,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:16:16,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:16:17,963 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:16:20,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:25,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:28,730 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:16:30,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:16:30,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:16:30,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:16:30,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:35,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:40,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:45,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:16:45,525 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:16:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:16:45,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:50,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:55,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:16:58,731 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:17:00,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:17:00,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:17:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:17:01,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:06,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:11,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:15,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:17:15,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:17:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:17:17,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:22,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:27,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:28,731 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:17:30,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:17:30,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:17:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:17:31,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:17:31,614 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:17:31,615 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:17:31,616 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:17:32,009 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:17:33,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:34,011 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:17:38,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:43,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:45,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:17:45,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:17:45,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:17:48,694 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:53,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:58,695 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:17:58,732 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:18:00,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:18:00,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:18:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:18:03,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:08,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:13,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:15,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:18:15,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:18:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:18:19,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:24,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:28,733 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:18:30,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:18:30,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:18:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:18:30,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:35,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:40,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:45,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:18:45,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:18:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:18:45,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:18:45,793 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:18:45,793 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:18:45,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:45,795 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:18:46,052 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:18:48,054 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:18:50,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:55,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:18:58,734 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:19:00,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:19:00,526 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:19:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:19:01,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:06,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:11,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:15,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:19:15,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:19:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:19:16,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:21,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:26,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:28,735 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:19:30,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:19:30,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:19:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:19:31,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:36,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:41,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:45,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:19:45,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:19:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:19:47,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:52,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:57,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:19:58,736 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:19:59,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:19:59,907 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:19:59,907 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:19:59,908 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:20:00,097 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:20:00,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:20:00,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:20:00,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:20:02,099 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:20:02,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:07,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:12,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:15,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:20:15,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:20:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:20:17,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:22,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:27,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:28,737 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:20:30,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:20:30,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:20:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:20:32,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:37,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:42,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:45,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:20:45,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:20:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:20:48,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:53,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:58,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:20:58,738 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:21:00,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:21:00,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:21:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:21:03,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:08,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:13,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:13,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:21:13,891 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:21:13,892 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:21:13,893 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:21:14,146 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:21:15,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:21:15,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:21:15,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:21:16,147 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:21:18,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:23,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:28,739 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:21:29,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:30,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:21:30,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:21:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:21:35,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:40,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:45,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:21:45,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:21:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:21:45,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:50,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:55,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:21:58,740 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:22:00,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:22:00,527 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:22:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:22:00,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:05,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:10,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:15,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:22:15,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:22:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:22:16,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:21,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:26,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:27,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:22:27,843 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:22:27,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:22:27,844 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:22:28,190 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:22:28,742 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:22:30,191 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:22:30,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:22:30,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:22:30,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:22:31,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:36,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:41,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:45,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:22:45,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:22:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:22:47,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:52,700 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:57,701 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:22:58,742 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:23:00,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:23:00,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:23:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:23:02,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:07,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:12,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:15,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:23:15,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:23:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:23:17,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:22,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:27,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:28,743 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:23:30,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:23:30,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:23:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:23:32,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:37,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:41,911 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:23:41,913 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:23:41,913 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:23:41,914 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:23:42,233 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:23:42,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:44,235 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:23:45,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:23:45,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:23:45,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:23:48,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:53,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:23:58,744 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:23:59,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:00,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:24:00,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:24:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:24:04,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:09,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:14,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:15,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:24:15,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:24:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:24:19,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:24,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:28,747 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:24:30,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:24:30,528 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:24:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:24:30,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:35,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:40,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:45,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:24:45,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:24:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:24:46,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:51,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:56,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:24:56,093 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:24:56,093 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:24:56,094 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:24:56,279 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:24:57,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:24:58,280 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:24:58,746 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:25:00,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:25:00,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:25:00,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:25:02,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:07,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:12,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:15,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:25:15,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:25:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:25:18,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:23,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:28,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:28,747 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:25:30,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:25:30,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:25:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:25:33,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:38,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:43,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:45,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:25:45,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:25:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:25:49,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:54,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:25:58,748 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:25:59,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:00,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:26:00,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:26:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:26:04,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:09,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:10,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:26:10,748 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:26:10,748 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:26:10,749 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:26:11,325 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:26:12,326 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:26:15,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:26:15,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:26:15,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:26:15,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:20,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:25,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:28,748 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:26:30,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:26:30,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:26:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:26:31,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:36,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:41,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:45,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:26:45,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:26:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:26:46,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:51,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:56,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:26:58,749 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:27:00,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:27:00,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:27:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:27:01,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:06,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:11,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:15,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:27:15,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:27:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:27:17,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:22,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:24,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:27:24,866 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:27:24,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:27:24,867 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:27:25,369 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:27:26,370 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:27:27,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:28,750 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:27:30,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:27:30,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:27:30,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:27:33,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:38,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:43,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:45,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:27:45,529 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:27:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:27:49,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:54,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:27:58,751 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:28:00,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:28:00,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:28:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:28:00,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:05,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:10,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:15,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:28:15,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:28:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:28:16,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:21,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:26,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:28,752 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:28:30,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:28:30,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:28:30,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:28:31,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:36,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:38,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:28:38,872 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:28:38,872 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:28:38,873 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:28:39,412 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:28:39,445 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:28:40,413 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:28:42,129 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:42,415 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:28:44,417 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:28:45,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:28:45,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:28:45,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:28:47,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:52,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:57,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:28:58,753 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:29:00,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:29:00,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:29:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:29:03,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:08,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:13,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:15,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:29:15,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:29:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:29:19,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:24,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:28,754 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:29:29,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:30,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:29:30,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:29:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:29:35,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:40,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:45,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:29:45,530 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:29:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:29:45,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:50,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:55,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:29:57,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:29:57,002 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:29:57,003 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:29:57,004 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:29:57,464 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:29:58,461 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:29:58,755 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:30:00,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:30:00,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:30:00,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:30:00,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:05,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:10,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:15,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:30:15,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:30:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:30:16,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:21,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:26,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:28,756 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:30:30,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:30:30,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:30:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:30:32,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:37,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:42,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:45,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:30:45,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:30:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:30:48,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:53,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:58,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:30:58,756 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:31:00,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:31:00,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:31:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:31:03,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:08,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:11,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:31:11,191 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:31:11,191 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:31:11,193 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:31:11,506 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:31:12,506 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:31:14,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:15,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:31:15,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:31:15,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:31:19,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:24,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:28,757 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:31:30,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:31:30,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:31:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:31:30,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:35,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:40,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:45,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:31:45,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:31:45,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:31:46,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:51,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:56,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:31:58,759 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:32:00,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:32:00,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:32:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:32:01,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:06,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:11,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:15,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:32:15,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:32:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:32:17,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:22,703 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:25,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:32:25,427 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:32:25,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:32:25,429 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:32:25,553 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:32:26,553 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:32:28,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:28,759 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:32:30,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:32:30,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:32:30,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:32:33,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:38,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:43,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:45,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:32:45,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:32:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:32:48,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:53,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:58,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:32:58,761 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:33:00,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:33:00,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:33:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:33:04,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:09,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:14,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:15,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:33:15,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:33:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:33:19,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:24,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:28,761 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:33:29,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:30,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:33:30,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:33:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:33:35,521 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:39,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:33:39,980 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:33:39,981 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:33:39,982 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:33:40,600 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:33:40,600 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:33:41,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:45,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:33:45,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:33:45,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:33:46,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:51,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:56,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:33:58,762 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:34:00,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:34:00,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:34:00,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:34:02,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:07,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:12,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:15,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:34:15,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:34:15,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:34:18,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:23,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:28,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:28,763 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:34:30,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:34:30,532 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:34:30,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:34:33,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:38,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:43,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:45,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:34:45,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:34:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:34:48,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:53,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:34:54,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:34:54,268 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:34:54,268 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:34:54,270 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:34:54,645 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:34:54,645 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:34:58,764 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:34:58,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:00,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:35:00,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:35:00,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:35:04,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:09,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:14,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:15,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:35:15,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:35:15,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:35:19,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:24,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:28,765 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:35:29,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:30,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:35:30,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:35:30,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:35:34,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:39,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:44,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:45,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:35:45,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:35:45,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:35:50,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:55,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:35:58,766 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:36:00,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:36:00,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:36:00,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:36:01,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:06,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:08,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:36:08,263 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:36:08,263 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:36:08,264 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:36:08,688 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:36:09,688 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:36:12,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:15,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:36:15,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:36:15,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:36:17,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:22,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:27,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:28,767 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:36:30,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:36:30,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:36:30,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:36:32,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:37,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:42,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:45,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:36:45,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:36:45,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:36:48,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:53,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:58,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:36:58,768 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:37:00,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:37:00,533 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:37:00,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:37:03,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:08,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:13,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:15,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:37:15,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:37:15,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:37:19,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:22,488 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:37:22,490 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:37:22,490 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:37:22,492 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:37:22,733 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:37:23,734 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:37:25,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:28,768 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:37:30,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:37:30,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:37:30,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:37:30,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:35,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:40,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:45,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:37:45,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:37:45,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:37:46,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:51,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:56,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:37:58,769 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:38:00,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:38:00,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:38:00,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:38:02,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:07,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:12,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:15,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:38:15,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:38:15,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:38:18,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:23,712 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:28,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:28,770 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:38:30,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:38:30,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:38:30,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:38:33,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:36,660 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:38:36,662 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:38:36,662 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:38:36,663 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:38:36,777 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:38:39,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:39,779 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:38:44,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:45,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:38:45,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:38:45,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:38:49,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:54,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:38:58,771 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:39:00,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:39:00,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:39:00,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:39:00,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:05,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:10,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:15,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:39:15,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:39:15,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:39:16,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:21,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:26,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:28,772 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:39:30,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:39:30,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:39:30,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:39:32,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:37,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:42,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:45,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:39:45,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:39:45,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:39:48,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:50,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:39:50,803 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:39:50,803 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:39:50,804 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:39:50,819 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:39:53,821 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:39:53,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:39:58,773 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:39:59,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:00,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:40:00,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:40:00,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:40:05,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:10,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:15,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:40:15,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:40:15,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:40:16,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:21,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:26,730 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:28,774 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:40:30,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:40:30,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:40:30,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:40:31,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:36,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:41,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:45,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:40:45,535 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:40:45,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:40:46,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:51,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:56,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:40:58,775 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:41:00,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:41:00,536 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:41:00,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:41:02,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:04,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:41:04,787 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:41:04,787 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:41:04,788 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:41:04,864 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:41:05,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:41:05,865 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:41:07,867 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:41:08,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:08,868 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:41:09,868 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:41:11,870 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:41:13,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:15,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:41:15,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:41:15,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:41:19,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:24,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:28,776 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:41:29,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:30,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:41:30,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:41:30,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:41:35,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:40,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:45,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:41:45,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:41:45,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:41:45,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:50,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:55,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:41:58,777 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:42:00,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:42:00,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:42:00,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:42:01,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:06,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:11,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:15,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:42:15,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:42:15,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:42:17,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:22,741 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:23,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:42:23,615 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:42:23,615 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:42:23,616 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:42:23,916 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:42:25,917 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:42:28,658 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:28,777 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:42:30,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:42:30,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:42:30,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:42:33,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:38,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:43,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:45,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:42:45,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:42:45,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:42:49,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:54,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:42:58,778 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:42:59,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:00,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:43:00,560 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:43:00,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:43:04,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:09,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:14,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:15,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:43:15,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:43:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:43:20,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:25,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:28,779 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:43:30,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:43:30,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:43:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:43:30,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:35,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:38,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:43:38,062 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:43:38,062 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:43:38,063 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:43:38,960 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:43:39,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:43:41,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:45,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:43:45,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:43:45,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:43:46,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:51,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:56,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:43:58,780 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:44:00,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:44:00,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:44:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:44:02,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:07,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:12,763 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:15,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:44:15,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:44:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:44:17,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:22,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:27,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:28,781 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:44:30,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:44:30,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:44:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:44:32,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:37,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:42,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:45,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:44:45,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:44:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:44:48,729 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:52,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:44:52,497 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:44:52,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:44:52,498 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:44:53,005 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:44:54,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:44:54,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:44:58,782 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:44:59,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:00,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:45:00,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:45:00,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:45:04,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:09,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:14,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:15,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:45:15,561 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:45:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:45:20,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:25,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:28,783 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:45:30,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:45:30,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:45:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:45:30,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:35,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:40,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:45,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:45:45,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:45:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:45:46,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:51,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:56,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:45:58,784 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:46:00,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:46:00,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:46:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:46:01,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:06,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:46:06,637 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:46:06,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:46:06,639 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:46:07,050 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:46:07,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:08,051 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:46:12,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:15,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:46:15,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:46:15,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:46:17,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:22,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:27,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:28,785 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:46:30,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:46:30,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:46:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:46:33,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:38,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:43,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:45,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:46:45,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:46:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:46:48,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:53,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:46:58,786 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:46:59,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:00,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:47:00,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:47:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:47:05,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:10,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:15,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:47:15,562 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:47:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:47:15,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:20,671 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:47:20,674 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:47:20,674 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:47:20,675 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:47:21,094 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:47:21,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:22,095 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:47:26,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:28,787 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:47:30,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:47:30,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:47:30,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:47:31,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:36,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:41,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:45,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:47:45,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:47:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:47:46,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:51,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:56,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:47:58,788 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:48:00,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:48:00,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:48:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:48:02,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:07,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:12,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:15,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:48:15,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:48:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:48:18,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:23,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:28,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:28,788 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:48:30,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:48:30,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:48:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:48:33,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:34,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:48:34,582 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:48:34,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:48:34,583 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:48:35,137 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:48:36,138 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:48:39,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:44,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:45,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:48:45,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:48:45,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:48:49,909 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:54,910 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:48:58,790 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:49:00,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:49:00,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:49:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:49:02,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:07,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:12,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:15,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:49:15,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:49:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:49:18,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:23,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:28,790 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:49:29,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:30,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:49:30,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:49:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:49:34,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:39,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:44,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:45,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:49:45,563 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:49:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:49:48,438 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:49:48,440 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:49:48,441 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:49:48,442 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:49:49,182 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:49:50,183 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:49:50,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:55,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:49:58,791 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:50:00,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:50:00,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:50:00,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:50:00,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:05,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:10,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:15,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:50:15,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:50:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:50:16,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:21,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:26,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:28,792 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:50:30,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:50:30,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:50:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:50:31,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:36,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:41,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:45,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:50:45,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:50:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:50:46,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:51,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:56,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:50:58,793 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:51:00,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:51:00,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:51:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:51:02,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:03,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:51:03,028 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:51:03,028 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:51:03,029 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:51:03,232 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:51:04,233 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:51:08,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:13,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:15,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:51:15,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:51:15,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:51:18,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:23,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:28,794 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:51:29,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:30,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:51:30,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:51:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:51:34,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:39,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:44,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:45,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:51:45,564 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:51:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:51:50,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:55,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:51:58,795 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:52:00,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:52:00,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:52:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:52:01,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:06,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:11,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:15,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:52:15,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:52:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:52:16,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:17,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:52:17,275 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:52:17,275 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:52:17,276 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:52:17,277 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:52:18,278 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:52:22,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:27,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:28,796 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:52:30,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:52:30,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:52:30,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:52:32,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:37,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:42,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:45,564 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:52:45,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:52:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:52:48,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:53,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:58,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:52:58,797 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:53:00,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:53:00,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:53:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:53:03,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:08,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:13,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:15,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:53:15,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:53:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:53:18,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:23,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:28,798 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:53:29,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:30,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:53:30,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:53:30,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:53:31,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:53:31,279 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:53:31,279 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:53:31,280 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:53:31,322 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:53:31,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:53:32,323 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:53:34,325 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:53:35,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:36,326 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:53:40,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:45,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:45,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:53:45,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:53:45,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:53:50,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:55,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:53:58,799 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:54:00,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:54:00,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:54:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:54:01,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:06,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:11,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:15,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:54:15,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:54:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:54:17,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:22,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:27,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:28,801 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:54:30,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:54:30,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:54:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:54:32,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:37,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:42,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:45,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:54:45,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:54:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:54:47,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:49,425 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:54:49,426 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:54:49,427 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:54:49,428 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:54:50,371 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:54:50,371 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:54:53,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:58,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:54:58,801 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:55:00,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:55:00,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:55:00,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:55:03,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:08,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:13,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:15,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:55:15,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:55:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:55:18,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:23,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:28,803 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:55:29,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:30,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:55:30,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:55:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:55:35,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:40,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:45,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:55:45,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:55:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:55:46,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:51,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:56,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:55:58,803 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:56:00,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:56:00,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:56:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:56:01,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:03,486 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:56:03,518 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:56:03,518 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:56:03,519 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:56:04,415 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:56:04,415 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:56:07,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:12,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:15,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:56:15,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:56:15,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:56:17,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:22,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:27,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:28,804 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:56:30,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:56:30,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:56:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:56:33,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:38,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:43,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:45,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:56:45,566 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:56:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:56:48,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:53,764 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:58,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:56:58,805 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:57:00,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:57:00,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:57:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:57:03,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:08,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:13,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:15,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:57:15,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:57:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:57:17,635 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:57:17,637 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:57:17,637 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:57:17,638 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:57:18,460 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:57:18,460 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:57:19,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:24,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:28,806 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:57:29,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:30,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:57:30,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:57:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:57:34,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:39,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:44,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:45,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:57:45,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:57:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:57:50,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:55,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:57:58,807 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:58:00,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:58:00,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:58:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:58:00,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:05,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:10,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:15,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:58:15,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:58:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:58:16,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:21,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:26,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:28,808 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:58:30,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:58:30,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:58:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:58:32,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:58:32,117 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:58:32,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:58:32,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:32,118 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:58:32,507 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:58:34,509 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:58:37,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:42,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:45,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:58:45,567 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:58:45,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:58:47,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:52,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:57,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:58:58,809 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:59:00,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:59:00,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:59:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:59:03,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:08,713 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:13,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:15,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:59:15,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:59:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:59:18,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:23,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:28,810 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 03:59:29,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:30,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:59:30,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:59:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:59:35,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:40,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 03:59:45,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 03:59:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 03:59:46,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 03:59:46,573 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 03:59:46,574 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 03:59:46,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:46,575 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 03:59:47,559 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 03:59:48,559 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 03:59:51,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:56,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 03:59:58,811 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:00:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:00:00,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:00:00,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:00:01,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:06,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:11,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:00:15,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:00:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:00:17,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:22,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:27,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:28,812 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:00:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:00:30,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:00:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:00:33,748 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:38,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:43,749 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:00:45,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:00:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:00:48,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:53,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:58,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:00:58,813 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:01:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:01:00,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:01:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:01:00,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:01:00,886 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:01:00,886 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:01:00,888 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:01:01,604 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:01:02,605 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:01:03,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:08,926 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:13,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:01:15,568 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:01:15,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:01:19,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:24,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:28,814 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:01:29,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:01:30,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:01:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:01:35,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:40,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:45,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:01:45,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:01:45,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:01:45,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:50,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:55,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:01:58,815 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:02:00,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:02:00,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:02:00,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:02:00,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:05,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:10,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:14,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:02:14,947 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:02:14,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:02:14,948 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:02:15,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:02:15,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:02:15,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:02:15,650 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:02:15,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:16,650 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:02:20,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:25,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:28,816 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:02:30,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:02:30,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:02:30,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:02:31,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:36,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:41,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:45,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:02:45,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:02:45,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:02:47,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:52,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:57,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:02:58,816 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:03:00,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:03:00,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:03:00,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:03:02,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:07,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:12,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:15,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:03:15,569 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:03:15,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:03:17,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:22,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:27,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:28,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:03:28,759 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:03:28,759 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:03:28,761 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:03:28,817 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:03:29,694 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:03:30,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:03:30,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:03:30,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:03:30,695 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:03:33,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:38,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:43,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:45,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:03:45,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:03:45,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:03:49,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:54,737 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:03:58,818 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:03:59,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:00,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:04:00,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:04:00,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:04:05,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:10,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:15,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:04:15,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:04:15,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:04:15,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:20,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:25,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:28,819 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:04:30,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:04:30,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:04:30,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:04:31,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:36,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:41,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:42,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:04:42,987 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:04:42,987 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:04:42,989 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:04:43,738 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:04:44,739 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:04:45,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:04:45,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:04:45,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:04:47,738 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:52,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:57,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:04:58,820 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:05:00,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:05:00,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:05:00,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:05:02,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:07,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:12,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:15,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:05:15,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:05:15,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:05:18,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:23,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:28,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:28,821 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:05:30,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:05:30,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:05:30,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:05:34,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:39,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:44,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:45,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:05:45,570 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:05:45,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:05:50,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:55,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:05:57,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:05:57,273 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:05:57,273 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:05:57,275 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:05:57,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:05:57,885 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:05:58,822 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:05:58,886 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:06:00,629 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:06:00,630 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:06:00,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:06:00,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:00,887 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:06:02,889 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:06:06,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:11,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:15,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:06:15,579 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:06:15,614 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:06:16,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:21,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:26,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:28,823 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:06:30,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:06:30,579 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:06:30,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:06:31,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:36,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:41,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:45,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:06:45,580 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:06:45,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:06:47,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:52,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:57,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:06:58,824 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:07:00,579 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:07:00,580 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:07:00,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:07:02,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:07,796 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:12,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:15,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:07:15,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:07:15,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:07:15,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:07:16,440 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:07:16,440 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:07:16,442 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:07:16,934 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:07:16,935 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:07:18,442 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:23,443 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:28,444 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:28,825 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:07:30,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:07:30,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:07:30,615 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:07:33,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:38,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:43,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:45,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:07:45,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:07:45,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:07:48,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:53,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:07:58,826 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:07:59,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:00,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:08:00,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:08:00,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:08:05,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:10,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:15,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:08:15,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:08:15,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:08:15,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:20,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:25,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:28,827 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:08:30,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:08:30,252 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:08:30,252 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:08:30,253 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:08:30,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:08:30,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:08:30,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:08:30,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:30,981 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:08:30,981 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:08:35,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:40,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:45,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:08:45,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:08:45,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:08:46,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:51,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:56,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:08:58,828 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:09:00,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:09:00,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:09:00,636 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:09:01,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:06,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:11,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:15,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:09:15,595 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:09:15,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:09:16,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:21,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:26,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:28,829 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:09:30,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:09:30,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:09:30,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:09:32,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:37,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:42,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:44,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:09:44,352 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:09:44,352 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:09:44,353 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:09:45,027 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:09:45,027 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:09:45,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:09:45,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:09:45,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:09:47,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:52,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:57,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:09:58,829 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:10:00,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:10:00,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:10:00,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:10:03,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:08,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:13,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:15,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:10:15,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:10:15,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:10:19,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:24,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:28,830 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:10:29,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:30,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:10:30,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:10:30,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:10:34,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:39,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:44,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:45,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:10:45,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:10:45,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:10:49,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:54,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:10:58,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:10:58,592 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:10:58,593 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:10:58,594 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:10:58,831 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:10:59,071 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:10:59,071 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:11:00,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:11:00,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:11:00,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:11:00,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:05,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:10,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:15,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:11:15,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:11:15,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:11:16,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:21,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:26,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:28,832 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:11:30,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:11:30,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:11:30,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:11:31,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:36,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:41,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:45,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:11:45,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:11:45,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:11:47,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:52,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:57,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:11:58,833 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:12:00,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:12:00,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:12:00,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:12:02,790 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:07,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:12,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:12:12,597 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:12:12,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:12:12,599 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:12:13,115 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:12:13,115 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:12:13,637 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:15,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:12:15,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:12:15,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:12:18,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:23,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:28,834 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:12:29,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:30,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:12:30,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:12:30,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:12:35,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:40,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:45,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:12:45,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:12:45,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:12:45,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:50,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:55,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:12:58,835 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:13:00,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:13:00,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:13:00,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:13:01,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:06,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:11,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:15,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:13:15,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:13:15,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:13:17,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:22,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:26,795 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:13:26,797 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:13:26,798 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:13:26,799 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:13:27,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:13:27,158 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:13:28,836 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:13:28,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:30,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:13:30,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:13:30,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:13:34,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:39,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:44,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:45,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:13:45,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:13:45,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:13:49,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:54,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:13:58,837 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:13:59,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:00,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:14:00,597 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:14:00,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:14:04,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:09,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:14,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:15,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:14:15,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:14:15,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:14:20,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:25,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:28,838 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:14:30,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:14:30,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:14:30,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:14:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:35,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:40,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:14:40,726 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:14:40,727 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:14:40,728 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:14:41,204 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:14:41,204 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:14:41,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:45,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:14:45,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:14:45,715 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:14:46,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:51,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:56,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:14:58,839 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:15:00,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:15:00,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:15:00,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:15:01,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:06,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:11,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:15,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:15:15,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:15:15,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:15:17,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:22,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:27,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:28,840 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:15:30,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:15:30,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:15:30,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:15:33,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:38,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:43,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:45,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:15:45,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:15:45,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:15:48,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:53,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:15:55,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:15:55,380 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:15:55,381 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:15:55,382 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:15:56,253 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:15:57,254 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:15:58,841 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:15:58,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:00,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:16:00,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:16:00,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:16:03,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:08,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:13,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:15,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:16:15,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:16:15,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:16:19,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:24,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:28,842 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:16:29,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:30,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:16:30,598 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:16:30,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:16:35,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:40,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:45,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:16:45,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:16:45,716 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:16:45,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:50,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:55,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:16:58,843 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:17:00,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:17:00,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:17:00,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:17:01,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:06,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:09,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:17:09,318 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:17:09,319 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:17:09,320 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:17:10,297 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:17:12,298 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:17:12,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:15,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:17:15,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:17:15,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:17:17,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:22,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:27,794 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:28,843 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:17:30,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:17:30,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:17:30,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:17:32,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:37,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:42,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:45,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:17:45,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:17:45,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:17:48,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:53,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:58,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:17:58,845 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:18:00,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:18:00,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:18:00,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:18:03,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:08,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:13,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:15,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:18:15,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:18:15,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:18:19,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:23,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:18:23,545 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:18:23,545 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:18:23,546 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:18:24,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:18:24,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:18:25,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:26,346 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:18:27,347 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:18:28,845 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:18:30,348 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:18:30,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:18:30,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:18:30,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:18:30,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:35,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:40,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:45,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:18:45,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:18:45,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:18:46,833 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:51,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:56,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:18:58,846 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:19:00,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:19:00,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:19:00,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:19:02,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:07,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:12,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:15,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:19:15,599 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:19:15,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:19:18,765 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:23,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:28,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:28,848 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:19:30,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:19:30,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:19:30,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:19:33,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:38,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:41,440 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:19:41,442 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:19:41,442 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:19:41,443 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:19:42,393 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:19:44,394 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:19:44,482 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:45,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:19:45,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:19:45,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:19:49,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:54,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:19:58,848 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:19:59,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:00,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:20:00,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:20:00,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:20:05,775 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:10,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:15,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:20:15,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:20:15,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:20:16,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:21,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:26,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:28,849 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:20:30,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:20:30,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:20:30,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:20:31,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:36,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:41,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:45,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:20:45,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:20:45,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:20:47,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:52,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:55,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:20:55,823 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:20:55,823 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:20:55,824 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:20:56,436 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:20:57,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:20:58,437 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:20:58,850 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:21:00,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:21:00,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:21:00,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:21:03,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:08,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:13,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:15,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:21:15,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:21:15,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:21:19,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:24,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:28,851 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:21:29,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:30,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:21:30,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:21:30,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:21:35,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:40,755 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:45,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:21:45,600 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:21:45,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:21:45,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:50,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:55,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:21:58,852 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:22:00,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:22:00,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:22:00,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:22:00,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:05,793 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:09,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:22:09,816 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:22:09,816 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:22:09,817 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:22:10,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:22:10,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:12,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:22:15,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:22:15,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:22:15,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:22:16,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:21,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:26,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:28,853 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:22:30,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:22:30,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:22:30,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:22:31,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:36,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:41,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:45,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:22:45,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:22:45,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:22:47,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:52,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:57,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:22:58,854 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:23:00,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:23:00,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:23:00,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:23:02,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:07,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:12,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:15,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:23:15,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:23:15,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:23:17,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:22,951 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:23,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:23:23,992 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:23:23,992 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:23:23,994 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:23:24,527 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:23:26,528 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:23:28,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:28,855 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:23:30,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:23:30,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:23:30,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:23:33,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:38,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:43,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:45,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:23:45,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:23:45,719 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:23:49,824 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:54,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:23:58,856 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:23:59,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:00,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:24:00,601 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:24:00,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:24:05,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:10,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:15,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:24:15,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:24:15,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:24:15,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:20,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:25,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:28,857 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:24:30,601 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:24:30,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:24:30,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:24:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:35,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:38,405 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:24:38,407 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:24:38,407 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:24:38,408 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:24:38,583 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:24:40,584 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:24:41,450 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:45,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:24:45,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:24:45,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:24:46,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:51,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:56,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:24:58,859 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:25:00,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:25:00,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:25:00,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:25:02,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:07,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:12,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:15,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:25:15,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:25:15,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:25:17,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:22,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:27,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:28,859 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:25:30,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:25:30,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:25:30,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:25:32,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:37,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:42,932 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:45,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:25:45,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:25:45,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:25:48,800 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:52,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:25:52,478 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:25:52,478 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:25:52,480 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:25:52,629 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:25:54,517 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:25:54,630 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:25:58,859 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:25:59,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:00,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:26:00,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:26:00,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:26:05,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:10,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:15,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:26:15,602 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:26:15,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:26:16,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:21,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:26,841 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:28,861 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:26:30,602 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:26:30,603 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:26:30,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:26:32,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:37,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:42,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:45,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:26:45,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:26:45,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:26:47,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:52,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:57,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:26:58,863 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:27:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:27:00,604 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:27:00,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:27:02,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:06,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:27:06,858 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:27:06,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:27:06,860 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:27:07,674 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:27:07,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:08,675 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:27:12,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:27:15,604 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:27:15,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:27:18,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:23,810 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:28,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:28,863 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:27:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:27:30,604 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:27:30,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:27:34,750 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:39,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:44,751 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:27:45,604 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:27:45,721 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:27:49,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:54,838 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:27:58,864 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:27:59,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:28:00,604 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:28:00,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:28:05,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:10,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:28:15,604 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:28:15,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:28:15,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:20,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:20,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:28:20,992 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:28:20,993 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:28:20,994 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:28:21,721 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:28:22,721 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:28:26,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:28,865 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:28:30,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:28:30,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:28:30,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:28:31,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:36,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:41,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:45,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:28:45,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:28:45,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:28:46,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:51,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:56,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:28:58,866 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:29:00,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:29:00,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:29:00,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:29:02,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:07,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:12,771 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:15,604 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:29:15,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:29:15,722 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:29:17,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:22,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:27,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:28,867 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:29:30,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:29:30,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:29:30,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:29:33,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:34,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:29:34,979 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:29:34,979 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:29:34,980 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:29:35,766 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:29:36,767 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:29:39,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:44,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:45,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:29:45,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:29:45,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:29:49,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:54,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:29:58,868 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:29:59,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:00,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:30:00,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:30:00,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:30:05,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:10,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:15,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:30:15,606 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:30:15,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:30:16,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:21,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:26,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:28,869 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:30:30,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:30:30,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:30:30,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:30:31,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:36,781 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:41,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:45,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:30:45,605 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:30:45,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:30:46,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:48,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:30:48,944 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:30:48,945 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:30:48,946 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:30:49,520 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:30:49,813 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:30:50,813 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:30:52,205 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:52,815 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:30:54,816 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:30:57,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:30:58,870 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:31:00,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:31:00,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:31:00,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:31:03,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:08,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:13,773 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:15,606 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:31:15,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:31:15,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:31:19,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:24,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:28,871 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:31:29,872 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:30,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:31:30,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:31:30,723 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:31:35,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:40,774 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:45,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:31:45,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:31:45,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:31:45,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:50,839 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:55,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:31:58,872 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:32:00,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:32:00,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:32:00,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:32:01,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:06,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:07,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:32:07,393 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:32:07,393 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:32:07,394 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:32:07,863 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:32:08,864 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:32:12,435 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:15,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:32:15,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:32:15,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:32:17,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:22,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:27,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:28,873 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:32:30,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:32:30,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:32:30,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:32:33,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:38,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:43,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:45,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:32:45,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:32:45,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:32:48,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:53,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:58,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:32:58,873 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:33:00,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:33:00,607 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:33:00,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:33:04,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:09,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:14,780 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:15,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:33:15,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:33:15,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:33:19,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:21,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:33:21,596 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:33:21,596 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:33:21,598 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:33:21,910 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:33:22,911 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:33:25,638 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:28,874 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:33:30,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:33:30,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:33:30,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:33:30,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:35,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:40,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:45,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:33:45,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:33:45,724 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:33:46,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:51,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:56,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:33:58,875 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:34:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:34:00,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:34:00,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:34:01,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:06,791 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:11,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:34:15,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:34:15,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:34:17,776 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:22,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:27,777 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:28,877 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:34:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:34:30,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:34:30,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:34:32,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:35,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:34:35,860 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:34:35,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:34:35,862 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:34:35,965 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:34:36,966 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:34:37,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:42,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:34:45,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:34:45,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:34:48,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:53,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:58,823 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:34:58,878 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:35:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:35:00,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:35:00,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:35:04,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:09,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:14,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:35:15,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:35:15,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:35:20,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:25,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:28,879 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:35:30,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:35:30,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:35:30,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:35:30,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:35,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:40,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:45,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:35:45,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:35:45,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:35:45,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:49,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:35:49,948 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:35:49,948 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:35:49,949 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:35:50,011 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:35:50,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:51,011 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:35:55,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:35:58,880 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:36:00,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:36:00,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:36:00,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:36:01,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:06,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:11,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:15,608 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:36:15,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:36:15,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:36:16,806 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:21,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:28,881 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:36:30,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:36:30,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:36:30,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:36:32,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:37,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:42,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:45,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:36:45,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:36:45,725 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:36:48,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:53,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:58,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:36:58,881 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:37:00,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:37:00,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:37:00,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:37:03,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:03,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:37:03,814 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:37:03,814 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:37:03,815 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:37:04,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:37:05,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:37:08,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:13,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:15,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:37:15,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:37:15,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:37:19,798 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:24,799 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:28,882 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:37:29,883 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:30,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:37:30,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:37:30,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:37:35,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:40,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:45,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:37:45,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:37:45,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:37:46,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:51,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:56,779 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:37:58,883 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:38:00,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:38:00,609 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:38:00,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:38:01,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:06,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:11,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:15,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:38:15,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:38:15,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:38:17,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:18,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:38:18,012 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:38:18,013 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:38:18,014 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:38:18,104 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:38:19,104 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:38:23,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:28,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:28,884 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:38:30,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:38:30,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:38:30,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:38:33,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:38,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:43,830 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:45,609 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:38:45,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:38:45,726 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:38:48,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:53,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:58,832 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:38:58,885 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:39:00,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:39:00,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:39:00,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:39:04,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:09,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:14,821 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:15,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:39:15,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:39:15,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:39:20,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:25,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:28,886 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:39:30,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:39:30,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:39:30,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:39:31,772 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:32,208 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:39:32,210 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:39:32,210 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:39:32,211 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:39:33,150 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:39:33,150 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:39:37,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:42,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:45,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:39:45,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:39:45,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:39:47,817 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:52,818 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:57,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:39:58,887 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:40:00,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:40:00,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:40:00,727 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:40:02,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:07,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:12,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:15,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:40:15,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:40:15,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:40:18,766 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:23,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:28,767 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:28,889 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:40:30,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:40:30,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:40:30,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:40:34,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:39,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:44,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:45,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:40:45,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:40:45,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:40:46,792 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:40:46,794 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:40:46,794 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:40:46,795 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:40:47,196 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:40:49,197 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:40:49,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:54,834 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:40:58,889 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:40:59,889 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:00,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:41:00,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:41:00,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:41:05,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:10,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:15,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:41:15,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:41:15,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:41:16,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:21,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:26,788 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:28,890 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:41:30,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:41:30,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:41:30,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:41:31,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:36,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:41,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:45,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:41:45,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:41:45,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:41:47,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:52,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:57,812 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:41:58,890 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:42:00,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:42:00,611 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:42:00,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:42:00,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:42:00,806 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:42:00,806 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:42:00,808 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:42:01,241 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:42:02,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:03,242 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:42:07,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:12,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:15,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:42:15,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:42:15,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:42:18,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:23,783 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:28,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:28,891 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:42:30,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:42:30,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:42:30,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:42:33,804 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:38,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:43,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:45,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:42:45,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:42:45,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:42:48,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:53,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:58,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:42:58,892 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:43:00,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:43:00,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:43:00,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:43:03,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:08,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:13,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:14,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:43:14,830 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:43:14,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:43:14,832 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:43:15,286 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:43:15,406 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:43:15,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:43:15,778 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:43:15,820 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:43:17,288 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:43:19,291 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:43:19,815 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:24,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:28,893 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:43:29,894 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:30,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:43:30,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:43:30,756 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:43:34,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:39,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:44,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:45,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:43:45,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:43:45,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:43:50,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:55,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:43:58,894 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:44:00,706 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:44:00,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:44:00,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:44:01,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:06,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:11,875 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:15,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:44:15,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:44:15,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:44:16,959 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:21,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:26,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:28,895 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:44:30,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:44:30,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:44:30,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:44:32,591 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:44:34,269 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:44:34,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:34,269 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:44:34,271 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:44:34,338 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:44:35,339 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:44:39,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:44,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:45,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:44:45,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:44:45,757 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:44:49,905 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:54,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:44:58,896 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:45:00,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:45:00,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:45:00,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:45:02,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:07,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:12,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:15,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:45:15,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:45:15,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:45:18,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:23,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:28,897 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:45:29,898 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:30,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:45:30,707 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:45:30,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:45:35,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:40,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:45,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:45:45,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:45:45,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:45:45,948 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:46,492 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:45:46,494 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:45:46,494 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:45:46,495 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:45:47,384 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:45:47,384 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:45:51,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:56,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:45:58,898 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:46:00,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:46:00,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:46:00,758 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:46:01,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:06,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:11,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:15,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:46:15,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:46:15,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:46:16,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:21,936 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:26,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:28,899 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:46:30,707 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:46:30,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:46:30,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:46:32,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:37,927 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:42,928 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:45,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:46:45,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:46:45,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:46:48,890 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:53,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:58,891 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:46:58,899 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:47:00,393 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:47:00,394 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:47:00,395 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:47:00,396 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:47:00,430 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:47:00,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:47:00,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:47:00,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:47:01,430 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:47:03,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:08,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:13,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:15,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:47:15,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:47:15,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:47:18,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:23,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:28,902 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:47:29,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:30,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:47:30,708 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:47:30,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:47:35,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:40,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:45,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:47:45,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:47:45,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:47:46,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:51,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:56,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:47:58,901 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:48:00,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:48:00,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:48:00,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:48:01,876 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:06,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:11,877 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:14,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:48:14,469 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:48:14,469 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:48:14,471 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:48:14,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:48:15,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:48:15,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:48:15,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:48:15,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:48:16,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:21,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:26,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:28,902 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:48:30,708 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:48:30,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:48:30,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:48:31,907 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:36,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:41,908 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:45,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:48:45,709 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:48:45,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:48:46,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:51,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:56,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:48:58,904 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:49:00,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:49:00,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:49:00,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:49:02,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:07,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:12,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:15,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:49:15,742 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:49:15,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:49:17,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:22,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:27,958 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:28,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:49:28,905 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:49:28,906 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:49:28,907 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:49:28,908 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:49:29,533 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:49:29,534 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:49:30,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:49:30,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:49:30,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:49:32,966 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:37,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:42,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:45,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:49:45,742 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:49:45,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:49:47,991 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:52,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:57,993 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:49:58,906 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:50:00,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:50:00,742 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:50:00,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:50:03,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:08,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:13,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:15,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:50:15,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:50:15,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:50:19,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:24,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:28,907 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:50:30,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:50:30,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:50:30,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:50:30,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:35,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:40,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:42,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:50:42,970 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:50:42,970 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:50:42,971 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:50:43,583 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:50:43,583 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:50:45,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:50:45,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:50:45,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:50:45,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:50,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:55,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:50:58,908 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:51:00,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:51:00,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:51:00,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:51:01,900 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:06,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:11,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:15,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:51:15,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:51:15,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:51:16,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:21,945 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:26,946 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:28,909 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:51:30,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:51:30,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:51:30,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:51:31,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:36,971 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:41,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:45,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:51:45,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:51:45,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:51:46,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:51,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:56,944 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:51:56,946 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:51:56,947 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:51:56,948 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:51:56,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:51:57,627 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:51:57,628 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:51:58,910 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:52:00,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:52:00,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:52:00,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:52:02,005 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:07,005 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:12,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:15,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:52:15,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:52:15,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:52:17,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:22,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:27,996 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:28,910 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:52:30,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:52:30,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:52:30,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:52:33,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:38,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:43,995 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:45,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:52:45,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:52:45,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:52:49,930 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:54,931 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:52:58,911 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:53:00,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:53:00,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:53:00,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:53:00,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:05,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:10,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:53:10,803 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:53:10,803 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:53:10,804 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:53:11,678 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:53:11,679 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:53:11,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:15,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:53:15,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:53:15,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:53:16,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:21,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:26,961 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:28,912 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:53:30,743 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:53:30,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:53:30,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:53:32,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:37,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:42,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:45,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:53:45,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:53:45,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:53:47,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:52,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:57,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:53:58,913 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:54:00,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:54:00,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:54:00,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:54:02,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:07,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:12,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:15,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:54:15,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:54:15,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:54:18,952 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:23,953 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:25,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:54:25,086 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:54:25,087 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:54:25,102 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:54:25,724 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:54:25,724 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:54:28,914 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:54:29,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:30,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:54:30,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:54:30,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:54:34,992 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:39,993 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:44,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:45,744 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:54:45,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:54:45,784 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:54:50,915 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:55,916 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:54:58,915 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:55:00,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:55:00,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:55:00,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:55:01,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:06,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:11,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:15,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:55:15,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:55:15,842 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:55:16,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:21,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:26,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:28,916 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:55:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:55:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:55:30,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:55:32,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:37,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:39,245 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:55:39,248 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:55:39,248 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:55:39,250 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:55:39,772 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:55:39,772 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:55:39,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:55:41,773 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:55:42,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:43,774 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:55:45,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:55:45,842 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:55:45,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:55:48,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:53,010 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:58,011 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:55:58,917 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:56:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:56:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:56:00,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:56:03,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:08,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:13,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:15,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:56:15,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:56:15,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:56:18,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:23,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:28,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:28,918 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:56:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:56:30,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:56:30,884 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:56:33,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:38,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:43,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:45,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:56:45,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:56:45,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:56:48,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:53,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:57,718 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:56:57,720 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:56:57,720 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:56:57,721 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:56:57,823 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:56:58,761 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:56:58,919 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:56:59,825 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:57:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:57:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:57:00,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:57:04,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:09,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:14,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:15,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:57:15,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:57:15,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:57:20,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:25,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:28,920 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:57:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:57:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:57:30,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:57:36,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:41,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:45,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:57:45,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:57:45,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:57:47,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:52,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:57,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:57:58,921 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:58:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:58:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:58:00,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:58:02,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:07,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:11,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:58:11,859 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:58:11,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:58:11,860 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:58:11,875 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:58:12,901 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:14,876 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:58:15,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:58:15,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:58:15,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:58:18,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:23,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:28,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:28,922 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:58:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:58:30,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:58:30,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:58:34,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:39,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:44,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:45,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:58:45,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:58:45,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:58:49,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:54,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:58:58,923 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:58:59,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:00,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:59:00,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:59:00,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:59:05,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:10,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:15,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:15,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:59:15,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:59:15,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:59:21,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:25,831 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 04:59:25,834 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 04:59:25,834 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 04:59:25,835 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 04:59:25,921 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 04:59:26,836 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:26,921 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 04:59:28,924 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 04:59:30,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:59:30,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:59:30,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:59:32,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:37,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:42,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:45,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 04:59:45,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 04:59:45,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 04:59:47,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:52,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:57,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 04:59:58,924 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:00:00,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:00:00,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:00:00,844 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:00:03,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:08,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:13,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:15,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:00:15,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:00:15,845 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:00:18,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:23,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:28,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:28,925 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:00:30,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:00:30,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:00:30,845 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:00:34,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:39,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:39,929 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:00:39,931 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:00:39,932 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:00:39,933 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:00:39,965 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:00:40,966 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:00:44,934 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:45,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:00:45,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:00:45,845 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:00:50,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:55,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:00:58,926 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:01:00,844 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:00,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:01:00,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:01:00,845 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:01:06,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:11,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:15,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:01:15,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:01:15,845 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:01:17,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:22,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:27,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:28,927 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:01:30,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:01:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:01:30,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:01:32,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:37,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:42,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:45,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:01:45,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:01:45,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:01:48,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:53,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:54,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:01:54,165 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:01:54,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:01:54,168 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:01:55,011 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:01:57,012 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:01:58,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:01:58,928 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:02:00,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:02:00,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:02:00,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:02:04,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:09,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:14,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:15,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:02:15,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:02:15,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:02:20,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:25,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:28,929 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:02:30,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:02:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:02:30,846 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:02:36,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:41,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:45,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:02:45,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:02:45,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:02:46,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:51,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:56,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:02:58,931 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:03:00,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:03:00,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:03:00,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:03:02,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:07,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:08,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:03:08,059 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:03:08,060 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:03:08,062 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:03:09,058 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:03:09,058 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:03:12,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:15,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:03:15,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:03:15,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:03:17,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:22,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:27,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:28,930 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:03:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:03:30,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:03:30,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:03:33,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:38,017 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:43,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:45,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:03:45,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:03:45,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:03:48,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:53,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:58,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:03:58,932 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:04:00,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:04:00,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:04:00,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:04:03,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:08,192 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:13,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:15,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:04:15,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:04:15,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:04:19,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:22,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:04:22,422 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:04:22,423 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:04:22,425 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:04:23,107 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:04:24,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:25,109 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:04:28,932 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:04:29,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:30,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:04:30,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:04:30,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:04:35,005 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:40,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:45,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:45,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:04:45,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:04:45,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:04:50,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:55,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:04:58,933 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:05:00,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:00,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:05:00,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:05:00,847 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:05:06,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:11,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:15,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:05:15,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:05:15,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:05:16,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:21,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:26,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:28,934 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:05:30,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:05:30,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:05:30,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:05:31,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:36,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:41,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:45,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:05:45,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:05:45,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:05:47,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:49,648 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:05:49,650 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:05:49,650 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:05:49,652 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:05:50,164 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:05:51,165 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:05:52,653 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:57,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:05:58,935 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:06:00,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:06:00,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:06:00,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:06:03,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:08,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:13,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:15,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:06:15,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:06:15,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:06:19,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:24,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:28,936 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:06:29,937 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:30,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:06:30,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:06:30,848 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:06:35,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:40,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:45,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:45,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:06:45,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:06:45,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:06:51,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:56,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:06:58,937 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:07:00,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:07:00,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:07:00,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:07:02,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:07,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:12,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:15,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:07:15,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:07:15,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:07:17,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:22,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:27,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:28,938 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:07:30,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:07:30,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:07:30,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:07:33,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:38,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:38,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:07:38,656 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:07:38,656 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:07:38,658 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:07:39,230 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:07:41,232 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:07:43,659 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:45,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:07:45,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:07:45,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:07:49,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:54,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:07:58,939 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:07:59,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:00,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:08:00,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:08:00,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:08:05,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:10,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:15,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:15,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:08:15,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:08:15,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:08:21,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:26,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:28,940 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:08:30,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:08:30,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:08:30,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:08:32,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:37,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:42,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:45,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:08:45,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:08:45,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:08:47,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:52,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:57,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:08:58,941 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:09:00,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:09:00,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:09:00,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:09:02,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:07,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:12,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:15,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:09:15,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:09:15,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:09:18,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:23,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:23,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:09:23,813 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:09:23,813 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:09:23,815 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:09:24,292 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:09:24,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:09:25,293 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:09:27,295 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:09:28,942 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:09:28,943 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:29,296 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:09:30,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:09:30,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:09:30,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:09:34,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:39,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:44,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:45,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:09:45,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:09:45,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:09:50,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:55,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:09:58,944 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:10:00,848 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:00,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:10:00,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:10:00,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:10:06,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:11,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:15,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:10:15,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:10:15,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:10:16,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:21,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:26,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:28,944 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:10:30,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:10:30,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:10:30,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:10:32,785 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:37,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:42,787 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:45,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:10:45,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:10:45,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:10:48,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:51,422 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:10:51,424 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:10:51,424 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:10:51,426 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:10:52,345 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:10:53,346 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:10:53,427 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:58,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:10:58,945 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:11:00,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:11:00,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:11:00,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:11:03,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:08,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:13,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:15,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:11:15,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:11:15,851 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:11:18,895 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:23,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:28,896 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:28,946 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:11:30,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:11:30,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:11:30,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:11:34,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:39,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:44,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:45,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:11:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:11:45,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:11:50,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:55,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:11:58,947 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:12:00,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:00,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:12:00,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:12:00,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:12:05,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:12:05,613 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:12:05,613 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:12:05,615 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:12:06,394 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:12:06,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:07,395 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:12:11,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:15,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:12:15,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:12:15,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:12:17,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:22,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:27,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:28,948 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:12:30,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:12:30,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:12:30,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:12:33,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:38,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:43,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:45,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:12:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:12:45,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:12:48,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:53,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:58,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:12:58,949 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:13:00,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:13:00,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:13:00,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:13:04,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:09,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:14,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:15,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:13:15,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:13:15,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:13:19,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:20,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:13:20,329 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:13:20,329 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:13:20,331 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:13:20,444 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:13:21,444 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:13:24,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:28,949 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:13:29,950 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:30,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:13:30,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:13:30,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:13:35,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:40,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:45,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:45,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:13:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:13:45,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:13:50,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:55,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:13:58,950 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:14:00,849 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:00,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:14:00,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:14:00,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:14:06,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:11,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:15,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:14:15,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:14:15,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:14:16,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:21,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:26,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:28,951 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:14:30,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:14:30,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:14:30,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:14:32,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:34,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:14:34,108 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:14:34,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:14:34,110 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:14:34,492 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:14:35,493 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:14:37,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:42,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:14:45,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:14:45,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:14:48,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:53,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:58,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:14:58,952 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:15:00,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:15:00,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:15:00,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:15:04,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:09,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:14,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:15,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:15:15,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:15:15,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:15:20,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:25,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:28,953 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:15:30,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:30,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:15:30,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:15:30,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:15:36,083 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:41,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:15:45,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:15:45,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:15:47,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:47,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:15:47,935 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:15:47,936 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:15:47,937 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:15:48,536 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:15:49,537 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:15:52,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:57,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:15:58,954 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:16:00,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:16:00,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:16:00,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:16:03,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:08,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:13,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:15,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:16:15,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:16:15,853 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:16:19,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:24,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:28,955 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:16:29,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:30,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:16:30,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:16:30,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:16:35,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:40,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:45,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:45,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:16:45,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:16:45,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:16:50,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:55,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:16:58,956 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:17:00,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:00,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:17:00,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:17:00,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:17:01,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:17:01,848 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:17:01,849 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:17:01,850 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:17:02,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:17:03,581 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:17:05,851 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:10,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:15,852 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:17:15,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:17:15,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:17:16,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:21,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:26,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:28,957 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:17:30,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:17:30,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:17:30,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:17:31,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:36,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:41,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:45,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:17:45,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:17:45,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:17:46,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:51,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:56,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:17:58,958 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:18:00,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:18:00,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:18:00,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:18:02,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:07,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:12,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:15,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:18:15,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:18:15,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:18:15,941 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:18:16,031 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:18:16,031 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:18:16,056 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:18:16,626 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:18:17,627 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:18:18,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:23,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:28,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:28,959 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:18:30,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:18:30,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:18:30,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:18:34,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:39,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:44,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:45,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:18:45,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:18:45,854 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:18:49,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:54,096 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:18:58,961 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:18:59,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:00,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:19:00,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:19:00,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:19:05,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:10,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:15,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:15,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:19:15,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:19:15,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:19:20,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:25,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:28,961 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:19:29,837 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:19:29,839 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:19:29,841 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:19:29,842 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:19:30,671 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:19:30,843 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:30,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:19:30,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:19:30,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:19:31,672 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:19:36,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:41,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:45,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:19:45,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:19:45,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:19:47,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:52,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:57,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:19:58,962 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:20:00,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:20:00,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:20:00,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:20:03,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:08,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:13,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:15,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:20:15,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:20:15,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:20:18,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:23,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:28,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:28,963 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:20:30,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:20:30,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:20:30,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:20:33,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:38,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:43,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:45,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:20:45,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:20:45,855 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:20:49,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:54,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:20:58,964 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:20:59,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:00,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:21:00,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:21:00,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:21:02,018 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:21:02,020 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:21:02,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:21:02,045 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:21:02,730 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:21:03,730 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:21:05,046 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:10,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:15,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:15,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:21:15,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:21:15,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:21:20,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:25,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:28,964 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:21:30,853 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:30,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:21:30,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:21:30,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:21:36,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:41,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:45,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:21:45,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:21:45,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:21:47,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:52,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:57,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:21:58,965 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:22:00,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:22:00,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:22:00,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:22:03,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:08,030 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:13,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:15,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:22:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:22:15,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:22:18,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:23,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:28,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:28,966 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:22:30,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:22:30,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:22:30,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:22:34,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:39,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:44,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:45,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:22:45,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:22:45,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:22:49,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:51,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:22:51,058 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:22:51,059 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:22:51,060 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:22:51,802 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:22:51,802 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:22:51,871 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:22:53,803 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:22:54,551 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:22:55,804 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:22:58,967 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:22:59,968 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:00,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:23:00,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:23:00,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:23:05,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:10,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:15,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:15,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:23:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:23:15,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:23:20,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:25,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:28,968 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:23:30,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:30,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:23:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:23:30,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:23:36,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:41,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:45,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:23:45,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:23:45,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:23:46,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:51,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:56,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:23:58,969 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:24:00,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:24:00,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:24:00,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:24:01,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:06,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:11,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:24:15,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:24:15,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:24:17,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:22,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:27,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:28,970 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:24:30,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:24:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:24:30,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:24:32,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:37,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:38,734 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:24:38,737 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:24:38,737 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:24:38,739 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:24:38,867 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:24:39,868 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:24:42,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:45,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:24:45,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:24:45,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:24:48,034 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:53,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:58,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:24:58,971 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:25:00,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:25:00,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:25:00,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:25:04,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:09,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:14,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:25:15,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:25:15,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:25:19,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:24,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:28,972 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:25:29,972 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:30,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:25:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:25:30,857 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:25:35,956 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:40,957 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:45,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:25:45,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:25:45,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:25:46,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:51,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:56,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:25:57,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:25:57,870 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:25:57,871 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:25:57,872 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:25:57,916 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:25:58,973 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:25:59,917 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:26:00,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:26:00,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:26:00,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:26:02,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:07,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:12,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:15,856 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:26:15,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:26:15,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:26:18,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:23,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:28,039 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:28,975 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:26:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:26:30,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:26:30,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:26:33,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:38,063 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:43,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:45,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:26:45,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:26:45,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:26:48,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:53,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:58,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:26:58,974 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:27:00,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:27:00,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:27:00,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:27:04,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:09,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:11,914 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:27:11,916 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:27:11,916 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:27:11,919 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:27:11,961 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:27:13,962 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:27:14,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:15,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:27:15,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:27:15,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:27:20,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:25,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:28,975 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:27:30,855 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:27:30,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:27:30,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:27:36,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:41,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:45,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:27:45,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:27:45,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:27:46,094 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:51,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:56,095 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:27:58,977 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:28:00,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:28:00,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:28:00,858 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:28:02,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:07,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:12,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:15,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:28:15,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:28:15,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:28:17,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:22,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:25,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:28:25,883 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:28:25,884 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:28:25,885 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:28:26,007 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:28:27,886 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:28,008 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:28:28,977 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:28:30,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:28:30,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:28:30,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:28:33,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:38,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:43,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:45,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:28:45,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:28:45,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:28:49,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:54,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:28:58,978 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:28:59,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:00,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:29:00,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:29:00,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:29:05,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:10,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:15,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:15,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:29:15,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:29:15,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:29:21,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:26,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:28,979 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:29:30,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:29:30,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:29:30,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:29:32,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:37,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:40,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:29:40,749 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:29:40,750 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:29:40,751 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:29:41,055 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:29:42,055 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:29:42,752 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:45,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:29:45,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:29:45,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:29:48,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:53,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:58,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:29:58,980 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:30:00,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:30:00,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:30:00,859 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:30:03,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:08,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:13,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:15,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:30:15,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:30:15,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:30:18,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:23,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:28,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:28,981 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:30:30,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:30:30,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:30:30,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:30:33,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:38,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:43,157 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:45,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:30:45,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:30:45,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:30:49,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:54,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:30:55,207 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:30:55,209 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:30:55,209 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:30:55,236 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:30:56,103 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:30:56,103 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:30:58,982 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:30:59,982 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:00,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:31:00,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:31:00,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:31:05,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:10,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:15,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:15,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:31:15,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:31:15,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:31:21,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:26,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:28,982 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:31:30,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:31:30,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:31:30,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:31:31,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:36,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:41,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:45,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:31:45,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:31:45,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:31:47,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:52,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:57,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:31:58,984 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:32:00,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:32:00,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:32:00,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:32:02,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:07,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:12,087 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:15,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:32:15,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:32:15,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:32:17,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:22,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:27,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:28,984 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:32:30,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:32:30,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:32:30,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:32:32,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:37,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:41,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:32:41,195 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:32:41,196 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:32:41,197 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:32:42,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:32:42,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:32:42,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:45,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:32:45,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:32:45,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:32:48,084 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:53,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:58,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:32:58,985 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:33:00,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:33:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:33:00,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:33:03,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:08,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:13,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:15,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:33:15,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:33:15,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:33:19,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:24,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:28,986 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:33:29,987 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:30,859 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:33:30,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:33:30,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:33:35,088 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:40,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:45,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:45,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:33:45,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:33:45,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:33:50,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:55,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:33:58,987 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:34:00,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:34:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:34:00,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:34:06,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:11,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:15,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:34:15,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:34:15,861 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:34:17,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:22,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:27,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:28,988 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:34:29,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:34:29,354 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:34:29,355 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:34:29,356 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:34:30,243 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:34:30,243 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:34:30,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:34:30,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:34:30,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:34:33,024 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:38,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:43,026 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:45,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:34:45,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:34:45,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:34:48,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:53,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:58,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:34:58,990 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:35:00,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:35:00,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:35:00,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:35:03,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:08,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:13,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:15,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:35:15,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:35:15,862 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:35:19,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:24,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:28,990 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:35:29,990 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:30,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:35:30,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:35:30,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:35:35,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:40,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:45,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:45,860 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:35:45,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:35:45,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:35:51,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:56,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:35:58,991 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:36:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:36:00,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:36:00,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:36:01,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:06,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:11,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:12,847 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:36:12,849 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:36:12,850 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:36:12,852 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:36:13,309 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:36:14,310 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:36:15,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:36:15,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:36:15,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:36:17,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:22,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:27,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:28,991 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:36:30,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:36:30,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:36:30,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:36:33,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:38,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:43,042 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:45,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:36:45,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:36:45,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:36:48,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:53,048 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:58,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:36:58,992 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:37:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:37:00,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:37:00,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:37:03,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:08,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:13,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:15,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:37:15,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:37:15,863 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:37:19,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:24,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:28,993 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:37:29,994 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:30,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:37:30,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:37:30,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:37:35,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:40,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:45,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:45,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:37:45,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:37:45,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:37:50,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:53,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:37:53,679 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:37:53,680 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:37:53,681 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:37:54,379 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:37:54,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:37:55,381 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:37:55,455 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:37:57,382 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:37:58,994 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:37:59,384 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:38:00,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:38:00,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:00,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:38:00,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:38:06,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:11,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:15,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:38:15,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:38:15,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:38:16,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:21,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:26,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:28,995 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:38:30,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:38:30,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:38:30,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:38:32,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:37,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:42,092 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:45,861 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:38:45,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:38:45,864 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:38:48,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:53,040 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:58,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:38:58,996 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:39:00,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:39:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:39:00,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:39:04,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:09,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:14,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:15,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:39:15,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:39:15,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:39:19,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:24,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:26,509 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:39:26,512 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:39:26,513 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:39:26,514 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:39:27,447 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:39:28,997 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:39:29,448 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:39:29,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:30,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:39:30,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:39:30,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:39:35,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:40,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:45,016 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:45,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:39:45,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:39:45,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:39:50,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:55,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:39:58,998 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:40:00,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:40:00,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:40:00,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:40:06,071 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:11,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:15,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:40:15,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:40:15,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:40:16,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:21,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:26,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:28,999 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:40:30,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:40:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:40:30,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:40:32,097 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:37,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:40,586 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:40:40,588 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:40:40,588 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:40:40,590 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:40:41,495 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:40:42,590 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:43,496 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:40:45,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:40:45,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:40:45,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:40:48,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:53,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:58,082 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:40:59,000 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:41:00,862 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:41:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:41:00,865 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:41:03,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:08,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:13,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:15,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:41:15,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:41:15,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:41:18,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:23,101 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:28,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:29,001 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:41:30,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:41:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:41:30,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:41:33,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:38,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:43,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:45,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:41:45,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:41:45,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:41:49,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:54,060 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:41:54,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:41:54,540 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:41:54,541 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:41:54,542 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:41:54,543 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:41:55,543 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:41:59,001 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:42:00,003 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:00,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:42:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:42:00,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:42:05,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:10,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:15,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:15,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:42:15,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:42:15,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:42:20,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:25,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:29,002 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:42:30,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:42:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:30,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:42:30,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:42:36,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:41,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:45,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:42:45,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:42:45,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:42:46,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:51,085 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:56,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:42:59,004 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:43:00,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:43:00,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:43:00,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:43:02,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:07,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:08,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:43:08,327 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:43:08,327 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:43:08,329 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:43:08,591 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:43:09,592 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:43:12,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:15,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:43:15,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:43:15,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:43:18,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:23,089 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:28,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:29,004 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:43:30,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:43:30,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:43:30,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:43:33,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:38,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:43,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:45,863 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:43:45,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:43:45,866 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:43:49,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:54,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:43:59,006 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:44:00,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:00,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:44:00,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:44:00,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:44:05,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:10,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:15,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:15,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:44:15,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:44:15,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:44:21,023 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:22,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:44:22,575 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:44:22,576 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:44:22,577 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:44:22,639 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:44:23,640 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:44:26,578 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:29,006 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:44:30,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:44:30,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:44:30,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:44:32,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:37,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:42,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:45,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:44:45,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:44:45,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:44:47,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:52,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:57,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:44:59,037 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:45:00,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:45:00,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:45:00,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:45:02,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:07,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:12,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:15,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:45:15,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:45:15,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:45:18,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:23,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:28,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:29,038 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:45:30,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:45:30,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:45:30,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:45:34,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:39,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:44,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:45,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:45:45,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:45:45,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:45:46,431 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:45:46,434 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:45:46,434 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:45:46,436 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:45:46,694 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:45:47,695 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:45:49,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:54,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:45:59,040 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:46:00,041 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:00,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:46:00,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:46:00,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:46:05,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:10,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:15,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:15,864 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:46:15,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:46:15,867 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:46:21,031 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:26,032 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:29,042 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:46:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:46:30,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:46:30,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:46:31,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:36,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:41,057 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:45,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:46:45,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:46:45,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:46:46,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:51,069 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:56,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:46:59,043 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:47:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:47:00,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:47:00,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:47:01,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:06,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:11,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:15,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:47:15,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:47:15,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:47:17,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:22,013 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:27,014 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:29,044 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:47:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:47:30,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:47:30,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:47:32,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:35,840 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:47:35,843 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:47:35,843 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:47:35,845 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:47:36,770 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:47:37,771 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:47:37,845 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:42,846 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:45,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:47:45,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:47:45,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:47:48,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:53,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:58,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:47:59,045 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:48:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:48:00,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:48:00,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:48:03,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:08,074 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:13,075 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:15,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:48:15,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:48:15,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:48:19,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:24,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:29,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:29,046 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:48:30,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:48:30,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:48:30,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:48:34,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:39,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:44,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:45,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:48:45,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:48:45,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:48:50,028 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:55,029 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:48:59,047 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:49:00,047 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:00,865 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:49:00,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:49:00,868 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:49:05,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:10,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:15,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:15,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:49:15,868 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:49:15,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:49:21,090 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:22,382 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:49:22,384 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:49:22,385 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:49:22,386 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:49:22,837 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:49:23,838 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:49:26,387 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:29,048 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:49:30,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:49:30,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:49:30,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:49:32,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:37,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:42,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:45,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:49:45,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:49:45,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:49:48,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:53,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:58,051 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:49:59,048 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:50:00,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:50:00,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:50:00,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:50:04,036 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:09,037 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:14,038 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:15,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:50:15,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:50:15,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:50:19,061 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:24,062 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:29,050 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:50:30,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:30,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:50:30,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:50:30,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:50:35,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:40,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:45,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:45,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:50:45,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:50:45,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:50:51,102 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:56,103 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:50:59,051 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:51:00,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:51:00,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:51:00,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:51:02,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:04,067 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:51:04,070 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:51:04,070 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:51:04,071 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:51:04,903 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:51:05,904 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:51:07,072 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:12,073 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:15,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:51:15,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:51:15,869 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:51:17,105 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:22,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:27,106 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:29,053 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:51:30,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:51:30,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:51:30,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:51:33,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:38,049 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:43,050 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:45,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:51:45,869 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:51:45,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:51:48,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:53,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:58,108 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:51:59,052 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:52:00,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:52:00,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:52:00,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:52:04,076 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:09,077 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:14,078 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:15,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:52:15,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:52:15,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:52:20,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:25,025 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:29,053 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:52:30,055 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:30,867 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:52:30,870 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:52:30,870 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:52:35,064 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:40,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:43,066 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:52:43,068 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:52:43,069 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:52:43,070 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:52:43,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:52:43,965 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:52:43,965 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:52:45,460 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:45,966 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:52:46,027 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:52:46,027 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:52:46,068 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:52:46,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:52:47,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:52:51,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:56,239 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:52:59,056 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:53:00,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:53:00,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:53:00,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:53:02,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:07,198 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:12,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:15,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:53:15,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:53:15,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:53:18,145 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:23,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:28,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:29,057 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:53:30,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:53:30,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:53:30,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:53:33,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:38,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:43,170 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:45,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:53:45,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:53:45,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:53:49,162 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:54,163 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:53:59,058 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:54:00,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:00,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:54:00,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:54:00,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:54:05,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:07,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:54:07,244 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:54:07,245 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:54:07,246 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:54:08,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:54:08,020 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:54:10,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:15,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:15,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:54:15,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:54:15,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:54:21,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:26,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:29,058 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:54:30,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:54:30,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:54:30,998 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:54:32,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:37,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:42,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:45,983 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:54:45,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:54:45,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:54:47,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:52,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:57,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:54:59,059 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:55:00,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:55:00,998 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:55:00,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:55:02,248 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:07,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:12,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:15,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:55:15,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:55:15,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:55:18,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:21,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:55:21,537 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:55:21,538 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:55:21,539 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:55:22,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:55:23,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:24,069 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:55:28,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:29,060 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:55:30,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:55:30,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:55:30,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:55:34,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:39,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:44,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:45,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:55:45,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:55:45,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:55:50,158 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:55,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:55:59,061 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:56:00,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:00,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:56:00,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:56:00,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:56:06,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:11,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:15,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:56:15,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:56:15,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:56:17,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:22,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:27,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:29,062 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:56:30,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:56:30,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:56:30,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:56:32,247 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:35,495 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:56:35,497 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:56:35,497 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:56:35,499 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:56:36,115 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:56:37,500 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:38,117 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:56:42,501 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:45,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:56:45,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:56:45,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:56:48,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:53,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:58,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:56:59,063 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:57:00,984 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:57:00,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:57:00,999 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:57:03,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:08,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:13,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:15,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:57:15,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:57:16,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:57:18,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:23,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:28,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:29,064 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:57:30,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:57:30,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:57:31,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:57:34,189 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:39,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:44,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:45,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:57:45,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:57:46,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:57:49,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:49,467 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:57:49,469 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:57:49,470 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:57:49,471 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:57:50,164 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:57:50,164 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:57:54,472 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:57:59,065 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:58:00,065 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:00,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:58:01,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:58:01,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:58:05,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:10,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:15,238 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:15,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:58:16,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:58:16,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:58:21,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:26,155 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:29,066 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:58:30,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:58:31,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:58:31,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:58:31,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:36,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:41,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:45,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:58:46,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:58:46,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:58:46,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:51,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:56,184 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:58:59,068 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:59:00,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:59:01,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:59:01,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:59:01,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:03,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 05:59:03,346 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 05:59:03,346 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 05:59:03,348 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 05:59:04,214 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 05:59:04,215 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 05:59:06,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:11,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:15,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:59:16,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:59:16,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:59:17,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:22,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:27,183 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:29,068 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 05:59:30,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:59:31,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:59:31,000 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:59:32,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:37,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:42,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:45,985 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 05:59:46,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 05:59:46,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 05:59:48,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:53,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:58,186 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 05:59:59,069 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:00:00,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:00:01,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:00:01,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:00:04,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:09,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:14,169 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:15,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:00:16,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:00:16,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:00:17,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:00:17,471 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:00:17,472 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:00:17,473 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:00:18,261 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:00:18,261 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:00:19,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:24,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:29,069 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:00:30,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:30,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:00:31,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:00:31,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:00:35,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:40,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:45,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:45,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:00:46,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:00:46,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:00:50,220 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:55,221 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:00:59,071 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:01:00,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:00,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:01:01,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:01:01,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:01:06,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:11,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:15,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:01:16,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:01:16,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:01:17,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:22,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:27,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:29,072 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:01:30,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:01:31,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:01:31,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:01:31,471 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:01:32,960 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:01:32,960 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:32,960 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:01:32,962 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:01:33,307 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:01:34,308 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:01:37,963 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:42,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:45,986 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:01:46,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:01:46,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:01:48,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:53,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:58,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:01:59,072 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:02:00,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:02:01,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:02:01,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:02:04,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:09,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:14,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:15,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:02:16,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:02:16,001 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:02:20,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:25,190 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:29,073 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:02:30,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:30,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:02:31,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:02:31,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:02:36,166 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:41,167 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:45,563 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:02:45,565 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:02:45,565 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:02:45,567 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:02:45,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:02:46,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:02:46,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:02:46,354 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:02:46,354 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:02:47,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:52,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:57,148 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:02:59,074 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:03:00,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:03:01,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:03:01,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:03:02,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:07,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:12,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:15,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:03:16,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:03:16,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:03:18,153 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:23,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:28,154 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:29,075 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:03:30,988 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:03:31,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:03:31,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:03:33,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:38,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:43,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:45,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:03:46,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:03:46,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:03:49,200 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:54,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:59,076 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:03:59,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:03:59,575 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:03:59,575 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:03:59,576 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:03:59,577 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:04:00,399 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:04:00,399 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:04:00,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:04:01,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:04:01,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:04:05,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:10,177 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:15,178 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:15,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:04:16,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:04:16,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:04:20,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:25,223 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:29,077 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:04:30,980 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:30,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:04:31,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:04:31,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:04:36,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:41,226 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:45,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:04:46,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:04:46,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:04:47,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:52,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:57,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:04:59,078 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:05:00,989 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:05:01,002 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:05:01,002 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:05:02,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:07,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:12,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:13,458 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:05:13,460 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:05:13,460 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:05:13,462 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:05:14,035 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:05:14,445 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:05:14,446 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:05:16,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:05:16,258 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:05:16,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:05:16,447 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:05:17,469 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:18,448 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:05:22,512 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:27,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:29,079 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:05:31,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:05:31,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:05:31,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:05:33,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:38,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:43,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:46,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:05:46,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:05:46,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:05:49,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:54,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:05:59,081 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:06:00,081 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:01,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:06:01,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:06:01,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:06:05,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:10,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:15,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:16,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:06:16,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:06:16,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:06:21,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:26,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:29,082 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:06:31,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:06:31,006 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:06:31,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:06:31,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:06:31,423 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:06:31,423 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:06:31,424 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:31,425 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:06:31,493 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:06:32,494 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:06:36,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:41,466 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:46,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:06:46,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:06:46,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:06:47,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:52,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:57,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:06:59,082 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:07:01,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:07:01,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:07:01,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:07:03,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:08,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:13,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:16,006 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:07:16,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:07:16,258 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:07:19,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:24,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:29,085 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:07:30,086 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:31,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:07:31,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:07:31,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:07:35,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:40,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:45,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:45,483 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:07:45,485 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:07:45,485 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:07:45,487 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:07:45,538 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:07:46,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:07:46,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:07:46,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:07:46,539 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:07:51,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:56,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:07:59,084 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:08:01,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:08:01,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:08:01,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:08:02,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:07,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:12,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:16,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:08:16,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:08:16,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:08:17,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:22,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:27,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:29,085 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:08:31,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:08:31,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:08:31,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:08:33,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:38,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:43,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:46,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:08:46,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:08:46,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:08:49,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:54,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:59,086 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:08:59,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:08:59,495 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:08:59,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:08:59,496 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:08:59,497 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:08:59,582 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:09:00,583 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:09:01,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:09:01,007 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:09:01,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:09:05,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:10,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:15,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:16,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:09:16,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:09:16,259 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:09:21,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:26,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:29,086 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:09:31,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:09:31,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:09:31,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:09:32,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:37,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:42,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:46,007 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:09:46,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:09:46,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:09:48,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:53,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:58,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:09:59,088 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:10:01,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:10:01,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:10:01,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:10:04,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:09,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:13,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:10:13,325 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:10:13,325 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:10:13,327 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:10:13,627 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:10:14,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:14,628 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:10:16,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:10:16,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:10:16,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:10:20,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:25,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:29,088 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:10:31,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:10:31,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:10:31,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:31,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:10:36,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:41,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:46,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:10:46,008 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:10:46,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:10:47,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:52,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:57,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:10:59,089 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:11:01,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:11:01,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:11:01,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:11:03,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:08,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:13,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:16,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:11:16,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:11:16,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:11:19,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:24,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:27,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:11:27,809 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:11:27,809 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:11:27,811 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:11:28,683 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:11:28,683 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:11:29,090 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:11:30,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:31,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:11:31,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:11:31,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:11:35,266 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:40,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:45,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:46,008 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:11:46,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:11:46,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:11:51,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:56,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:11:59,091 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:12:01,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:12:01,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:12:01,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:12:04,140 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:09,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:14,141 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:16,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:12:16,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:12:16,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:12:19,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:24,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:29,092 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:12:30,093 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:31,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:12:31,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:12:31,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:12:35,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:40,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:41,665 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:12:41,667 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:12:41,667 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:12:41,669 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:12:41,729 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:12:42,730 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:12:45,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:46,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:12:46,009 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:12:46,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:12:51,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:56,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:12:59,093 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:13:01,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:13:01,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:13:01,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:13:02,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:07,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:12,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:16,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:13:16,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:13:16,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:13:18,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:23,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:28,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:29,094 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:13:31,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:13:31,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:13:31,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:13:34,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:39,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:44,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:46,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:13:46,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:13:46,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:13:50,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:55,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:13:55,580 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:13:55,582 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:13:55,582 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:13:55,583 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:13:55,779 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:13:56,779 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:13:59,096 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:14:01,009 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:14:01,010 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:14:01,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:01,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:14:06,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:11,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:16,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:14:16,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:14:16,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:14:17,267 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:22,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:27,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:29,096 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:14:31,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:14:31,019 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:14:31,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:14:33,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:38,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:43,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:46,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:14:46,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:14:46,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:14:49,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:54,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:14:59,097 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:15:00,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:01,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:15:01,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:15:01,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:15:05,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:09,668 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:15:09,670 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:15:09,670 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:15:09,672 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:15:09,862 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:15:10,709 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:10,863 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:15:15,710 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:16,019 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:15:16,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:15:16,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:15:21,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:26,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:29,098 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:15:31,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:15:31,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:15:31,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:15:32,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:37,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:42,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:46,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:15:46,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:15:46,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:15:48,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:53,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:58,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:15:59,099 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:16:01,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:16:01,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:16:01,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:16:04,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:09,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:14,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:16,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:16:16,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:16:16,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:16:20,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:23,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:16:23,612 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:16:23,612 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:16:23,613 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:16:23,910 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:16:24,910 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:16:25,654 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:29,099 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:16:31,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:16:31,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:16:31,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:31,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:16:36,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:41,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:46,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:16:46,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:16:46,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:16:46,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:51,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:56,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:16:59,100 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:17:01,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:17:01,020 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:17:01,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:17:02,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:07,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:12,271 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:16,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:17:16,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:17:16,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:17:18,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:23,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:28,270 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:29,101 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:17:31,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:17:31,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:17:31,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:17:34,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:37,404 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:17:37,406 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:17:37,406 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:17:37,407 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:17:37,956 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:17:37,981 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:17:38,957 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:17:39,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:40,958 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:17:41,959 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:17:42,959 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:17:44,585 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:46,020 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:17:46,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:17:46,263 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:17:50,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:55,269 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:17:59,102 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:18:01,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:18:01,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:18:01,241 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:01,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:18:06,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:11,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:16,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:18:16,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:18:16,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:18:17,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:22,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:27,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:29,103 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:18:31,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:18:31,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:18:31,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:18:33,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:38,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:43,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:46,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:18:46,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:18:46,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:18:49,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:54,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:18:55,819 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:18:55,821 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:18:55,821 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:18:55,823 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:18:56,006 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:18:58,007 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:18:59,104 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:19:00,104 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:01,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:19:01,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:19:01,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:19:05,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:10,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:15,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:16,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:19:16,021 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:19:16,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:19:21,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:26,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:29,104 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:19:31,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:19:31,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:19:31,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:19:32,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:37,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:42,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:46,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:19:46,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:19:46,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:19:48,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:53,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:58,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:19:59,105 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:20:01,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:20:01,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:20:01,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:20:04,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:09,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:09,935 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:20:09,937 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:20:09,937 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:20:09,938 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:20:10,054 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:20:12,055 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:20:14,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:16,021 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:20:16,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:20:16,273 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:20:20,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:25,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:29,106 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:20:31,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:20:31,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:20:31,261 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:31,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:20:36,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:41,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:46,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:20:46,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:20:46,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:20:47,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:52,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:57,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:20:59,107 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:21:01,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:21:01,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:21:01,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:21:03,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:08,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:13,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:16,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:21:16,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:21:16,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:21:19,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:23,850 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:21:23,852 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:21:23,852 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:21:23,853 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:21:24,106 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:21:24,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:26,107 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:21:29,109 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:21:30,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:31,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:21:31,022 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:21:31,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:21:35,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:40,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:45,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:46,022 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:21:46,023 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:21:46,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:21:51,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:56,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:21:59,109 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:22:01,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:22:01,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:22:01,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:22:01,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:06,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:11,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:16,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:22:16,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:22:16,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:22:17,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:22,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:27,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:29,111 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:22:31,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:22:31,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:22:31,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:22:32,313 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:37,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:37,822 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:22:37,824 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:22:37,824 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:22:37,825 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:22:38,169 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:22:40,170 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:22:42,866 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:46,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:22:46,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:22:46,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:22:48,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:53,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:58,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:22:59,111 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:23:01,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:23:01,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:23:01,274 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:23:04,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:09,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:14,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:16,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:23:16,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:23:16,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:23:19,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:24,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:29,114 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:23:30,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:31,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:23:31,109 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:23:31,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:23:35,335 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:40,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:45,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:46,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:23:46,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:23:46,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:23:50,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:51,672 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:23:51,674 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:23:51,675 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:23:51,676 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:23:52,215 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:23:54,216 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:23:55,677 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:23:59,115 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:24:01,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:24:01,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:24:01,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:24:01,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:06,287 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:11,288 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:16,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:24:16,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:24:16,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:24:16,295 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:21,296 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:26,297 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:29,116 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:24:31,109 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:24:31,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:24:31,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:24:33,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:38,079 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:43,080 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:46,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:24:46,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:24:46,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:24:48,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:53,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:58,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:24:59,117 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:25:01,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:25:01,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:25:01,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:25:03,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:05,739 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:25:05,741 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:25:05,741 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:25:05,742 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:25:06,272 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:25:08,273 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:25:08,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:13,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:16,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:25:16,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:25:16,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:25:19,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:24,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:29,117 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:25:30,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:31,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:25:31,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:25:31,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:25:35,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:40,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:45,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:46,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:25:46,110 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:25:46,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:25:51,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:56,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:25:59,118 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:26:01,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:26:01,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:26:01,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:26:01,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:06,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:11,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:16,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:26:16,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:26:16,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:26:17,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:19,605 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:26:19,608 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:26:19,608 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:26:19,610 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:26:20,331 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:26:22,333 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:26:22,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:27,611 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:29,120 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:26:31,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:26:31,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:26:31,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:26:33,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:38,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:43,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:46,110 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:26:46,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:26:46,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:26:49,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:54,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:26:59,121 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:27:00,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:01,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:27:01,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:27:01,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:27:05,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:10,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:15,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:16,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:27:16,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:27:16,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:27:20,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:25,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:29,122 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:27:31,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:27:31,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:27:31,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:27:31,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:33,762 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:27:33,765 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:27:33,766 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:27:33,767 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:27:34,381 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:27:36,383 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:27:36,768 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:41,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:46,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:27:46,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:27:46,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:27:47,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:52,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:57,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:27:59,123 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:28:01,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:28:01,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:28:01,276 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:28:02,807 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:07,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:12,809 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:16,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:28:16,111 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:28:16,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:28:18,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:23,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:28,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:29,125 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:28:31,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:28:31,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:28:31,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:28:34,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:39,314 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:44,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:46,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:28:46,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:28:46,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:28:47,949 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:28:47,951 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:28:47,952 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:28:47,954 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:28:48,430 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:28:49,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:50,431 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:28:54,955 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:28:59,125 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:29:00,127 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:01,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:29:01,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:29:01,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:29:05,289 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:10,290 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:15,291 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:16,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:29:16,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:29:16,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:29:20,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:25,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:29,127 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:29:31,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:29:31,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:29:31,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:31,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:29:36,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:41,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:46,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:29:46,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:29:46,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:29:46,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:51,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:56,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:29:59,127 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:30:01,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:30:01,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:30:01,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:30:01,964 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:30:01,966 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:30:01,966 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:30:01,967 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:01,968 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:30:02,480 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:30:02,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:30:04,481 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:30:06,482 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:30:07,483 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:30:07,892 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:08,484 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:30:12,893 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:16,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:30:16,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:30:16,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:30:18,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:23,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:28,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:29,128 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:30:31,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:30:31,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:30:31,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:30:34,284 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:39,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:44,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:46,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:30:46,112 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:30:46,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:30:50,285 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:55,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:30:59,130 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:31:01,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:31:01,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:31:01,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:31:01,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:06,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:11,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:16,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:31:16,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:31:16,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:31:17,280 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:19,829 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:31:19,831 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:31:19,831 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:31:19,832 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:31:20,529 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:31:22,530 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:31:22,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:27,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:29,130 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:31:31,112 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:31:31,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:31:31,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:31:33,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:38,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:43,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:46,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:31:46,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:31:46,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:31:48,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:53,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:58,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:31:59,131 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:32:01,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:32:01,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:32:01,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:32:04,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:09,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:14,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:16,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:32:16,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:32:16,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:32:19,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:24,311 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:29,132 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:32:30,133 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:31,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:32:31,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:32:31,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:32:33,740 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:32:33,743 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:32:33,743 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:32:33,745 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:32:34,576 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:32:34,577 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:32:35,745 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:40,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:45,747 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:46,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:32:46,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:32:46,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:32:51,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:56,352 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:32:59,133 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:33:01,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:33:01,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:33:01,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:33:02,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:07,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:12,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:16,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:33:16,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:33:16,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:33:18,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:23,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:28,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:29,134 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:33:31,113 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:33:31,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:33:31,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:33:34,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:39,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:44,282 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:46,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:33:46,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:33:46,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:33:47,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:33:47,813 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:33:47,814 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:33:47,815 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:33:48,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:33:48,624 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:33:49,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:54,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:33:59,135 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:34:00,136 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:01,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:34:01,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:34:01,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:34:05,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:10,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:15,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:16,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:34:16,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:34:16,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:34:21,315 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:26,316 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:29,136 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:34:31,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:34:31,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:34:31,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:34:32,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:37,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:42,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:46,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:34:46,114 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:34:46,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:34:47,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:52,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:57,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:34:59,137 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:35:01,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:35:01,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:35:01,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:35:01,643 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:35:01,645 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:35:01,645 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:35:01,646 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:35:01,671 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:35:02,671 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:35:02,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:07,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:12,687 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:16,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:35:16,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:35:16,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:35:18,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:23,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:28,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:29,138 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:35:31,114 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:35:31,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:35:31,279 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:35:33,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:38,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:43,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:46,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:35:46,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:35:46,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:35:48,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:53,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:58,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:35:59,139 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:36:01,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:36:01,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:36:01,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:36:04,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:09,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:14,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:16,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:36:16,060 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:36:16,060 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:36:16,062 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:36:16,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:36:16,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:36:16,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:36:16,738 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:36:18,739 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:36:19,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:24,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:29,140 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:36:30,142 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:31,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:36:31,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:36:31,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:36:35,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:40,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:45,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:46,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:36:46,115 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:36:46,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:36:51,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:56,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:36:59,142 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:37:01,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:37:01,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:37:01,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:37:02,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:07,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:12,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:16,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:37:16,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:37:16,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:37:17,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:22,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:27,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:29,142 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:37:29,888 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:37:29,890 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:37:29,890 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:37:29,893 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:37:30,784 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:37:30,784 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:37:31,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:37:31,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:37:31,317 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:37:33,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:38,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:43,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:46,115 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:37:46,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:37:46,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:37:49,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:54,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:37:59,143 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:38:00,143 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:01,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:38:01,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:38:01,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:38:05,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:10,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:15,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:16,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:38:16,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:38:16,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:38:21,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:26,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:29,145 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:38:31,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:38:31,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:38:31,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:38:31,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:36,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:41,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:43,720 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:38:43,722 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:38:43,722 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:38:43,723 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:38:43,828 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:38:44,829 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:38:46,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:38:46,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:38:46,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:38:47,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:52,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:57,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:38:59,144 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:39:01,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:39:01,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:39:01,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:39:03,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:08,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:13,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:16,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:39:16,116 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:39:16,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:39:19,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:24,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:29,146 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:39:30,147 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:31,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:39:31,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:39:31,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:39:35,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:40,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:45,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:46,116 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:39:46,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:39:46,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:39:51,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:56,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:39:57,760 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:39:57,762 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:39:57,762 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:39:57,764 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:39:57,876 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:39:58,877 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:39:59,146 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:40:01,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:40:01,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:40:01,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:40:01,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:06,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:11,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:16,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:40:16,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:40:16,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:40:17,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:22,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:27,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:29,147 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:40:31,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:40:31,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:40:31,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:40:33,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:38,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:43,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:46,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:40:46,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:40:46,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:40:49,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:54,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:40:59,148 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:41:00,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:01,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:41:01,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:41:01,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:41:05,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:10,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:11,702 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:41:11,704 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:41:11,704 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:41:11,706 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:41:11,923 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:41:12,923 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:41:15,746 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:16,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:41:16,117 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:41:16,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:41:21,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:26,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:29,149 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:41:31,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:41:31,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:41:31,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:41:32,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:37,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:42,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:46,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:41:46,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:41:46,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:41:47,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:52,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:57,351 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:41:59,150 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:42:01,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:42:01,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:42:01,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:42:02,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:07,827 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:12,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:16,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:42:16,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:42:16,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:42:18,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:23,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:25,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:42:25,533 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:42:25,534 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:42:25,535 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:42:25,968 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:42:26,107 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:42:26,969 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:42:28,805 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:28,971 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:42:29,151 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:42:29,972 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:42:30,973 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:42:31,117 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:42:31,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:42:31,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:42:34,731 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:39,732 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:44,733 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:46,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:42:46,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:42:46,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:42:50,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:55,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:42:59,152 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:43:01,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:43:01,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:43:01,298 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:01,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:43:06,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:11,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:16,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:43:16,118 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:43:16,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:43:16,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:21,348 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:26,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:29,153 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:43:31,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:43:31,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:43:31,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:43:32,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:37,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:42,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:43,673 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:43:43,674 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:43:43,675 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:43:43,676 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:43:44,018 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:43:45,019 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:43:46,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:43:46,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:43:46,320 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:43:47,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:52,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:57,342 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:43:59,156 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:44:01,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:44:01,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:44:01,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:44:03,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:08,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:13,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:16,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:44:16,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:44:16,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:44:19,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:24,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:29,155 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:44:30,156 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:31,118 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:44:31,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:44:31,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:44:35,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:40,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:45,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:46,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:44:46,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:44:46,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:44:51,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:56,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:44:57,939 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:44:57,941 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:44:57,941 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:44:57,943 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:44:58,067 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:44:59,068 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:44:59,156 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:45:01,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:45:01,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:45:01,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:45:02,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:07,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:12,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:16,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:45:16,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:45:16,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:45:18,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:23,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:28,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:29,158 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:45:31,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:45:31,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:45:31,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:45:33,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:38,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:43,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:46,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:45:46,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:45:46,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:45:49,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:54,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:45:59,158 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:46:00,159 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:01,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:46:01,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:46:01,321 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:46:05,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:10,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:12,015 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:46:12,017 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:46:12,017 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:46:12,019 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:46:12,113 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:46:13,127 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:46:16,058 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:16,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:46:16,119 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:46:16,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:46:21,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:26,383 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:29,159 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:46:31,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:46:31,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:46:31,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:46:32,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:37,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:42,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:46,119 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:46:46,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:46:46,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:46:47,353 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:52,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:57,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:46:59,160 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:47:01,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:47:01,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:47:01,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:47:02,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:07,364 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:12,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:16,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:47:16,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:47:16,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:47:18,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:23,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:25,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:47:25,919 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:47:25,919 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:47:25,920 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:47:26,174 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:47:27,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:47:28,962 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:29,161 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:47:31,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:47:31,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:47:31,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:47:34,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:39,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:44,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:46,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:47:46,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:47:46,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:47:50,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:55,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:47:59,162 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:48:01,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:48:01,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:48:01,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:48:01,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:06,357 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:11,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:16,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:48:16,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:48:16,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:48:17,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:22,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:27,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:29,162 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:48:31,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:48:31,120 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:48:31,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:48:33,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:38,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:39,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:48:39,837 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:48:39,837 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:48:39,838 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:48:40,221 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:48:41,222 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:48:43,878 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:46,120 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:48:46,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:48:46,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:48:49,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:54,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:48:59,163 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:49:00,165 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:01,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:49:01,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:49:01,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:49:05,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:10,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:15,338 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:16,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:49:16,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:49:16,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:49:21,336 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:26,337 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:29,164 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:49:31,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:49:31,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:49:31,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:49:32,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:37,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:42,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:46,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:49:46,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:49:46,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:49:47,344 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:52,345 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:53,736 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:49:53,738 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:49:53,738 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:49:53,739 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:49:54,275 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:49:55,276 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:49:57,778 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:49:59,165 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:50:01,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:50:01,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:50:01,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:50:03,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:08,377 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:13,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:16,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:50:16,121 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:50:16,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:50:19,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:24,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:29,166 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:50:30,168 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:31,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:50:31,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:50:31,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:50:35,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:40,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:45,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:46,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:50:46,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:50:46,323 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:50:50,374 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:55,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:50:59,167 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:51:01,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:51:01,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:51:01,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:51:01,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:06,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:07,813 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:51:07,815 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:51:07,816 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:51:07,817 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:51:08,335 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:51:09,335 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:51:11,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:16,121 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:51:16,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:51:16,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:51:17,367 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:22,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:27,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:29,168 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:51:31,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:51:31,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:51:31,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:51:33,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:38,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:43,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:46,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:51:46,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:51:46,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:51:49,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:54,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:51:59,169 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:52:00,171 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:01,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:52:01,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:52:01,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:52:05,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:10,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:15,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:16,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:52:16,122 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:52:16,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:52:21,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:22,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:52:22,281 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:52:22,282 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:52:22,283 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:52:22,382 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:52:23,383 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:52:27,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:29,170 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:52:31,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:52:31,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:52:31,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:52:32,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:37,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:42,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:46,122 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:52:46,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:52:46,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:52:47,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:52,340 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:57,341 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:52:59,171 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:53:01,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:53:01,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:53:01,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:53:03,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:08,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:13,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:16,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:53:16,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:53:16,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:53:19,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:24,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:29,172 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:53:30,173 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:31,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:53:31,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:53:31,324 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:53:35,339 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:36,322 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:53:36,324 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:53:36,324 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:53:36,325 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:53:36,426 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:53:37,427 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:53:40,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:45,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:46,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:53:46,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:53:46,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:53:51,332 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:56,333 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:53:59,173 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:54:01,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:54:01,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:54:01,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:54:01,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:06,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:11,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:16,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:54:16,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:54:16,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:54:17,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:22,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:27,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:29,174 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:54:31,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:54:31,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:54:31,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:54:33,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:38,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:43,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:46,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:54:46,123 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:54:46,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:54:49,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:50,494 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:54:50,495 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:54:50,496 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:54:50,497 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:54:51,070 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:54:51,472 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:54:51,472 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:54:53,473 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:54:54,474 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:54:54,678 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:54:55,474 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:54:59,174 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:55:00,176 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:01,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:55:01,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:55:01,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:55:05,999 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:11,000 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:16,001 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:16,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:55:16,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:55:16,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:55:21,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:26,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:29,175 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:55:31,123 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:55:31,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:55:31,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:55:32,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:37,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:42,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:46,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:55:46,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:55:46,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:55:48,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:53,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:58,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:55:59,176 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:56:01,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:56:01,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:56:01,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:56:04,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:08,782 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:56:08,784 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:56:08,784 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:56:08,786 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:56:09,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:56:09,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:56:09,825 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:14,826 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:16,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:56:16,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:56:16,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:56:20,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:25,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:29,177 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:56:31,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:56:31,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:56:31,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:56:31,354 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:36,355 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:41,356 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:46,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:56:46,124 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:56:46,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:56:47,325 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:52,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:57,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:56:59,179 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:57:01,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:57:01,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:57:01,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:57:02,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:07,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:12,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:16,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:57:16,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:57:16,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:57:18,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:22,814 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:57:22,816 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:57:22,816 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:57:22,818 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:57:23,564 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:57:23,565 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:57:23,857 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:28,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:29,179 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:57:31,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:57:31,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:57:31,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:57:34,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:39,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:44,331 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:46,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:57:46,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:57:46,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:57:49,365 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:54,366 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:57:59,180 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:58:00,182 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:01,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:58:01,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:58:01,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:58:05,389 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:10,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:15,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:16,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:58:16,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:58:16,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:58:21,326 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:26,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:29,181 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:58:31,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:58:31,149 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:58:31,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:58:31,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:36,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:36,899 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:58:36,901 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:58:36,901 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:58:36,902 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:58:37,612 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:58:37,612 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:58:41,942 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:46,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:58:46,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:58:46,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:58:47,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:52,346 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:57,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:58:59,182 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:59:01,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:59:01,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:59:01,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:59:03,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:08,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:13,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:16,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:59:16,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:59:16,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:59:18,378 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:23,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:28,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:29,184 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 06:59:31,149 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:59:31,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:59:31,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:59:34,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:39,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:44,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:46,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 06:59:46,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 06:59:46,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 06:59:49,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:50,858 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 06:59:50,860 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 06:59:50,860 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 06:59:50,861 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 06:59:51,656 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 06:59:52,657 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 06:59:54,902 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 06:59:59,184 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:00:00,185 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:01,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:00:01,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:00:01,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:00:05,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:10,372 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:15,373 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:16,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:00:16,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:00:16,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:00:20,412 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:25,413 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:29,186 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:00:31,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:00:31,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:00:31,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:00:31,375 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:36,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:41,376 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:46,150 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:00:46,150 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:00:46,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:00:47,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:52,349 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:57,350 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:00:59,186 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:01:01,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:01:01,211 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:01:01,327 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:01:02,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:05,319 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:01:05,321 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:01:05,321 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:01:05,322 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:01:05,706 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:01:06,707 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:01:08,361 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:13,362 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:16,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:01:16,211 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:01:16,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:01:18,461 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:23,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:28,462 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:29,187 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:01:31,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:01:31,211 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:01:31,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:01:34,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:39,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:44,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:46,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:01:46,211 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:01:46,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:01:50,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:55,363 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:01:59,188 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:02:01,188 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:01,211 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:02:01,211 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:02:01,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:02:06,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:11,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:16,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:02:16,212 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:02:16,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:02:16,428 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:19,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:02:19,349 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:02:19,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:02:19,350 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:02:19,753 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:02:20,754 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:02:22,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:27,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:29,188 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:02:31,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:02:31,212 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:02:31,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:02:32,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:37,447 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:42,448 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:46,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:02:46,212 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:02:46,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:02:48,429 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:53,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:58,430 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:02:59,189 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:03:01,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:03:01,212 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:03:01,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:03:04,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:09,395 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:14,396 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:16,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:03:16,212 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:03:16,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:03:20,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:25,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:29,190 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:03:31,191 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:31,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:03:31,212 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:03:31,328 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:03:33,426 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:03:33,428 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:03:33,428 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:03:33,429 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:03:33,800 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:03:34,801 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:03:36,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:41,470 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:46,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:03:46,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:03:46,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:03:47,436 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:52,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:57,437 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:03:59,191 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:04:01,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:04:01,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:04:01,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:04:03,414 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:08,415 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:13,416 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:16,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:04:16,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:04:16,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:04:19,370 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:24,371 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:29,192 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:04:30,193 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:31,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:04:31,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:04:31,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:04:35,368 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:40,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:45,369 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:46,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:04:46,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:04:46,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:04:47,505 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:04:47,507 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:04:47,507 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:04:47,508 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:04:47,847 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:04:48,848 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:04:50,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:55,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:04:59,193 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:05:01,194 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:01,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:05:01,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:05:01,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:05:06,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:11,392 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:16,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:05:16,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:05:16,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:05:17,358 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:22,359 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:27,360 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:29,194 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:05:31,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:05:31,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:05:31,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:05:32,419 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:37,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:42,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:46,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:05:46,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:05:46,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:05:48,379 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:53,380 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:58,381 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:05:59,195 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:06:01,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:06:01,213 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:06:01,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:06:01,759 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:06:01,760 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:06:01,760 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:06:01,762 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:06:01,894 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:06:02,894 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:06:03,801 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:08,802 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:13,803 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:16,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:06:16,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:06:16,329 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:06:19,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:24,407 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:29,196 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:06:30,196 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:31,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:06:31,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:06:31,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:06:35,408 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:40,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:45,409 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:46,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:06:46,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:06:46,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:06:50,420 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:55,421 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:06:59,197 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:07:01,197 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:01,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:07:01,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:07:01,330 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:07:06,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:11,474 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:15,639 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:07:15,641 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:07:15,641 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:07:15,642 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:07:15,939 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:07:16,213 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:07:16,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:07:16,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:07:16,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:07:16,610 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:16,940 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:07:18,941 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:07:19,942 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:07:20,942 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:07:21,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:26,808 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:29,198 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:07:31,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:07:31,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:07:31,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:07:32,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:37,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:42,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:46,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:07:46,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:07:46,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:07:48,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:53,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:58,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:07:59,199 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:08:01,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:08:01,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:08:01,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:08:04,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:09,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:14,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:16,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:08:16,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:08:16,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:08:20,522 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:25,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:29,200 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:08:31,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:31,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:08:31,214 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:08:31,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:08:34,347 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:08:34,348 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:08:34,349 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:08:34,350 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:08:34,990 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:08:36,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:36,991 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:08:41,390 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:46,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:08:46,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:08:46,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:46,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:08:51,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:56,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:08:59,201 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:09:01,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:09:01,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:09:01,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:09:02,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:07,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:12,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:16,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:09:16,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:09:16,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:09:18,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:23,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:28,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:29,202 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:09:31,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:09:31,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:09:31,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:09:34,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:39,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:44,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:46,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:09:46,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:09:46,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:09:48,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:09:48,881 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:09:48,882 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:09:48,883 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:09:49,037 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:09:49,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:51,039 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:09:54,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:09:59,202 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:10:00,203 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:01,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:10:01,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:10:01,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:10:05,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:10,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:15,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:16,215 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:10:16,215 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:10:16,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:10:21,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:26,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:29,204 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:10:31,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:10:31,227 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:10:31,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:10:32,523 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:37,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:42,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:46,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:10:46,227 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:10:46,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:10:48,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:53,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:58,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:10:59,205 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:11:01,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:11:01,227 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:11:01,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:11:02,874 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:11:02,876 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:11:02,876 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:11:02,877 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:11:03,084 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:11:03,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:05,085 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:11:08,918 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:13,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:16,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:11:16,227 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:11:16,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:11:19,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:24,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:29,206 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:11:30,206 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:31,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:11:31,227 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:11:31,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:11:35,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:40,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:45,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:46,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:11:46,227 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:11:46,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:11:51,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:56,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:11:59,207 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:12:01,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:12:01,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:12:01,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:12:02,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:07,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:12,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:16,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:12:16,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:12:16,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:12:16,933 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:12:16,935 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:12:16,935 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:12:16,936 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:12:17,129 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:12:17,977 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:19,130 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:12:22,978 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:27,979 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:29,207 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:12:31,227 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:12:31,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:12:31,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:12:33,524 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:38,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:43,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:46,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:12:46,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:12:46,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:12:49,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:54,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:12:59,208 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:13:00,209 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:01,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:13:01,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:13:01,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:13:05,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:10,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:15,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:16,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:13:16,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:13:16,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:13:21,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:26,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:29,209 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:13:31,111 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:13:31,113 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:13:31,113 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:13:31,115 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:13:31,174 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:13:31,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:13:31,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:13:31,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:13:32,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:33,175 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:13:37,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:42,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:46,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:13:46,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:13:46,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:13:48,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:53,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:58,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:13:59,210 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:14:01,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:14:01,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:14:01,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:14:04,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:09,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:14,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:16,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:14:16,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:14:16,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:14:20,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:25,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:29,211 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:14:31,212 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:31,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:14:31,228 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:14:31,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:14:36,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:41,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:45,059 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:14:45,062 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:14:45,062 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:14:45,064 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:14:45,219 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:14:46,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:14:46,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:14:46,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:14:47,220 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:14:47,525 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:52,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:57,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:14:59,212 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:15:01,228 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:15:01,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:15:01,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:15:03,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:08,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:13,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:16,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:15:16,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:15:16,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:15:19,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:24,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:29,213 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:15:30,214 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:31,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:15:31,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:15:31,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:15:35,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:40,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:45,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:46,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:15:46,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:15:46,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:15:51,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:56,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:15:59,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:15:59,126 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:15:59,127 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:15:59,128 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:15:59,214 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:15:59,266 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:16:01,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:16:01,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:16:01,267 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:16:01,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:16:02,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:07,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:12,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:16,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:16:16,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:16:16,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:16:18,526 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:23,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:28,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:29,215 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:16:31,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:16:31,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:16:31,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:16:34,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:39,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:44,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:46,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:16:46,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:16:46,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:16:50,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:55,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:16:59,216 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:17:01,217 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:01,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:17:01,229 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:17:01,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:17:06,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:11,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:13,954 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:17:13,956 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:17:13,957 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:17:13,958 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:17:14,316 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:17:15,317 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:17:16,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:17:16,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:17:16,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:17:17,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:22,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:27,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:29,217 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:17:31,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:17:31,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:17:31,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:17:34,043 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:39,044 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:44,045 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:46,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:17:46,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:17:46,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:17:49,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:54,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:17:59,218 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:18:00,219 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:01,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:18:01,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:18:01,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:18:05,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:10,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:15,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:16,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:18:16,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:18:16,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:18:21,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:26,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:27,970 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:18:27,972 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:18:27,972 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:18:27,974 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:18:28,362 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:18:29,219 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:18:29,363 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:18:31,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:18:31,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:18:31,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:18:32,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:37,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:42,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:46,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:18:46,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:18:46,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:18:48,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:53,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:58,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:18:59,220 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:19:01,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:19:01,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:19:01,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:19:04,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:09,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:14,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:16,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:19:16,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:19:16,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:19:20,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:25,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:29,221 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:19:31,222 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:31,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:19:31,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:19:31,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:19:36,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:41,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:42,052 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:19:42,053 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:19:42,054 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:19:42,055 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:19:42,407 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:19:42,631 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:19:43,408 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:19:45,410 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:19:46,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:19:46,230 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:19:46,411 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:19:46,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:19:47,527 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:52,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:57,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:19:59,222 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:20:01,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:20:01,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:20:01,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:20:03,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:08,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:13,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:16,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:20:16,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:20:16,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:20:19,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:24,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:29,223 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:20:30,225 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:31,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:20:31,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:20:31,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:20:35,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:40,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:45,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:46,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:20:46,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:20:46,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:20:51,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:56,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:20:59,224 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:21:00,124 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:21:00,126 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:21:00,126 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:21:00,127 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:21:00,455 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:21:01,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:21:01,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:21:01,456 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:21:01,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:21:02,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:07,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:12,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:16,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:21:16,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:21:16,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:21:18,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:23,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:28,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:29,225 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:21:31,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:21:31,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:21:31,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:21:34,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:39,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:44,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:46,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:21:46,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:21:46,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:21:50,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:55,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:21:59,227 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:22:01,229 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:01,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:22:01,231 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:22:01,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:22:06,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:11,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:14,033 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:22:14,035 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:22:14,035 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:22:14,036 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:22:14,500 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:22:15,501 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:22:16,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:22:16,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:22:16,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:22:17,528 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:22,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:27,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:29,227 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:22:31,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:22:31,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:22:31,537 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:22:33,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:38,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:43,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:46,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:22:46,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:22:46,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:22:49,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:54,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:22:59,229 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:23:00,230 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:01,231 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:23:01,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:23:01,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:23:05,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:10,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:15,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:16,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:23:16,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:23:16,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:23:21,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:26,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:27,917 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:23:27,919 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:23:27,919 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:23:27,920 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:23:28,545 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:23:29,229 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:23:29,546 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:23:31,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:23:31,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:23:31,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:23:32,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:37,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:42,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:46,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:23:46,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:23:46,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:23:48,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:53,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:58,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:23:59,230 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:24:01,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:24:01,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:24:01,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:24:04,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:09,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:14,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:16,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:24:16,232 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:24:16,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:24:20,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:25,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:29,231 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:24:31,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:24:31,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:31,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:24:31,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:24:36,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:41,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:41,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:24:41,856 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:24:41,856 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:24:41,857 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:24:42,593 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:24:43,594 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:24:46,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:24:46,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:24:46,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:24:47,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:52,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:57,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:24:59,232 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:25:01,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:25:01,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:25:01,538 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:25:03,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:08,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:13,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:16,232 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:25:16,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:25:16,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:25:19,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:24,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:29,233 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:25:30,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:31,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:25:31,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:25:31,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:25:35,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:40,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:45,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:46,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:25:46,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:25:46,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:25:51,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:56,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:25:56,717 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:25:56,719 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:25:56,719 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:25:56,721 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:25:57,644 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:25:57,644 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:25:59,234 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:26:01,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:26:01,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:26:01,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:26:02,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:07,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:12,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:16,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:26:16,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:26:16,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:26:18,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:23,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:28,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:29,235 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:26:31,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:26:31,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:26:31,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:26:34,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:39,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:44,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:46,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:26:46,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:26:46,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:26:50,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:55,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:26:59,236 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:27:01,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:27:01,233 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:27:01,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:01,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:27:06,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:10,754 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:27:10,756 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:27:10,756 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:27:10,757 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:27:11,689 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:27:11,689 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:27:11,797 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:16,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:27:16,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:27:16,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:27:17,530 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:22,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:27,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:29,236 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:27:31,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:27:31,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:27:31,539 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:27:33,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:38,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:43,100 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:46,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:27:46,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:27:46,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:27:48,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:53,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:58,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:27:59,237 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:28:01,233 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:28:01,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:28:01,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:28:04,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:09,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:14,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:16,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:28:16,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:28:16,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:28:20,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:24,711 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:28:24,713 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:28:24,713 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:28:24,714 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:28:24,732 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:28:25,733 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:28:25,753 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:29,238 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:28:31,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:28:31,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:28:31,459 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:31,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:28:36,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:41,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:46,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:28:46,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:28:46,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:28:47,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:52,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:57,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:28:59,239 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:29:01,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:29:01,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:29:01,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:29:03,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:08,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:13,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:16,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:29:16,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:29:16,540 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:29:19,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:24,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:29,240 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:29:30,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:31,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:29:31,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:29:31,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:29:35,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:38,728 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:29:38,730 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:29:38,730 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:29:38,731 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:29:38,779 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:29:39,780 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:29:40,769 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:45,770 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:46,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:29:46,234 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:29:46,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:29:51,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:56,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:29:59,242 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:30:01,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:30:01,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:30:01,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:30:02,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:07,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:12,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:16,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:30:16,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:30:16,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:30:18,532 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:23,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:28,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:29,242 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:30:31,234 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:30:31,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:30:31,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:30:34,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:39,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:44,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:46,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:30:46,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:30:46,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:30:50,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:52,811 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:30:52,813 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:30:52,813 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:30:52,815 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:30:52,824 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:30:53,824 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:30:55,854 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:30:59,243 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:31:01,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:31:01,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:31:01,487 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:01,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:31:06,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:11,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:16,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:31:16,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:31:16,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:31:17,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:22,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:27,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:29,244 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:31:31,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:31:31,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:31:31,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:31:33,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:38,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:43,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:46,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:31:46,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:31:46,541 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:31:49,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:54,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:31:59,245 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:32:00,246 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:01,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:32:01,235 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:32:01,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:32:05,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:06,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:32:06,685 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:32:06,686 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:32:06,687 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:32:06,869 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:32:07,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:32:07,869 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:32:09,871 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:32:10,906 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:11,872 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:32:15,938 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:16,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:32:16,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:32:16,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:32:21,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:26,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:29,246 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:32:31,235 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:32:31,236 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:32:31,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:32:32,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:37,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:42,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:46,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:32:46,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:32:46,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:32:48,533 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:53,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:58,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:32:59,247 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:33:01,236 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:33:01,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:33:01,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:33:03,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:08,920 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:13,921 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:16,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:33:16,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:33:16,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:33:19,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:24,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:25,164 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:33:25,166 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:33:25,166 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:33:25,167 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:33:25,919 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:33:25,919 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:33:29,248 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:33:30,249 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:31,237 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:33:31,237 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:33:31,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:33:35,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:40,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:45,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:46,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:33:46,300 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:33:46,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:33:50,570 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:55,571 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:33:59,249 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:34:01,250 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:01,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:34:01,300 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:34:01,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:34:06,552 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:11,553 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:16,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:34:16,300 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:34:16,542 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:34:16,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:21,561 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:26,562 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:29,250 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:34:31,300 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:34:31,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:34:31,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:34:32,534 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:37,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:39,669 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:34:39,671 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:34:39,671 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:34:39,672 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:34:39,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:34:39,967 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:34:42,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:46,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:34:46,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:34:46,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:34:48,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:53,559 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:58,560 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:34:59,250 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:35:01,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:35:01,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:35:01,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:35:03,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:08,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:13,569 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:16,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:35:16,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:35:16,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:35:19,567 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:24,568 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:29,251 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:35:30,252 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:31,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:35:31,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:35:31,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:35:35,572 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:40,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:45,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:46,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:35:46,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:35:46,543 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:35:51,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:53,655 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:35:53,657 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:35:53,658 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:35:53,659 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:35:54,012 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:35:54,012 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:35:56,698 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:35:59,252 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:36:01,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:36:01,301 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:36:01,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:36:02,535 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:07,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:12,536 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:16,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:36:16,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:36:16,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:36:17,565 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:22,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:27,566 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:29,253 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:36:31,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:36:31,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:36:31,544 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:36:33,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:38,557 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:43,558 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:46,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:36:46,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:36:46,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:36:48,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:53,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:58,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:36:59,254 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:37:01,301 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:37:01,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:37:01,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:37:04,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:07,789 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:37:07,791 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:37:07,791 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:37:07,793 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:37:08,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:37:08,057 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:37:09,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:14,835 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:16,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:37:16,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:37:16,593 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:37:20,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:25,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:29,255 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:37:31,256 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:31,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:37:31,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:37:31,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:37:36,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:41,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:46,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:37:46,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:37:46,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:37:47,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:52,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:57,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:37:59,256 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:38:01,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:38:01,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:38:01,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:38:03,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:08,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:13,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:16,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:38:16,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:38:16,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:38:19,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:21,742 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:38:21,744 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:38:21,744 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:38:21,745 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:38:22,101 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:38:22,101 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:38:24,786 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:29,257 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:38:30,257 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:31,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:38:31,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:38:31,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:38:35,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:40,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:45,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:46,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:38:46,302 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:38:46,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:38:51,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:56,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:38:59,258 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:39:01,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:39:01,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:39:01,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:39:02,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:07,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:12,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:16,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:39:16,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:39:16,594 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:39:18,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:23,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:28,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:29,259 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:39:31,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:39:31,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:39:31,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:39:34,573 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:35,607 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:39:35,609 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:39:35,610 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:39:35,611 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:39:36,146 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:39:36,146 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:39:39,612 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:44,613 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:46,302 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:39:46,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:39:46,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:39:50,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:55,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:39:59,260 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:40:01,260 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:01,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:40:01,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:40:01,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:40:06,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:11,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:16,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:40:16,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:40:16,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:40:17,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:22,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:27,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:29,261 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:40:31,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:40:31,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:40:31,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:40:33,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:38,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:43,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:46,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:40:46,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:40:46,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:40:49,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:49,670 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:40:49,672 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:40:49,672 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:40:49,673 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:40:50,190 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:40:51,191 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:40:54,714 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:40:59,261 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:41:00,262 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:01,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:41:01,303 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:41:01,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:41:05,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:10,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:15,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:16,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:41:16,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:41:16,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:41:21,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:26,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:29,262 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:41:31,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:41:31,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:41:31,595 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:41:32,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:37,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:42,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:46,303 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:41:46,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:41:46,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:41:48,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:53,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:58,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:41:59,264 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:42:01,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:42:01,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:42:01,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:42:04,054 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:42:04,056 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:42:04,056 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:42:04,056 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:04,057 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:42:04,239 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:42:05,239 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:42:09,098 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:14,099 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:16,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:42:16,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:42:16,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:42:19,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:24,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:29,265 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:42:30,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:31,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:42:31,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:42:31,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:42:36,264 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:41,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:46,265 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:46,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:42:46,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:42:46,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:42:51,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:56,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:42:59,266 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:43:01,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:43:01,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:43:01,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:43:02,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:07,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:12,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:16,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:43:16,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:43:16,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:43:18,199 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:43:18,200 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:43:18,201 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:43:18,201 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:18,202 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:43:18,286 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:43:19,287 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:43:23,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:28,242 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:29,266 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:43:31,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:43:31,304 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:43:31,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:43:33,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:38,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:43,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:46,304 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:43:46,305 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:43:46,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:43:49,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:54,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:43:59,267 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:44:00,268 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:01,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:44:01,305 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:44:01,596 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:44:05,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:10,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:15,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:16,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:44:16,305 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:44:16,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:44:21,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:26,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:29,268 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:44:31,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:44:31,305 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:44:31,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:44:32,476 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:44:32,478 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:44:32,479 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:44:32,479 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:32,480 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:44:33,053 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:44:33,333 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:44:35,334 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:44:36,335 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:44:37,549 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:39,337 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:44:42,550 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:46,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:44:46,305 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:44:46,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:44:47,598 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:52,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:57,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:44:59,269 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:45:01,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:45:01,305 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:45:01,597 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:45:03,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:08,599 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:13,600 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:16,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:45:16,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:45:16,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:45:18,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:23,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:28,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:29,270 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:45:31,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:45:31,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:45:31,616 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:45:34,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:39,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:44,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:46,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:45:46,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:45:46,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:45:50,529 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:45:50,531 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:45:50,531 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:45:50,531 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:50,533 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:45:51,381 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:45:53,382 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:45:55,574 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:45:59,271 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:46:01,272 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:01,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:46:01,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:46:01,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:46:06,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:11,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:16,305 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:46:16,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:46:16,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:46:17,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:22,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:27,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:29,272 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:46:31,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:46:31,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:46:31,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:46:32,632 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:37,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:42,633 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:46,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:46:46,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:46:46,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:46:48,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:53,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:58,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:46:59,273 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:47:01,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:47:01,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:47:01,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:47:04,511 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:47:04,512 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:47:04,513 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:47:04,513 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:04,514 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:47:05,425 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:47:05,425 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:47:09,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:14,554 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:16,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:47:16,306 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:47:16,617 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:47:19,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:24,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:29,274 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:47:30,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:31,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:47:31,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:47:31,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:47:35,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:40,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:45,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:46,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:47:46,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:47:46,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:47:51,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:56,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:47:59,275 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:48:01,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:48:01,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:48:01,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:48:02,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:07,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:12,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:16,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:48:16,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:48:16,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:48:18,391 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:48:18,393 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:48:18,393 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:48:18,394 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:18,395 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:48:18,471 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:48:19,471 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:48:23,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:28,434 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:29,276 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:48:31,306 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:48:31,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:48:31,618 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:48:33,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:38,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:43,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:46,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:48:46,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:48:46,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:48:49,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:54,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:48:59,277 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:49:00,278 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:01,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:49:01,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:49:01,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:49:05,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:10,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:15,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:16,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:49:16,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:49:16,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:49:21,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:26,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:29,278 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:49:31,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:49:31,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:49:31,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:49:32,275 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:49:32,276 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:49:32,277 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:49:32,277 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:32,278 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:49:32,517 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:49:33,518 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:49:37,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:42,318 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:46,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:49:46,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:49:46,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:49:47,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:52,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:57,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:49:59,279 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:50:01,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:50:01,307 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:50:01,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:50:03,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:08,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:13,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:16,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:50:16,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:50:16,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:50:19,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:24,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:29,280 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:50:30,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:31,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:50:31,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:50:31,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:50:35,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:40,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:45,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:46,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:50:46,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:50:46,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:50:47,091 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:50:47,093 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:50:47,093 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:50:47,094 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:50:47,568 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:50:49,569 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:50:51,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:56,134 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:50:59,281 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:51:01,281 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:01,307 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:51:01,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:51:01,619 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:51:06,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:11,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:16,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:51:16,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:51:16,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:51:17,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:22,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:27,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:29,282 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:51:31,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:51:31,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:51:31,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:51:33,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:38,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:43,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:46,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:51:46,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:51:46,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:51:49,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:54,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:51:59,283 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:52:00,283 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:01,146 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:52:01,147 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:52:01,148 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:52:01,149 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:52:01,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:52:01,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:52:01,613 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:52:01,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:52:03,614 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:52:05,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:10,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:15,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:16,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:52:16,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:52:16,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:52:21,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:26,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:29,284 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:52:31,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:52:31,308 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:52:31,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:52:32,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:37,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:42,623 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:46,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:52:46,309 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:52:46,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:52:48,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:53,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:58,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:52:59,286 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:53:01,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:53:01,309 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:53:01,620 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:53:04,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:09,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:14,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:14,828 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: partial_history +2024-08-13 07:53:14,830 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:53:14,830 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:53:14,831 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:53:15,659 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:53:15,659 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:53:16,309 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:53:16,309 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:53:16,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:53:20,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:25,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:29,285 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:53:31,286 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:31,308 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: stop_status +2024-08-13 07:53:31,309 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: stop_status +2024-08-13 07:53:31,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-13 07:53:36,621 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:41,622 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:43,879 DEBUG SenderThread:14795 [sender.py:send():382] send: exit +2024-08-13 07:53:43,879 INFO SenderThread:14795 [sender.py:send_exit():589] handling exit code: 255 +2024-08-13 07:53:43,879 INFO SenderThread:14795 [sender.py:send_exit():591] handling runtime: 87227 +2024-08-13 07:53:43,880 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:53:43,881 INFO SenderThread:14795 [sender.py:send_exit():597] send defer +2024-08-13 07:53:43,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,881 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-13 07:53:43,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,881 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-13 07:53:43,881 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 1 +2024-08-13 07:53:43,881 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,881 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-13 07:53:43,881 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,881 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-13 07:53:43,881 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 2 +2024-08-13 07:53:43,882 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,882 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-13 07:53:43,882 INFO HandlerThread:14795 [system_monitor.py:finish():203] Stopping system monitor +2024-08-13 07:53:43,882 INFO HandlerThread:14795 [interfaces.py:finish():202] Joined cpu monitor +2024-08-13 07:53:43,882 INFO HandlerThread:14795 [interfaces.py:finish():202] Joined disk monitor +2024-08-13 07:53:43,882 DEBUG SystemMonitor:14795 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-13 07:53:43,882 DEBUG SystemMonitor:14795 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-13 07:53:43,918 INFO HandlerThread:14795 [interfaces.py:finish():202] Joined gpu monitor +2024-08-13 07:53:43,918 INFO HandlerThread:14795 [interfaces.py:finish():202] Joined memory monitor +2024-08-13 07:53:43,918 INFO HandlerThread:14795 [interfaces.py:finish():202] Joined network monitor +2024-08-13 07:53:43,919 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,919 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-13 07:53:43,919 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 3 +2024-08-13 07:53:43,919 DEBUG SenderThread:14795 [sender.py:send():382] send: stats +2024-08-13 07:53:43,919 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,919 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-13 07:53:43,921 DEBUG SenderThread:14795 [sender.py:send():382] send: history +2024-08-13 07:53:43,921 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: summary_record +2024-08-13 07:53:43,922 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:53:43,922 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,922 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-13 07:53:43,922 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 4 +2024-08-13 07:53:43,922 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,923 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-13 07:53:43,923 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,923 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-13 07:53:43,923 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 5 +2024-08-13 07:53:43,923 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,923 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-13 07:53:43,924 DEBUG SenderThread:14795 [sender.py:send():382] send: summary +2024-08-13 07:53:43,925 INFO SenderThread:14795 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-13 07:53:43,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,925 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-13 07:53:43,925 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 6 +2024-08-13 07:53:43,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,925 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-13 07:53:43,925 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,925 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-13 07:53:43,925 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 7 +2024-08-13 07:53:43,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:43,925 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:43,926 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-13 07:53:43,926 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:43,926 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-13 07:53:44,677 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json +2024-08-13 07:53:44,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-13 07:53:46,676 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 8 +2024-08-13 07:53:46,676 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: poll_exit +2024-08-13 07:53:46,676 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:46,677 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-13 07:53:46,677 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:46,677 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-13 07:53:46,677 INFO SenderThread:14795 [job_builder.py:build():296] Attempting to build job artifact +2024-08-13 07:53:46,679 INFO SenderThread:14795 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-13 07:53:46,679 INFO Thread-12 :14795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log +2024-08-13 07:53:46,805 INFO SenderThread:14795 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-13 07:53:46,815 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 9 +2024-08-13 07:53:46,816 DEBUG SenderThread:14795 [sender.py:send():382] send: artifact +2024-08-13 07:53:46,816 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:46,817 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-13 07:53:46,879 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-13 07:53:47,646 INFO SenderThread:14795 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}} +2024-08-13 07:53:47,646 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:47,646 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-13 07:53:47,646 INFO SenderThread:14795 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-13 07:53:47,680 INFO SenderThread:14795 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_073955-ikoro1zp/files +2024-08-13 07:53:47,681 INFO SenderThread:14795 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073955-ikoro1zp/files/requirements.txt requirements.txt +2024-08-13 07:53:47,681 INFO SenderThread:14795 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073955-ikoro1zp/files/config.yaml config.yaml +2024-08-13 07:53:47,681 INFO SenderThread:14795 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-metadata.json wandb-metadata.json +2024-08-13 07:53:47,681 INFO SenderThread:14795 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json wandb-summary.json +2024-08-13 07:53:47,681 INFO SenderThread:14795 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073955-ikoro1zp/files/output.log output.log +2024-08-13 07:53:47,681 INFO SenderThread:14795 [sender.py:transition_state():617] send defer: 10 +2024-08-13 07:53:47,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: poll_exit +2024-08-13 07:53:47,682 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: defer +2024-08-13 07:53:47,682 INFO HandlerThread:14795 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-13 07:53:47,682 DEBUG SenderThread:14795 [sender.py:send_request():409] send_request: defer +2024-08-13 07:53:47,682 INFO SenderThread:14795 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-13 07:53:47,682 INFO SenderThread:14795 [file_pusher.py:finish():172] shutting down file pusher +2024-08-13 07:53:51,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:53:56,683 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:54:01,684 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:54:06,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:54:11,685 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:54:16,686 DEBUG HandlerThread:14795 [handler.py:handle_request():146] handle_request: status_report +2024-08-13 07:54:21,540 WARNING StreamThr :14795 [internal.py:is_dead():414] Internal process exiting, parent pid 14724 disappeared +2024-08-13 07:54:21,541 ERROR StreamThr :14795 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-13 07:54:21,686 INFO SenderThread:14795 [sender.py:finish():1572] shutting down sender +2024-08-13 07:54:21,687 INFO SenderThread:14795 [file_pusher.py:finish():172] shutting down file pusher +2024-08-13 07:54:21,687 INFO SenderThread:14795 [file_pusher.py:join():178] waiting for file pusher +2024-08-13 07:54:21,687 INFO SenderThread:14795 [file_stream.py:finish():595] file stream finish called +2024-08-13 07:54:21,687 INFO WriterThread:14795 [datastore.py:close():296] close: /project/wandb/run-20240812_073955-ikoro1zp/run-ikoro1zp.wandb +2024-08-13 07:54:21,687 INFO HandlerThread:14795 [handler.py:finish():869] shutting down handler +2024-08-13 07:54:21,901 INFO SenderThread:14795 [file_stream.py:finish():599] file stream finish is done diff --git a/wandb/run-20240812_073955-ikoro1zp/logs/debug.log b/wandb/run-20240812_073955-ikoro1zp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8f95a90e1750d7cea4c6b27fbb358c9e201e6e36 --- /dev/null +++ b/wandb/run-20240812_073955-ikoro1zp/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-12 07:39:55,678 INFO MainThread:14724 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-12 07:39:55,678 INFO MainThread:14724 [wandb_setup.py:_flush():76] Configure stats pid to 14724 +2024-08-12 07:39:55,678 INFO MainThread:14724 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-12 07:39:55,678 INFO MainThread:14724 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-12 07:39:55,678 INFO MainThread:14724 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train Qwen2'} +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_073955-ikoro1zp/logs/debug.log +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_073955-ikoro1zp/logs/debug-internal.log +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_init.py:init():566] calling init triggers +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'test_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-qwen2-0.5B_train_2024-08-12-07:39:43', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 320} +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_init.py:init():616] starting backend +2024-08-12 07:39:55,679 INFO MainThread:14724 [wandb_init.py:init():620] setting up manager +2024-08-12 07:39:55,684 INFO MainThread:14724 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-12 07:39:55,685 INFO MainThread:14724 [wandb_init.py:init():628] backend started and connected +2024-08-12 07:39:55,689 INFO MainThread:14724 [wandb_init.py:init():720] updated telemetry +2024-08-12 07:39:55,704 INFO MainThread:14724 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-12 07:39:56,202 INFO MainThread:14724 [wandb_run.py:_on_init():2262] communicating current version +2024-08-12 07:39:56,287 INFO MainThread:14724 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-12 07:39:56,287 INFO MainThread:14724 [wandb_init.py:init():804] starting run threads in backend +2024-08-12 07:39:56,346 INFO MainThread:14724 [wandb_run.py:_console_start():2241] atexit reg +2024-08-12 07:39:56,346 INFO MainThread:14724 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-12 07:39:56,346 INFO MainThread:14724 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-12 07:39:56,347 INFO MainThread:14724 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-12 07:39:56,348 INFO MainThread:14724 [wandb_init.py:init():847] run started, returning control to user process +2024-08-12 07:40:02,086 INFO MainThread:14724 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-12 07:40:02,086 INFO MainThread:14724 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240823_160642-78xnl14c/files/config.yaml b/wandb/run-20240823_160642-78xnl14c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e740de8c91a4d04c12d6f135b2f21eed6591767c --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +test_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 2048 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: Qwen2-0.5b-0.2_train_2024-08-23-16:06:29 +wandb_project: + desc: null + value: llm_tutorial-0.2 +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +save: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 7500 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 7500 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 5 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/Qwen2-0.5b-0.2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 64 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724396802.555005 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 2048 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240823_160642-78xnl14c/files/output.log b/wandb/run-20240823_160642-78xnl14c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..433592dce7d9046268dcf70c74519f540b90948d --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/files/output.log @@ -0,0 +1,253 @@ +Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/model.pt +Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 2400000 + validation: 2403200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/optimizer.pt +Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 2048, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-23 16:06:47,708] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 78.6590174462143, Tokens per sec: 22522.217665366814, Loss: 4.254438400268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 82.22378668182658, Tokens per sec: 23542.90812474309, Loss: 4.243721008300781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 81.9584539835788, Tokens per sec: 23466.93615131027, Loss: 4.228161334991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 81.93344116019101, Tokens per sec: 23459.774299165976, Loss: 4.26573371887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 82.18786108647078, Tokens per sec: 23532.62164895645, Loss: 4.256962776184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 81.96618258354832, Tokens per sec: 23469.14905984146, Loss: 4.25914192199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 81.87438758063158, Tokens per sec: 23442.865651995726, Loss: 4.231760025024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 81.88852755364577, Tokens per sec: 23446.914311136756, Loss: 4.2337870597839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 82.2474099184413, Tokens per sec: 23549.672100371965, Loss: 4.217740535736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 81.96982275621532, Tokens per sec: 23470.191340355606, Loss: 4.259789943695068 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020 +Saving model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/model.pt +Saving optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/optimizer.pt +[rank0]:[2024-08-23 16:11:31,233] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006403441000202292, 'preprocessing_with_comm': 0.0006401519999599259, 'state_converting': 0.9829786160003096, : 0.9914544049997858}) +Saved optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/scheduler.pt +Saving RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/rng.pt +Saved RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/rng.pt +Saved checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020, took 4.19s + eval ppl=57.051639556884766, eval loss=4.043956756591797 +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 81.67173393242047, Tokens per sec: 23384.840396610194, Loss: 4.235799789428711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 82.17554686977023, Tokens per sec: 23529.095753542137, Loss: 4.2608537673950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 81.93471100946451, Tokens per sec: 23460.13789157621, Loss: 4.211125373840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 81.95296715435393, Tokens per sec: 23465.36512276062, Loss: 4.202465534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 81.86218501721527, Tokens per sec: 23439.37172595571, Loss: 4.217883586883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 82.2054193168547, Tokens per sec: 23537.64904822184, Loss: 4.235620021820068 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 81.90650355273718, Tokens per sec: 23452.06132895408, Loss: 4.211632251739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 81.86577887433305, Tokens per sec: 23440.400745067473, Loss: 4.186619758605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 81.98724461570576, Tokens per sec: 23475.179692922393, Loss: 4.187148571014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 82.14778410156913, Tokens per sec: 23521.146517348767, Loss: 4.202610492706299 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030 +Saving model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/model.pt +Saved model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/model.pt +Saving optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/optimizer.pt +[rank0]:[2024-08-23 16:16:16,915] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063674119996903755, 'preprocessing_with_comm': 0.0006565419998878497, 'state_converting': 0.9940128050002386, : 1.0024299110000356}) +Saved optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/scheduler.pt +Saving RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/rng.pt +Saved RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/rng.pt +Saved checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030, took 4.25s + eval ppl=49.190242767333984, eval loss=3.895695209503174 +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 81.49741363388435, Tokens per sec: 23334.92775042431, Loss: 4.200056552886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 81.78042346796904, Tokens per sec: 23415.961168990125, Loss: 4.181160926818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 81.92238077096499, Tokens per sec: 23456.60740868476, Loss: 4.155094623565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 82.11937602485122, Tokens per sec: 23513.012511762787, Loss: 4.177372932434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 81.71716264856666, Tokens per sec: 23397.847874538613, Loss: 4.142157077789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 81.89085188246439, Tokens per sec: 23447.57982974198, Loss: 4.166767597198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 81.95670790833275, Tokens per sec: 23466.436202441997, Loss: 4.169678688049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 82.18953848455327, Tokens per sec: 23533.101933683516, Loss: 4.145669937133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 81.94531984778251, Tokens per sec: 23463.175490741974, Loss: 4.136501312255859 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 81.88752676960887, Tokens per sec: 23446.627759427276, Loss: 4.12642240524292 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040 +Saving model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +Saved model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +[rank0]:[2024-08-23 16:21:02,356] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006349127999783377, 'preprocessing_with_comm': 0.0006216020001375, 'state_converting': 0.7473425819998738, : 0.7556547180001871}) +Saving optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/scheduler.pt +Saving RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/rng.pt +Saved RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/rng.pt +Saved checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040, took 3.86s + eval ppl=30.580110549926758, eval loss=3.4203498363494873 +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 81.74253389997872, Tokens per sec: 23405.112354382087, Loss: 4.1325507164001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 81.9581488586999, Tokens per sec: 23466.848785752438, Loss: 4.099006652832031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 81.91532895406834, Tokens per sec: 23454.588281568267, Loss: 4.136029243469238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 81.95839032196434, Tokens per sec: 23466.917923257286, Loss: 4.140143871307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 82.23276643961596, Tokens per sec: 23545.47927381259, Loss: 4.161101341247559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 81.96124008786207, Tokens per sec: 23467.733888799798, Loss: 4.099796772003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 81.96352747485656, Tokens per sec: 23468.388829955275, Loss: 4.10368537902832 +------------------------------------------------------------------ +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train + loss.backward() + File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward + torch.autograd.backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward + _engine_run_backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20240823_160642-78xnl14c/files/requirements.txt b/wandb/run-20240823_160642-78xnl14c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240823_160642-78xnl14c/files/wandb-metadata.json b/wandb/run-20240823_160642-78xnl14c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..21dc25ba995c71520a0cc584979c39a8910cad8d --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-23T07:06:43.074166", + "startedAt": "2024-08-23T07:06:42.542542", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "2048", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "5", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "7500", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--test-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "7500", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--load", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/Qwen2-0.5b-0.2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial-0.2", + "--wandb-name", + "Qwen2-0.5b-0.2_train_2024-08-23-16:06:29" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487831115722656 + } +} diff --git a/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json b/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b2f2b2e2ae6a9b6237301a36d4c7f1e9d16fc8f5 --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 4.10368537902832, "training/perplexity": 60.56307470314165, "utils/batch_size": 5, "utils/global_batch_size": 320, "utils/seq_len": 2049, "utils/gradient_accumulation_steps": 64, "utils/iteration": 47, "optimizer/lr": 2.786e-06, "optimizer/variance_l2": 0.05447102242525233, "optimizer/variance_sqrt_l2": 0.9553866833854993, "optimizer/momentum_l2": 0.9463549769133376, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.9140548706054688, "optimizer/variance_sqrt_l1": 4069.0, "optimizer/momentum_l1": 3366.75, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.044921875, "optimizer/variance_sqrt_abs_max": 0.2119140625, "optimizer/momentum_abs_max": 0.2353515625, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 27.938858723999147, "stats/tokens_per_sec": 23468.388829955275, "stats/tokens_per_sec_per_gpu": 23468.388829955275, "stats/tflops": 81.96352747485656, "_timestamp": 1724397861.433749, "_runtime": 1058.8787438869476, "_step": 47, "evaluation/val_loss": 3.4203498363494873, "evaluation/val_ppl": 30.580110549926758, "_wandb": {"runtime": 1064}} \ No newline at end of file diff --git a/wandb/run-20240823_160642-78xnl14c/logs/debug-internal.log b/wandb/run-20240823_160642-78xnl14c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ac811c02e24a27e908111a9d7debbdc6cc771cba --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/logs/debug-internal.log @@ -0,0 +1,872 @@ +2024-08-23 16:06:42,557 INFO StreamThr :10929 [internal.py:wandb_internal():86] W&B internal server running at pid: 10929, started at: 2024-08-23 16:06:42.556306 +2024-08-23 16:06:42,558 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status +2024-08-23 16:06:42,560 INFO WriterThread:10929 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_160642-78xnl14c/run-78xnl14c.wandb +2024-08-23 16:06:42,560 DEBUG SenderThread:10929 [sender.py:send():382] send: header +2024-08-23 16:06:42,574 DEBUG SenderThread:10929 [sender.py:send():382] send: run +2024-08-23 16:06:42,981 INFO SenderThread:10929 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_160642-78xnl14c/files +2024-08-23 16:06:42,981 INFO SenderThread:10929 [sender.py:_start_run_threads():1136] run started: 78xnl14c with start time 1724396802.555005 +2024-08-23 16:06:42,986 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: check_version +2024-08-23 16:06:42,986 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: check_version +2024-08-23 16:06:43,055 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: run_start +2024-08-23 16:06:43,061 DEBUG HandlerThread:10929 [system_info.py:__init__():27] System info init +2024-08-23 16:06:43,062 DEBUG HandlerThread:10929 [system_info.py:__init__():42] System info init done +2024-08-23 16:06:43,062 INFO HandlerThread:10929 [system_monitor.py:start():194] Starting system monitor +2024-08-23 16:06:43,062 INFO SystemMonitor:10929 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-23 16:06:43,062 INFO HandlerThread:10929 [system_monitor.py:probe():214] Collecting system info +2024-08-23 16:06:43,062 INFO SystemMonitor:10929 [interfaces.py:start():190] Started cpu monitoring +2024-08-23 16:06:43,063 INFO SystemMonitor:10929 [interfaces.py:start():190] Started disk monitoring +2024-08-23 16:06:43,064 INFO SystemMonitor:10929 [interfaces.py:start():190] Started gpu monitoring +2024-08-23 16:06:43,064 INFO SystemMonitor:10929 [interfaces.py:start():190] Started memory monitoring +2024-08-23 16:06:43,065 INFO SystemMonitor:10929 [interfaces.py:start():190] Started network monitoring +2024-08-23 16:06:43,074 DEBUG HandlerThread:10929 [system_info.py:probe():151] Probing system +2024-08-23 16:06:43,076 DEBUG HandlerThread:10929 [system_info.py:_probe_git():136] Probing git +2024-08-23 16:06:43,087 DEBUG HandlerThread:10929 [system_info.py:_probe_git():144] Probing git done +2024-08-23 16:06:43,087 DEBUG HandlerThread:10929 [system_info.py:probe():199] Probing system done +2024-08-23 16:06:43,087 DEBUG HandlerThread:10929 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:06:43.074166', 'startedAt': '2024-08-23T07:06:42.542542', 'docker': None, 'cuda': None, 'args': ('--seq-length', '2048', '--sliding-window-size', '131072', '--micro-batch-size', '5', '--valid_micro_batch_size', '1', '--global-batch-size', '320', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:06:29'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}} +2024-08-23 16:06:43,088 INFO HandlerThread:10929 [system_monitor.py:probe():224] Finished collecting system info +2024-08-23 16:06:43,088 INFO HandlerThread:10929 [system_monitor.py:probe():227] Publishing system info +2024-08-23 16:06:43,089 INFO HandlerThread:10929 [system_monitor.py:probe():229] Finished publishing system info +2024-08-23 16:06:43,094 DEBUG SenderThread:10929 [sender.py:send():382] send: files +2024-08-23 16:06:43,094 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-23 16:06:43,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: python_packages +2024-08-23 16:06:43,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: python_packages +2024-08-23 16:06:43,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:06:43,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:06:43,147 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:06:43,338 DEBUG SenderThread:10929 [sender.py:send():382] send: telemetry +2024-08-23 16:06:43,704 INFO wandb-upload_0:10929 [upload_job.py:push():131] Uploaded file /tmp/tmpxq043eqbwandb/2jg1z1u0-wandb-metadata.json +2024-08-23 16:06:43,982 INFO Thread-12 :10929 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:06:43,983 INFO Thread-12 :10929 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_160642-78xnl14c/files/requirements.txt +2024-08-23 16:06:43,983 INFO Thread-12 :10929 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-metadata.json +2024-08-23 16:06:45,983 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:06:46,983 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:06:47,709 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:06:47,984 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:06:47,997 DEBUG SenderThread:10929 [sender.py:send():382] send: config +2024-08-23 16:06:47,997 DEBUG SenderThread:10929 [sender.py:send():382] send: config +2024-08-23 16:06:49,985 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:06:50,986 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:06:52,998 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:06:57,999 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:06:58,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:06:58,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:06:58,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:07:03,666 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:08,667 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:13,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:07:13,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:07:13,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:07:14,298 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:14,997 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/config.yaml +2024-08-23 16:07:17,523 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:07:19,568 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:20,000 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:07:24,569 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:28,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:07:28,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:07:28,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:07:30,354 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:35,355 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:40,355 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:43,065 DEBUG SystemMonitor:10929 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-23 16:07:43,067 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:07:43,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:07:43,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:07:43,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:07:45,361 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:45,375 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:07:45,378 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:07:45,378 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:07:45,380 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:07:46,013 INFO Thread-12 :10929 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:07:48,014 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:07:50,420 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:55,420 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:07:58,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:07:58,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:07:58,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:08:01,374 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:06,374 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:11,375 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:13,068 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:08:13,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:08:13,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:08:13,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:08:13,317 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:08:13,319 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:08:13,319 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:08:13,320 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:08:14,037 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:08:16,038 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:08:17,360 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:22,360 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:27,361 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:28,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:08:28,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:08:28,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:08:33,334 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:38,335 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:41,269 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:08:41,270 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:08:41,271 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:08:41,272 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:08:42,052 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:08:43,068 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:08:43,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:08:43,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:08:43,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:08:44,053 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:08:44,297 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:49,297 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:54,298 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:08:58,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:08:58,105 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:08:58,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:08:59,313 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:04,313 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:09,133 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:09:09,135 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:09:09,135 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:09:09,137 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:09:10,066 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:09:10,066 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:09:10,176 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:13,069 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:09:13,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:09:13,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:09:13,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:09:15,348 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:20,349 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:25,349 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:28,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:09:28,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:09:28,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:09:30,355 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:35,356 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:37,073 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:09:37,075 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:09:37,075 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:09:37,077 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:09:37,080 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:09:38,081 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:09:41,116 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:43,070 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:09:43,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:09:43,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:09:43,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:09:46,312 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:51,312 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:56,313 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:09:58,105 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:09:58,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:09:58,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:10:01,322 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:05,044 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:10:05,046 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:10:05,046 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:10:05,048 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:10:05,095 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:10:06,095 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:10:07,088 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:12,088 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:13,071 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:10:13,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:10:13,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:10:13,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:10:17,331 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:22,332 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:27,333 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:28,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:10:28,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:10:28,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:10:33,011 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:10:33,013 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:10:33,013 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:10:33,013 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:33,014 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:10:33,111 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:10:34,111 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:10:38,056 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:43,057 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:43,074 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:10:43,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:10:43,106 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:10:43,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:10:48,382 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:53,383 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:10:58,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:10:58,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:10:58,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:10:59,304 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:00,855 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:11:00,857 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:11:00,858 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:11:00,859 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:11:01,125 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:11:02,126 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:04,900 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:09,901 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:13,073 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:11:13,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:11:13,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:11:13,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:11:15,314 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:20,314 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:25,315 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:28,106 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:11:28,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:11:28,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:11:28,794 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:11:28,795 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:11:28,796 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:11:28,797 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:11:29,140 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:11:30,142 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:31,142 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:31,235 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:32,143 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:33,144 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:34,144 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:35,006 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:11:36,145 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:11:37,006 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:42,006 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:43,074 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:11:43,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:11:43,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:11:43,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:11:47,372 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:52,373 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:57,374 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:11:58,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:11:58,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:11:58,147 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:12:03,047 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:12:03,049 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:12:03,050 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:12:03,050 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:03,051 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:12:03,161 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:12:04,161 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:12:08,092 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:13,075 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:12:13,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:12:13,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:12:13,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:12:13,303 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:18,304 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:23,304 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:28,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:12:28,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:12:28,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:12:28,338 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:30,916 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:12:30,918 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:12:30,918 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:12:30,919 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:12:31,176 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:12:32,176 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:12:33,960 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:38,960 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:43,076 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:12:43,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:12:43,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:12:43,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:12:44,342 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:49,343 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:54,343 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:12:58,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:12:58,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:12:58,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:12:58,866 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:12:58,868 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:12:58,868 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:12:58,869 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:12:59,190 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:12:59,908 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:00,191 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:13:04,908 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:09,909 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:13,077 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:13:13,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:13:13,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:13:13,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:13:15,318 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:20,318 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:25,319 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:26,810 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:13:26,812 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:13:26,813 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:13:26,814 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:13:27,205 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:13:28,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:13:28,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:13:28,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:13:28,206 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:13:30,326 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:35,327 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:40,327 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:43,078 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:13:43,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:13:43,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:13:43,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:13:45,375 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:50,376 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:54,786 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:13:54,788 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:13:54,788 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:13:54,789 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:13:55,220 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:13:55,828 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:13:56,220 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:13:58,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:13:58,107 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:13:58,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:14:01,362 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:06,363 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:11,363 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:13,078 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:14:13,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:14:13,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:14:13,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:14:17,284 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:22,285 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:22,644 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:14:22,646 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:14:22,646 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:14:22,647 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:14:23,234 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:14:24,234 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:14:27,688 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:28,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:14:28,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:14:28,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:14:33,345 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:38,346 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:43,079 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:14:43,107 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:14:43,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:14:43,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:14:44,341 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:49,342 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:50,604 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:14:50,606 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:14:50,606 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:14:50,607 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:14:51,248 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:14:52,248 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:14:54,648 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:14:58,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:14:58,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:14:58,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:15:00,321 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:05,322 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:10,322 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:13,080 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:15:13,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:15:13,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:15:13,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:15:16,299 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:18,578 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:15:18,581 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:15:18,581 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:15:18,582 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:15:19,262 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:15:20,262 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:15:21,620 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:26,620 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:28,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:15:28,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:15:28,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:15:32,304 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:37,304 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:42,305 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:43,081 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:15:43,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:15:43,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:15:43,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:15:46,511 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:15:46,513 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:15:46,513 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:15:46,515 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:15:47,275 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:15:47,556 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:48,276 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:15:52,556 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:57,557 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:15:58,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:15:58,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:15:58,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:16:03,383 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:08,383 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:13,082 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:16:13,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:16:13,108 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:16:13,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:16:14,338 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:14,389 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:16:14,390 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:16:14,391 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:16:14,392 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:16:15,290 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:16:16,290 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:16:18,292 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:16:19,644 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:20,293 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:16:20,562 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:16:22,294 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:16:25,562 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:28,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:16:28,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:16:28,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:16:31,314 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:36,315 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:41,316 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:43,083 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:16:43,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:16:43,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:16:43,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:16:47,309 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:48,663 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:16:48,665 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:16:48,666 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:16:48,667 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:16:49,311 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:16:50,311 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:16:52,708 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:57,709 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:16:58,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:16:58,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:16:58,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:17:03,388 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:08,389 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:13,084 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:17:13,108 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:17:13,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:17:13,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:17:14,336 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:16,666 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:17:16,668 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:17:16,669 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:17:16,670 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:17:17,327 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:17:18,328 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:17:19,708 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:24,708 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:28,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:17:28,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:17:28,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:17:30,332 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:35,332 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:40,333 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:43,085 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:17:43,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:17:43,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:17:43,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:17:44,621 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:17:44,623 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:17:44,623 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:17:44,624 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:17:45,342 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:17:45,664 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:46,343 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:17:50,664 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:55,665 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:17:58,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:17:58,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:17:58,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:18:01,383 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:06,383 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:11,384 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:12,509 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:18:12,511 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:18:12,512 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:18:12,513 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:18:13,086 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:18:13,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:18:13,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:18:13,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:18:13,357 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:18:14,358 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:18:17,342 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:22,343 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:27,343 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:28,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:18:28,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:18:28,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:18:32,361 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:37,362 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:40,534 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:18:40,559 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:18:40,560 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:18:40,561 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:18:41,372 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:18:42,373 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:18:42,576 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:43,087 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:18:43,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:18:43,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:18:43,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:18:48,359 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:53,360 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:18:58,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:18:58,109 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:18:58,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:18:59,339 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:04,340 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:08,500 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:19:08,502 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:19:08,502 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:19:08,503 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:19:09,386 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:19:09,544 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:10,387 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:19:13,088 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:19:13,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:19:13,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:19:13,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:19:15,349 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:20,350 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:25,351 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:28,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:19:28,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:19:28,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:19:30,356 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:35,357 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:36,443 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:19:36,445 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:19:36,445 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:19:36,446 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:19:37,401 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:19:39,402 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:19:40,488 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:43,089 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:19:43,109 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:19:43,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:19:43,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:19:46,365 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:51,366 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:56,367 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:19:58,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:19:58,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:19:58,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:20:01,377 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:04,307 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:20:04,310 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:20:04,311 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:20:04,312 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:20:04,416 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:20:07,352 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:07,417 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:20:12,353 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:13,090 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:20:13,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:20:13,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:20:13,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:20:17,381 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:22,382 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:27,383 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:28,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:20:28,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:20:28,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:20:32,255 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:20:32,256 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:20:32,257 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:20:32,258 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:20:32,432 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:20:33,296 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:35,434 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:20:38,296 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:43,090 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:20:43,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:20:43,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:20:43,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:20:43,336 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:48,336 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:53,337 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:20:58,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:20:58,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:20:58,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:20:59,293 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:00,222 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:21:00,224 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:21:00,224 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:21:00,225 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:21:00,447 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:21:03,449 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:21:05,087 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:05,450 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:21:05,844 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:21:07,451 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:21:10,844 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:13,091 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:21:13,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:21:13,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:21:13,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:21:16,316 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:21,316 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:26,317 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:28,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:21:28,110 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:21:28,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:21:31,328 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:33,860 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:21:33,863 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:21:33,863 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:21:33,864 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:21:34,465 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:21:35,466 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:21:36,904 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:41,904 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:43,092 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:21:43,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:21:43,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:21:43,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:21:47,297 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:52,298 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:57,298 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:21:58,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:21:58,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:21:58,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:22:01,803 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:22:01,805 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:22:01,805 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:22:01,807 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:22:02,480 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:22:02,848 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:03,481 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:22:07,848 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:12,849 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:13,093 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:22:13,110 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:22:13,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:22:13,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:22:18,308 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:23,309 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:28,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:22:28,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:22:28,151 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:22:28,373 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:29,760 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:22:29,762 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:22:29,762 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:22:29,763 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:22:30,496 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:22:31,497 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:22:33,804 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:38,805 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:43,094 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:22:43,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:22:43,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:22:43,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:22:44,342 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:49,343 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:54,343 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:22:57,702 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:22:57,704 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:22:57,704 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:22:57,706 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:22:58,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:22:58,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:22:58,112 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:22:58,511 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:22:59,512 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:23:00,295 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:05,296 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:10,297 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:13,095 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:23:13,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:23:13,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:23:13,155 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:23:16,293 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:21,294 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:25,552 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:23:25,554 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:23:25,554 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:23:25,556 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:23:26,527 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:23:26,596 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:27,528 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:23:28,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:23:28,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:23:28,113 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:23:32,375 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:37,375 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:42,376 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:43,095 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:23:43,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:23:43,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:23:43,155 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:23:48,336 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:53,337 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:23:53,494 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:23:53,495 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:23:53,496 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:23:53,497 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:23:53,542 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:23:55,543 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:23:58,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:23:58,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:23:58,113 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:23:58,344 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:03,344 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:08,345 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:13,096 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:24:13,111 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:24:13,111 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:24:13,155 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:24:14,323 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:19,324 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:21,435 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:24:21,436 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:24:21,436 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:24:21,438 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:24:21,557 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:24:23,558 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:24:24,476 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:27,962 DEBUG SenderThread:10929 [sender.py:send():382] send: exit +2024-08-23 16:24:27,962 INFO SenderThread:10929 [sender.py:send_exit():589] handling exit code: 255 +2024-08-23 16:24:27,962 INFO SenderThread:10929 [sender.py:send_exit():591] handling runtime: 1064 +2024-08-23 16:24:27,963 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:24:27,963 INFO SenderThread:10929 [sender.py:send_exit():597] send defer +2024-08-23 16:24:27,963 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:27,964 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-23 16:24:27,964 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:27,964 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-23 16:24:27,964 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 1 +2024-08-23 16:24:27,964 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:27,964 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-23 16:24:27,964 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:27,964 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-23 16:24:27,964 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 2 +2024-08-23 16:24:27,964 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:27,964 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-23 16:24:27,964 INFO HandlerThread:10929 [system_monitor.py:finish():203] Stopping system monitor +2024-08-23 16:24:27,964 DEBUG SystemMonitor:10929 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-23 16:24:27,965 INFO HandlerThread:10929 [interfaces.py:finish():202] Joined cpu monitor +2024-08-23 16:24:27,965 DEBUG SystemMonitor:10929 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-23 16:24:27,965 INFO HandlerThread:10929 [interfaces.py:finish():202] Joined disk monitor +2024-08-23 16:24:27,998 INFO HandlerThread:10929 [interfaces.py:finish():202] Joined gpu monitor +2024-08-23 16:24:27,999 INFO HandlerThread:10929 [interfaces.py:finish():202] Joined memory monitor +2024-08-23 16:24:27,999 INFO HandlerThread:10929 [interfaces.py:finish():202] Joined network monitor +2024-08-23 16:24:27,999 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:27,999 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-23 16:24:27,999 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 3 +2024-08-23 16:24:27,999 DEBUG SenderThread:10929 [sender.py:send():382] send: stats +2024-08-23 16:24:27,999 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:28,000 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-23 16:24:28,001 DEBUG SenderThread:10929 [sender.py:send():382] send: history +2024-08-23 16:24:28,001 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:24:28,002 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:24:28,003 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:28,003 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-23 16:24:28,003 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 4 +2024-08-23 16:24:28,003 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:28,003 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-23 16:24:28,003 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:28,003 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-23 16:24:28,003 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 5 +2024-08-23 16:24:28,003 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:28,003 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-23 16:24:28,004 DEBUG SenderThread:10929 [sender.py:send():382] send: summary +2024-08-23 16:24:28,005 INFO SenderThread:10929 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:24:28,005 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:28,005 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-23 16:24:28,005 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 6 +2024-08-23 16:24:28,005 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:28,005 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-23 16:24:28,005 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:28,005 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-23 16:24:28,005 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 7 +2024-08-23 16:24:28,005 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:24:28,006 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:28,006 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-23 16:24:28,006 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:28,006 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-23 16:24:28,561 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:24:28,962 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:24:29,266 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 8 +2024-08-23 16:24:29,267 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:24:29,267 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:29,267 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-23 16:24:29,267 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:29,267 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-23 16:24:29,267 INFO SenderThread:10929 [job_builder.py:build():296] Attempting to build job artifact +2024-08-23 16:24:29,268 INFO SenderThread:10929 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-23 16:24:29,283 INFO SenderThread:10929 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-23 16:24:29,292 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 9 +2024-08-23 16:24:29,292 DEBUG SenderThread:10929 [sender.py:send():382] send: artifact +2024-08-23 16:24:29,292 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:29,293 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-23 16:24:29,562 INFO Thread-12 :10929 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:24:29,962 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:24:30,162 INFO SenderThread:10929 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MTk4ODkxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'versionIndex': 2}}} +2024-08-23 16:24:30,162 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:30,162 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-23 16:24:30,162 INFO SenderThread:10929 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-23 16:24:30,563 INFO SenderThread:10929 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_160642-78xnl14c/files +2024-08-23 16:24:30,563 INFO SenderThread:10929 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_160642-78xnl14c/files/requirements.txt requirements.txt +2024-08-23 16:24:30,563 INFO SenderThread:10929 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_160642-78xnl14c/files/config.yaml config.yaml +2024-08-23 16:24:30,565 INFO SenderThread:10929 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-metadata.json wandb-metadata.json +2024-08-23 16:24:30,565 INFO SenderThread:10929 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json wandb-summary.json +2024-08-23 16:24:30,566 INFO SenderThread:10929 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_160642-78xnl14c/files/output.log output.log +2024-08-23 16:24:30,568 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 10 +2024-08-23 16:24:30,568 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:24:30,568 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:30,569 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-23 16:24:30,570 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:30,570 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-23 16:24:30,570 INFO SenderThread:10929 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:24:30,963 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:24:30,963 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:24:31,088 INFO wandb-upload_0:10929 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_160642-78xnl14c/files/requirements.txt +2024-08-23 16:24:31,963 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:24:31,963 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:24:32,048 INFO wandb-upload_2:10929 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json +2024-08-23 16:24:32,127 INFO wandb-upload_1:10929 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_160642-78xnl14c/files/config.yaml +2024-08-23 16:24:32,201 INFO wandb-upload_3:10929 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_160642-78xnl14c/files/output.log +2024-08-23 16:24:32,401 INFO Thread-11 (_thread_body):10929 [sender.py:transition_state():617] send defer: 11 +2024-08-23 16:24:32,401 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:32,401 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-23 16:24:32,401 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:32,401 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-23 16:24:32,401 INFO SenderThread:10929 [file_pusher.py:join():178] waiting for file pusher +2024-08-23 16:24:32,402 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 12 +2024-08-23 16:24:32,402 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:32,402 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-23 16:24:32,402 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:32,402 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-23 16:24:32,402 INFO SenderThread:10929 [file_stream.py:finish():595] file stream finish called +2024-08-23 16:24:32,962 INFO SenderThread:10929 [file_stream.py:finish():599] file stream finish is done +2024-08-23 16:24:32,962 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 13 +2024-08-23 16:24:32,963 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:32,963 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-23 16:24:32,963 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:32,963 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-23 16:24:32,963 INFO SenderThread:10929 [sender.py:transition_state():617] send defer: 14 +2024-08-23 16:24:32,963 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:24:32,963 DEBUG SenderThread:10929 [sender.py:send():382] send: final +2024-08-23 16:24:32,963 INFO HandlerThread:10929 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-23 16:24:32,963 DEBUG SenderThread:10929 [sender.py:send():382] send: footer +2024-08-23 16:24:32,964 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: defer +2024-08-23 16:24:32,964 INFO SenderThread:10929 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-23 16:24:32,964 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:24:32,964 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:24:32,964 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:24:32,965 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: server_info +2024-08-23 16:24:32,965 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:24:32,965 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: get_summary +2024-08-23 16:24:32,965 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: server_info +2024-08-23 16:24:32,966 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-23 16:24:32,969 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:24:32,970 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: job_info +2024-08-23 16:24:33,128 DEBUG SenderThread:10929 [sender.py:send_request():409] send_request: job_info +2024-08-23 16:24:33,128 INFO MainThread:10929 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-23 16:24:33,129 INFO MainThread:10929 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-23 16:24:33,130 INFO MainThread:10929 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-23 16:24:33,130 DEBUG HandlerThread:10929 [handler.py:handle_request():146] handle_request: shutdown +2024-08-23 16:24:33,130 INFO HandlerThread:10929 [handler.py:finish():869] shutting down handler +2024-08-23 16:24:33,970 INFO WriterThread:10929 [datastore.py:close():296] close: /project/wandb/run-20240823_160642-78xnl14c/run-78xnl14c.wandb +2024-08-23 16:24:34,128 INFO SenderThread:10929 [sender.py:finish():1572] shutting down sender +2024-08-23 16:24:34,129 INFO SenderThread:10929 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:24:34,129 INFO SenderThread:10929 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240823_160642-78xnl14c/logs/debug.log b/wandb/run-20240823_160642-78xnl14c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cfd96459aeed4c4ae0ccdedd86eb7826ae701a22 --- /dev/null +++ b/wandb/run-20240823_160642-78xnl14c/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-23 16:06:42,548 INFO MainThread:10858 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_setup.py:_flush():76] Configure stats pid to 10858 +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_160642-78xnl14c/logs/debug.log +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_160642-78xnl14c/logs/debug-internal.log +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_init.py:init():566] calling init triggers +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 2048, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:06:29', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 5, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 64} +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_init.py:init():616] starting backend +2024-08-23 16:06:42,549 INFO MainThread:10858 [wandb_init.py:init():620] setting up manager +2024-08-23 16:06:42,554 INFO MainThread:10858 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-23 16:06:42,554 INFO MainThread:10858 [wandb_init.py:init():628] backend started and connected +2024-08-23 16:06:42,559 INFO MainThread:10858 [wandb_init.py:init():720] updated telemetry +2024-08-23 16:06:42,570 INFO MainThread:10858 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-23 16:06:42,985 INFO MainThread:10858 [wandb_run.py:_on_init():2262] communicating current version +2024-08-23 16:06:43,007 INFO MainThread:10858 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-23 16:06:43,007 INFO MainThread:10858 [wandb_init.py:init():804] starting run threads in backend +2024-08-23 16:06:43,104 INFO MainThread:10858 [wandb_run.py:_console_start():2241] atexit reg +2024-08-23 16:06:43,105 INFO MainThread:10858 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-23 16:06:43,105 INFO MainThread:10858 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-23 16:06:43,105 INFO MainThread:10858 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-23 16:06:43,106 INFO MainThread:10858 [wandb_init.py:init():847] run started, returning control to user process +2024-08-23 16:06:47,996 INFO MainThread:10858 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 2048, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-23 16:06:47,997 INFO MainThread:10858 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-23 16:24:34,131 WARNING MsgRouterThr:10858 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240823_160642-78xnl14c/run-78xnl14c.wandb b/wandb/run-20240823_160642-78xnl14c/run-78xnl14c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0e66381810a53fc97914eaafcd50de5ab621a4e3 Binary files /dev/null and b/wandb/run-20240823_160642-78xnl14c/run-78xnl14c.wandb differ diff --git a/wandb/run-20240823_162922-z3gs82jm/files/config.yaml b/wandb/run-20240823_162922-z3gs82jm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dc62fce52b8eb3da5fbfba96eeeb63e90a49daf --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +test_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 2048 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: Qwen2-0.5b-0.2_train_2024-08-23-16:29:10 +wandb_project: + desc: null + value: llm_tutorial-0.2 +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +save: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 7500 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 7500 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 640 +micro_batch_size: + desc: null + value: 5 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/Qwen2-0.5b-0.2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 128 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724398162.884223 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 2048 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240823_162922-z3gs82jm/files/output.log b/wandb/run-20240823_162922-z3gs82jm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..15443a4dc10bf3f3bb95c95d143847f74de868ad --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/files/output.log @@ -0,0 +1,174 @@ +Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 4800000 + validation: 4806400 + test: 6400 +> building train, validation, and test datasets for GPT ... +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 2048, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-23 16:29:30,218] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 80.56174819694237, Tokens per sec: 23067.0212685365, Loss: 4.141458988189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 82.11115614789513, Tokens per sec: 23510.658937258577, Loss: 4.13422155380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 82.06292558214139, Tokens per sec: 23496.84921352573, Loss: 4.125084400177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 82.21135543718982, Tokens per sec: 23539.348721045328, Loss: 4.142415523529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 82.19624054931023, Tokens per sec: 23535.020917242735, Loss: 4.127298831939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 82.08863919564567, Tokens per sec: 23504.21172095474, Loss: 4.131596565246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 82.19786737309032, Tokens per sec: 23535.486721170473, Loss: 4.140783786773682 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 82.07990700682468, Tokens per sec: 23501.711457619313, Loss: 4.111098289489746 +------------------------------------------------------------------ +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train + loss: torch.Tensor = model(**batch).loss + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 1054, in forward + outputs = self.model( + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 856, in forward + layer_outputs = decoder_layer( + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/algorithms/_checkpoint/checkpoint_wrapper.py", line 168, in forward + return self.checkpoint_fn( # type: ignore[misc] + File "/usr/local/lib/python3.10/dist-packages/torch/_compile.py", line 24, in inner + return torch._dynamo.disable(fn, recursive)(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py", line 417, in _fn + return fn(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/external_utils.py", line 25, in inner + return fn(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py", line 488, in checkpoint + ret = function(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 609, in forward + hidden_states = self.post_attention_layernorm(hidden_states) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 78, in forward + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py", line 1091, in pack_hook + with torch.no_grad(): + File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 149, in __new__ + def __new__(cls, orig_func=None): +KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt b/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json b/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a77f3d1c50db04ddcfba64b350f6feb8cb508845 --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-23T07:29:23.385958", + "startedAt": "2024-08-23T07:29:22.871856", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "2048", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "5", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "640", + "--train-iters", + "7500", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--test-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "7500", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--load", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/Qwen2-0.5b-0.2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial-0.2", + "--wandb-name", + "Qwen2-0.5b-0.2_train_2024-08-23-16:29:10" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487831115722656 + } +} diff --git a/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json b/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..da700913b898ffba779736617053bed664b091f5 --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 4.111098289489746, "training/perplexity": 61.013691480602496, "utils/batch_size": 5, "utils/global_batch_size": 640, "utils/seq_len": 2049, "utils/gradient_accumulation_steps": 128, "utils/iteration": 48, "optimizer/lr": 2.8240000000000004e-06, "optimizer/variance_l2": 0.05465312149531553, "optimizer/variance_sqrt_l2": 0.9576321330918345, "optimizer/momentum_l2": 0.9493419425990095, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.91644287109375, "optimizer/variance_sqrt_l1": 3987.5, "optimizer/momentum_l1": 3375.75, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.044921875, "optimizer/variance_sqrt_abs_max": 0.2119140625, "optimizer/momentum_abs_max": 0.234375, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 55.79848949999996, "stats/tokens_per_sec": 23501.711457619313, "stats/tokens_per_sec_per_gpu": 23501.711457619313, "stats/tflops": 82.07990700682468, "_timestamp": 1724398618.465386, "_runtime": 455.5811629295349, "_step": 48, "_wandb": {"runtime": 505}} \ No newline at end of file diff --git a/wandb/run-20240823_162922-z3gs82jm/logs/debug-internal.log b/wandb/run-20240823_162922-z3gs82jm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..099eff6713c5b34f880c97326746afa598a3729c --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/logs/debug-internal.log @@ -0,0 +1,453 @@ +2024-08-23 16:29:22,886 INFO StreamThr :11966 [internal.py:wandb_internal():86] W&B internal server running at pid: 11966, started at: 2024-08-23 16:29:22.885536 +2024-08-23 16:29:22,888 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status +2024-08-23 16:29:22,889 INFO WriterThread:11966 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb +2024-08-23 16:29:22,890 DEBUG SenderThread:11966 [sender.py:send():382] send: header +2024-08-23 16:29:22,904 DEBUG SenderThread:11966 [sender.py:send():382] send: run +2024-08-23 16:29:23,294 INFO SenderThread:11966 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_162922-z3gs82jm/files +2024-08-23 16:29:23,294 INFO SenderThread:11966 [sender.py:_start_run_threads():1136] run started: z3gs82jm with start time 1724398162.884223 +2024-08-23 16:29:23,300 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: check_version +2024-08-23 16:29:23,300 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: check_version +2024-08-23 16:29:23,367 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: run_start +2024-08-23 16:29:23,374 DEBUG HandlerThread:11966 [system_info.py:__init__():27] System info init +2024-08-23 16:29:23,374 DEBUG HandlerThread:11966 [system_info.py:__init__():42] System info init done +2024-08-23 16:29:23,374 INFO HandlerThread:11966 [system_monitor.py:start():194] Starting system monitor +2024-08-23 16:29:23,374 INFO SystemMonitor:11966 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-23 16:29:23,374 INFO HandlerThread:11966 [system_monitor.py:probe():214] Collecting system info +2024-08-23 16:29:23,374 INFO SystemMonitor:11966 [interfaces.py:start():190] Started cpu monitoring +2024-08-23 16:29:23,375 INFO SystemMonitor:11966 [interfaces.py:start():190] Started disk monitoring +2024-08-23 16:29:23,375 INFO SystemMonitor:11966 [interfaces.py:start():190] Started gpu monitoring +2024-08-23 16:29:23,377 INFO SystemMonitor:11966 [interfaces.py:start():190] Started memory monitoring +2024-08-23 16:29:23,378 INFO SystemMonitor:11966 [interfaces.py:start():190] Started network monitoring +2024-08-23 16:29:23,385 DEBUG HandlerThread:11966 [system_info.py:probe():151] Probing system +2024-08-23 16:29:23,387 DEBUG HandlerThread:11966 [system_info.py:_probe_git():136] Probing git +2024-08-23 16:29:23,400 DEBUG HandlerThread:11966 [system_info.py:_probe_git():144] Probing git done +2024-08-23 16:29:23,400 DEBUG HandlerThread:11966 [system_info.py:probe():199] Probing system done +2024-08-23 16:29:23,400 DEBUG HandlerThread:11966 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:29:23.385958', 'startedAt': '2024-08-23T07:29:22.871856', 'docker': None, 'cuda': None, 'args': ('--seq-length', '2048', '--sliding-window-size', '131072', '--micro-batch-size', '5', '--valid_micro_batch_size', '1', '--global-batch-size', '640', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:29:10'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}} +2024-08-23 16:29:23,400 INFO HandlerThread:11966 [system_monitor.py:probe():224] Finished collecting system info +2024-08-23 16:29:23,400 INFO HandlerThread:11966 [system_monitor.py:probe():227] Publishing system info +2024-08-23 16:29:23,402 INFO HandlerThread:11966 [system_monitor.py:probe():229] Finished publishing system info +2024-08-23 16:29:23,407 DEBUG SenderThread:11966 [sender.py:send():382] send: files +2024-08-23 16:29:23,407 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-23 16:29:23,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: python_packages +2024-08-23 16:29:23,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:29:23,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:29:23,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: python_packages +2024-08-23 16:29:23,421 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:29:23,617 DEBUG SenderThread:11966 [sender.py:send():382] send: telemetry +2024-08-23 16:29:24,296 INFO Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:29:24,296 INFO Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt +2024-08-23 16:29:24,297 INFO Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json +2024-08-23 16:29:24,474 INFO wandb-upload_0:11966 [upload_job.py:push():131] Uploaded file /tmp/tmpcv014twmwandb/xf5mvw68-wandb-metadata.json +2024-08-23 16:29:26,296 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:29:28,298 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:29:28,658 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:29:30,299 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:29:30,499 DEBUG SenderThread:11966 [sender.py:send():382] send: config +2024-08-23 16:29:30,500 DEBUG SenderThread:11966 [sender.py:send():382] send: config +2024-08-23 16:29:32,300 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:29:34,500 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:29:38,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:29:38,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:29:38,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:29:39,685 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:29:44,685 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:29:49,686 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:29:53,417 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:29:53,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:29:53,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:29:55,642 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:29:56,312 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/config.yaml +2024-08-23 16:30:00,842 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:05,842 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:08,417 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:30:08,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:30:08,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:30:11,641 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:16,641 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:21,642 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:23,378 DEBUG SystemMonitor:11966 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-23 16:30:23,380 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:30:23,417 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:30:23,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:30:23,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:30:27,617 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:28,122 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:30:30,329 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:30:33,164 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:38,165 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:38,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:30:38,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:30:38,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:30:43,611 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:48,611 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:53,380 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:30:53,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:30:53,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:30:53,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:30:53,639 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:30:58,640 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:03,640 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:08,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:31:08,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:31:08,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:31:08,666 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:13,666 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:18,667 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:23,382 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:31:23,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:31:23,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:31:23,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:31:23,682 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:23,901 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:31:23,904 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:31:23,904 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:31:23,906 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:31:24,355 INFO Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:31:26,356 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:31:28,944 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:33,945 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:38,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:31:38,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:31:38,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:31:39,638 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:44,639 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:49,639 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:53,383 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:31:53,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:31:53,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:31:53,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:31:54,679 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:31:59,679 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:04,680 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:08,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:32:08,418 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:32:08,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:32:10,613 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:15,613 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:19,715 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:32:19,717 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:32:19,717 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:32:19,719 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:32:20,383 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:32:20,760 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:22,384 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:32:23,385 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:32:23,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:32:23,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:32:23,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:32:26,593 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:31,594 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:36,594 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:38,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:32:38,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:32:38,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:32:41,674 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:46,675 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:51,676 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:32:53,384 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:32:53,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:32:53,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:32:53,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:32:57,603 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:02,604 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:07,604 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:08,418 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:33:08,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:33:08,459 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:33:12,676 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:15,428 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:33:15,430 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:33:15,430 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:33:15,432 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:33:16,412 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:33:18,413 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:33:18,472 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:23,385 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:33:23,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:33:23,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:33:23,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:33:23,682 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:28,682 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:33,683 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:38,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:33:38,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:33:38,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:33:39,667 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:44,667 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:49,668 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:33:53,386 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:33:53,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:33:53,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:33:53,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:33:55,611 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:00,612 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:05,612 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:08,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:34:08,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:34:08,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:34:10,679 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:11,150 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:34:11,152 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:34:11,152 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:34:11,154 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:34:11,439 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:34:12,439 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:34:16,192 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:21,193 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:23,387 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:34:23,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:34:23,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:34:23,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:34:26,589 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:31,590 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:36,590 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:38,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:34:38,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:34:38,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:34:41,674 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:46,674 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:51,674 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:34:53,389 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:34:53,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:34:53,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:34:53,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:34:57,671 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:02,672 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:06,945 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:35:06,948 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:35:06,948 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:35:06,949 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:35:07,467 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:35:07,988 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:08,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:35:08,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:35:08,421 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:35:08,468 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:35:13,689 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:18,689 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:23,389 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:35:23,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:35:23,419 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:35:23,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:35:24,596 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:29,597 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:34,597 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:38,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:35:38,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:35:38,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:35:39,689 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:44,690 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:49,690 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:35:53,390 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:35:53,419 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:35:53,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:35:53,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:35:55,688 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:00,688 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:02,665 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:36:02,667 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:36:02,667 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:36:02,668 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:36:03,495 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:36:04,496 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:36:05,708 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:08,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:36:08,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:36:08,421 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:36:11,641 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:16,641 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:21,642 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:23,391 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:36:23,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:36:23,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:36:23,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:36:26,684 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:31,685 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:36,685 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:38,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:36:38,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:36:38,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:36:41,692 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:46,692 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:51,693 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:53,392 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:36:53,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:36:53,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:36:53,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:36:57,615 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:36:58,466 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:36:58,468 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:36:58,468 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:36:58,469 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:36:58,523 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:37:00,524 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:37:03,508 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:08,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:37:08,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:37:08,422 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:37:08,688 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:13,688 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:18,689 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:23,393 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:37:23,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:37:23,420 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:37:23,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:37:24,586 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:29,587 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:34,587 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:38,420 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:37:38,421 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:37:38,463 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:37:40,196 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:45,197 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:48,676 DEBUG SenderThread:11966 [sender.py:send():382] send: exit +2024-08-23 16:37:48,676 INFO SenderThread:11966 [sender.py:send_exit():589] handling exit code: 255 +2024-08-23 16:37:48,676 INFO SenderThread:11966 [sender.py:send_exit():591] handling runtime: 505 +2024-08-23 16:37:48,677 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:37:48,677 INFO SenderThread:11966 [sender.py:send_exit():597] send defer +2024-08-23 16:37:48,678 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,678 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-23 16:37:48,678 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,678 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-23 16:37:48,678 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 1 +2024-08-23 16:37:48,678 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,678 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-23 16:37:48,678 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,678 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-23 16:37:48,678 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 2 +2024-08-23 16:37:48,678 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,678 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-23 16:37:48,678 INFO HandlerThread:11966 [system_monitor.py:finish():203] Stopping system monitor +2024-08-23 16:37:48,679 DEBUG SystemMonitor:11966 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-23 16:37:48,679 INFO HandlerThread:11966 [interfaces.py:finish():202] Joined cpu monitor +2024-08-23 16:37:48,679 DEBUG SystemMonitor:11966 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-23 16:37:48,679 INFO HandlerThread:11966 [interfaces.py:finish():202] Joined disk monitor +2024-08-23 16:37:48,712 INFO HandlerThread:11966 [interfaces.py:finish():202] Joined gpu monitor +2024-08-23 16:37:48,712 INFO HandlerThread:11966 [interfaces.py:finish():202] Joined memory monitor +2024-08-23 16:37:48,712 INFO HandlerThread:11966 [interfaces.py:finish():202] Joined network monitor +2024-08-23 16:37:48,713 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,713 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-23 16:37:48,713 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 3 +2024-08-23 16:37:48,713 DEBUG SenderThread:11966 [sender.py:send():382] send: stats +2024-08-23 16:37:48,713 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,713 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-23 16:37:48,715 DEBUG SenderThread:11966 [sender.py:send():382] send: history +2024-08-23 16:37:48,715 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:37:48,716 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:37:48,716 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,716 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-23 16:37:48,716 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 4 +2024-08-23 16:37:48,716 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,716 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-23 16:37:48,717 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,717 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-23 16:37:48,717 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 5 +2024-08-23 16:37:48,717 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,717 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-23 16:37:48,718 DEBUG SenderThread:11966 [sender.py:send():382] send: summary +2024-08-23 16:37:48,718 INFO SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:37:48,719 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,719 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-23 16:37:48,719 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 6 +2024-08-23 16:37:48,719 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,719 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-23 16:37:48,719 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,719 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-23 16:37:48,719 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 7 +2024-08-23 16:37:48,719 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:48,719 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:48,719 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-23 16:37:48,720 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:48,720 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-23 16:37:49,550 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:37:49,671 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:50,524 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 8 +2024-08-23 16:37:50,524 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:50,524 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:50,524 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-23 16:37:50,524 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:50,524 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-23 16:37:50,525 INFO SenderThread:11966 [job_builder.py:build():296] Attempting to build job artifact +2024-08-23 16:37:50,525 INFO SenderThread:11966 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-23 16:37:50,540 INFO SenderThread:11966 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-23 16:37:50,549 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 9 +2024-08-23 16:37:50,550 DEBUG SenderThread:11966 [sender.py:send():382] send: artifact +2024-08-23 16:37:50,550 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:50,551 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-23 16:37:50,551 INFO Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:37:50,671 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:51,630 INFO wandb-upload_0:11966 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmp7k_y5w7r +2024-08-23 16:37:52,084 INFO wandb-upload_1:11966 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp4s29y7vc +2024-08-23 16:37:53,444 INFO SenderThread:11966 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'versionIndex': 2}}} +2024-08-23 16:37:53,444 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:53,444 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-23 16:37:53,444 INFO SenderThread:11966 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-23 16:37:53,553 INFO SenderThread:11966 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_162922-z3gs82jm/files +2024-08-23 16:37:53,553 INFO SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt requirements.txt +2024-08-23 16:37:53,553 INFO SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/config.yaml config.yaml +2024-08-23 16:37:53,555 INFO SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json wandb-metadata.json +2024-08-23 16:37:53,555 INFO SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json wandb-summary.json +2024-08-23 16:37:53,556 INFO SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log output.log +2024-08-23 16:37:53,558 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 10 +2024-08-23 16:37:53,558 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:53,559 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:53,559 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-23 16:37:53,560 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:53,560 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-23 16:37:53,560 INFO SenderThread:11966 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:37:53,672 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:53,672 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:53,988 INFO wandb-upload_0:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt +2024-08-23 16:37:54,025 INFO wandb-upload_1:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/config.yaml +2024-08-23 16:37:54,673 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:54,673 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:55,582 INFO wandb-upload_2:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json +2024-08-23 16:37:55,638 INFO wandb-upload_3:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/output.log +2024-08-23 16:37:55,673 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:55,674 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:55,674 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:37:55,838 INFO Thread-11 (_thread_body):11966 [sender.py:transition_state():617] send defer: 11 +2024-08-23 16:37:55,839 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:55,839 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-23 16:37:55,839 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:55,839 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-23 16:37:55,839 INFO SenderThread:11966 [file_pusher.py:join():178] waiting for file pusher +2024-08-23 16:37:55,840 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 12 +2024-08-23 16:37:55,840 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:55,840 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-23 16:37:55,840 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:55,840 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-23 16:37:55,840 INFO SenderThread:11966 [file_stream.py:finish():595] file stream finish called +2024-08-23 16:37:56,021 INFO SenderThread:11966 [file_stream.py:finish():599] file stream finish is done +2024-08-23 16:37:56,021 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 13 +2024-08-23 16:37:56,021 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:56,021 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-23 16:37:56,021 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:56,021 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-23 16:37:56,021 INFO SenderThread:11966 [sender.py:transition_state():617] send defer: 14 +2024-08-23 16:37:56,021 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:37:56,021 DEBUG SenderThread:11966 [sender.py:send():382] send: final +2024-08-23 16:37:56,021 INFO HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-23 16:37:56,022 DEBUG SenderThread:11966 [sender.py:send():382] send: footer +2024-08-23 16:37:56,022 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: defer +2024-08-23 16:37:56,022 INFO SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-23 16:37:56,022 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:56,022 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:56,023 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:37:56,023 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:37:56,023 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: server_info +2024-08-23 16:37:56,023 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: get_summary +2024-08-23 16:37:56,024 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: server_info +2024-08-23 16:37:56,025 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-23 16:37:56,026 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:37:56,027 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: job_info +2024-08-23 16:37:56,184 DEBUG SenderThread:11966 [sender.py:send_request():409] send_request: job_info +2024-08-23 16:37:56,185 INFO MainThread:11966 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-23 16:37:56,185 INFO MainThread:11966 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-23 16:37:56,186 INFO MainThread:11966 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-23 16:37:56,186 DEBUG HandlerThread:11966 [handler.py:handle_request():146] handle_request: shutdown +2024-08-23 16:37:56,186 INFO HandlerThread:11966 [handler.py:finish():869] shutting down handler +2024-08-23 16:37:57,027 INFO WriterThread:11966 [datastore.py:close():296] close: /project/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb +2024-08-23 16:37:57,185 INFO SenderThread:11966 [sender.py:finish():1572] shutting down sender +2024-08-23 16:37:57,185 INFO SenderThread:11966 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:37:57,185 INFO SenderThread:11966 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240823_162922-z3gs82jm/logs/debug.log b/wandb/run-20240823_162922-z3gs82jm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fe64d2301b2964b3a7c1753f10da3386b74aa05a --- /dev/null +++ b/wandb/run-20240823_162922-z3gs82jm/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-23 16:29:22,877 INFO MainThread:11895 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_setup.py:_flush():76] Configure stats pid to 11895 +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_162922-z3gs82jm/logs/debug.log +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_162922-z3gs82jm/logs/debug-internal.log +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_init.py:init():566] calling init triggers +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 2048, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:29:10', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 640, 'micro_batch_size': 5, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 128} +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_init.py:init():616] starting backend +2024-08-23 16:29:22,878 INFO MainThread:11895 [wandb_init.py:init():620] setting up manager +2024-08-23 16:29:22,883 INFO MainThread:11895 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-23 16:29:22,884 INFO MainThread:11895 [wandb_init.py:init():628] backend started and connected +2024-08-23 16:29:22,889 INFO MainThread:11895 [wandb_init.py:init():720] updated telemetry +2024-08-23 16:29:22,900 INFO MainThread:11895 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-23 16:29:23,299 INFO MainThread:11895 [wandb_run.py:_on_init():2262] communicating current version +2024-08-23 16:29:23,322 INFO MainThread:11895 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-23 16:29:23,322 INFO MainThread:11895 [wandb_init.py:init():804] starting run threads in backend +2024-08-23 16:29:23,417 INFO MainThread:11895 [wandb_run.py:_console_start():2241] atexit reg +2024-08-23 16:29:23,418 INFO MainThread:11895 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-23 16:29:23,418 INFO MainThread:11895 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-23 16:29:23,418 INFO MainThread:11895 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-23 16:29:23,419 INFO MainThread:11895 [wandb_init.py:init():847] run started, returning control to user process +2024-08-23 16:29:30,498 INFO MainThread:11895 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 2048, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-23 16:29:30,499 INFO MainThread:11895 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-23 16:37:57,186 WARNING MsgRouterThr:11895 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb b/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..68acf99c53a42e469fe7f74a1a38f16042361cee Binary files /dev/null and b/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb differ diff --git a/wandb/run-20240824_202452-bsrfks94/files/config.yaml b/wandb/run-20240824_202452-bsrfks94/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ad6c748be80e241099a9d1165077f00730a6f6 --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/files/config.yaml @@ -0,0 +1,321 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +test_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B_train_2024-08-24-20:24:37 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 3.5e-06 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 23178 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 3.5e-07 +train_iters: + desc: null + value: 23178 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1280 +micro_batch_size: + desc: null + value: 16 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 8 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724498692.419655 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20240824_202452-bsrfks94/files/output.log b/wandb/run-20240824_202452-bsrfks94/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..29ec19129308866369d74ef906daf4374a816b20 --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/files/output.log @@ -0,0 +1,51 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Unable to save the indexes because path_to_cache is None +Traceback (most recent call last): + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 270, in build_generic_dataset + dataset = cls(*args) + File "/project/megatron_lm/megatron/core/datasets/indexed_dataset.py", line 359, in __init__ + self.initialize(path_prefix, multimodal) + File "/project/megatron_lm/megatron/core/datasets/indexed_dataset.py", line 374, in initialize + self.index = _IndexReader(get_idx_path(self.path_prefix), self.multimodal) + File "/project/megatron_lm/megatron/core/datasets/indexed_dataset.py", line 233, in __init__ + with open(idx_path, "rb") as stream: +FileNotFoundError: [Errno 2] No such file or directory: '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document.idx' +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 162, in main + train_dataset, validation_dataset, test_dataset = build_train_valid_test_datasets() + File "/project/src/llama_recipes/datasets/pretrain_dataset.py", line 76, in build_train_valid_test_datasets + return train_valid_test_datasets_provider(train_val_test_num_samples) + File "/project/src/llama_recipes/datasets/pretrain_dataset.py", line 46, in train_valid_test_datasets_provider + ).build() + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 56, in build + return self._build_blended_dataset_splits() + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 162, in _build_blended_dataset_splits + self._build_megatron_dataset_splits( + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 199, in _build_megatron_dataset_splits + indexed_dataset = self.build_generic_dataset( + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 278, in build_generic_dataset + raise Exception(log) from err +Exception: Failed to write dataset materials to the data cache directory. Please supply a directory to which you have write access via the path_to_cache attribute in BlendedMegatronDatasetConfig and retry. Refer to the preserved traceback above for more information. +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... \ No newline at end of file diff --git a/wandb/run-20240824_202452-bsrfks94/files/requirements.txt b/wandb/run-20240824_202452-bsrfks94/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240824_202452-bsrfks94/files/wandb-metadata.json b/wandb/run-20240824_202452-bsrfks94/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e9f69e6c228dc7237db5ed400753d314742c11f2 --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/files/wandb-metadata.json @@ -0,0 +1,880 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-24T11:24:53.025954", + "startedAt": "2024-08-24T11:24:52.405256", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "16", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "1280", + "--train-iters", + "23178", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--test-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--lr", + "3.5e-6", + "--min-lr", + "3.5e-7", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "23178", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B_train_2024-08-24-20:24:37" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 144, + "cpu_count_logical": 144, + "cpu_freq": { + "current": 2400.0340000000015, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 453.4449462890625 + } +} diff --git a/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json b/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e682bae6b5eaeba8295fd0fffdc51474a259249e --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 5}} \ No newline at end of file diff --git a/wandb/run-20240824_202452-bsrfks94/logs/debug-internal.log b/wandb/run-20240824_202452-bsrfks94/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b26be08046085477da659bba146df8874440ac14 --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/logs/debug-internal.log @@ -0,0 +1,190 @@ +2024-08-24 20:24:52,440 INFO StreamThr :30769 [internal.py:wandb_internal():86] W&B internal server running at pid: 30769, started at: 2024-08-24 20:24:52.421070 +2024-08-24 20:24:52,441 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: status +2024-08-24 20:24:52,443 INFO WriterThread:30769 [datastore.py:open_for_write():87] open: /project/wandb/run-20240824_202452-bsrfks94/run-bsrfks94.wandb +2024-08-24 20:24:52,445 DEBUG SenderThread:30769 [sender.py:send():382] send: header +2024-08-24 20:24:52,445 DEBUG SenderThread:30769 [sender.py:send():382] send: run +2024-08-24 20:24:52,876 INFO SenderThread:30769 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240824_202452-bsrfks94/files +2024-08-24 20:24:52,876 INFO SenderThread:30769 [sender.py:_start_run_threads():1136] run started: bsrfks94 with start time 1724498692.419655 +2024-08-24 20:24:52,882 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: check_version +2024-08-24 20:24:52,882 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: check_version +2024-08-24 20:24:52,951 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: run_start +2024-08-24 20:24:52,958 DEBUG HandlerThread:30769 [system_info.py:__init__():27] System info init +2024-08-24 20:24:52,958 DEBUG HandlerThread:30769 [system_info.py:__init__():42] System info init done +2024-08-24 20:24:52,958 INFO HandlerThread:30769 [system_monitor.py:start():194] Starting system monitor +2024-08-24 20:24:52,959 INFO SystemMonitor:30769 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-24 20:24:52,959 INFO HandlerThread:30769 [system_monitor.py:probe():214] Collecting system info +2024-08-24 20:24:52,959 INFO SystemMonitor:30769 [interfaces.py:start():190] Started cpu monitoring +2024-08-24 20:24:52,959 INFO SystemMonitor:30769 [interfaces.py:start():190] Started disk monitoring +2024-08-24 20:24:52,961 INFO SystemMonitor:30769 [interfaces.py:start():190] Started gpu monitoring +2024-08-24 20:24:52,962 INFO SystemMonitor:30769 [interfaces.py:start():190] Started memory monitoring +2024-08-24 20:24:52,962 INFO SystemMonitor:30769 [interfaces.py:start():190] Started network monitoring +2024-08-24 20:24:53,025 DEBUG HandlerThread:30769 [system_info.py:probe():151] Probing system +2024-08-24 20:24:53,028 DEBUG HandlerThread:30769 [system_info.py:_probe_git():136] Probing git +2024-08-24 20:24:53,041 DEBUG HandlerThread:30769 [system_info.py:_probe_git():144] Probing git done +2024-08-24 20:24:53,041 DEBUG HandlerThread:30769 [system_info.py:probe():199] Probing system done +2024-08-24 20:24:53,041 DEBUG HandlerThread:30769 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-24T11:24:53.025954', 'startedAt': '2024-08-24T11:24:52.405256', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '16', '--valid_micro_batch_size', '1', '--global-batch-size', '1280', '--train-iters', '23178', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--test-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--lr', '3.5e-6', '--min-lr', '3.5e-7', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '23178', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'NO_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-0.5B_train_2024-08-24-20:24:37'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 144, 'cpu_count_logical': 144, 'cpu_freq': {'current': 2400.0340000000015, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 453.4449462890625}} +2024-08-24 20:24:53,042 INFO HandlerThread:30769 [system_monitor.py:probe():224] Finished collecting system info +2024-08-24 20:24:53,042 INFO HandlerThread:30769 [system_monitor.py:probe():227] Publishing system info +2024-08-24 20:24:53,044 INFO HandlerThread:30769 [system_monitor.py:probe():229] Finished publishing system info +2024-08-24 20:24:53,050 DEBUG SenderThread:30769 [sender.py:send():382] send: files +2024-08-24 20:24:53,050 INFO SenderThread:30769 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-24 20:24:53,062 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: python_packages +2024-08-24 20:24:53,062 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: stop_status +2024-08-24 20:24:53,062 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-24 20:24:53,062 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: python_packages +2024-08-24 20:24:53,083 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: stop_status +2024-08-24 20:24:53,350 DEBUG SenderThread:30769 [sender.py:send():382] send: telemetry +2024-08-24 20:24:53,879 INFO Thread-12 :30769 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202452-bsrfks94/files/requirements.txt +2024-08-24 20:24:53,879 INFO Thread-12 :30769 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202452-bsrfks94/files/wandb-metadata.json +2024-08-24 20:24:53,879 INFO Thread-12 :30769 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202452-bsrfks94/files/output.log +2024-08-24 20:24:53,890 INFO wandb-upload_0:30769 [upload_job.py:push():131] Uploaded file /tmp/tmphj31b5_iwandb/35wh2nrr-wandb-metadata.json +2024-08-24 20:24:55,879 INFO Thread-12 :30769 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202452-bsrfks94/files/output.log +2024-08-24 20:24:57,881 INFO Thread-12 :30769 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202452-bsrfks94/files/output.log +2024-08-24 20:24:58,144 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: status_report +2024-08-24 20:24:58,755 DEBUG SenderThread:30769 [sender.py:send():382] send: exit +2024-08-24 20:24:58,755 INFO SenderThread:30769 [sender.py:send_exit():589] handling exit code: 1 +2024-08-24 20:24:58,755 INFO SenderThread:30769 [sender.py:send_exit():591] handling runtime: 5 +2024-08-24 20:24:58,793 INFO SenderThread:30769 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-24 20:24:58,793 INFO SenderThread:30769 [sender.py:send_exit():597] send defer +2024-08-24 20:24:58,794 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:58,794 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-24 20:24:58,794 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:58,794 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-24 20:24:58,794 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 1 +2024-08-24 20:24:58,794 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:58,794 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-24 20:24:58,794 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:58,794 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-24 20:24:58,794 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 2 +2024-08-24 20:24:58,795 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:58,795 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-24 20:24:58,795 INFO HandlerThread:30769 [system_monitor.py:finish():203] Stopping system monitor +2024-08-24 20:24:58,795 DEBUG SystemMonitor:30769 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-24 20:24:58,795 INFO HandlerThread:30769 [interfaces.py:finish():202] Joined cpu monitor +2024-08-24 20:24:58,795 DEBUG SystemMonitor:30769 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-24 20:24:58,796 INFO HandlerThread:30769 [interfaces.py:finish():202] Joined disk monitor +2024-08-24 20:24:58,796 DEBUG SystemMonitor:30769 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-24 20:24:58,882 INFO Thread-12 :30769 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json +2024-08-24 20:24:59,883 INFO Thread-12 :30769 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202452-bsrfks94/files/output.log +2024-08-24 20:24:59,960 INFO HandlerThread:30769 [interfaces.py:finish():202] Joined gpu monitor +2024-08-24 20:24:59,960 INFO HandlerThread:30769 [interfaces.py:finish():202] Joined memory monitor +2024-08-24 20:24:59,960 INFO HandlerThread:30769 [interfaces.py:finish():202] Joined network monitor +2024-08-24 20:24:59,961 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:24:59,962 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:59,962 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-24 20:24:59,962 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 3 +2024-08-24 20:24:59,962 DEBUG SenderThread:30769 [sender.py:send():382] send: stats +2024-08-24 20:24:59,963 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:24:59,963 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:59,963 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-24 20:24:59,964 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:59,964 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-24 20:24:59,964 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 4 +2024-08-24 20:24:59,964 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:59,964 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-24 20:24:59,964 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:59,964 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-24 20:24:59,964 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 5 +2024-08-24 20:24:59,964 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:59,964 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-24 20:24:59,964 DEBUG SenderThread:30769 [sender.py:send():382] send: summary +2024-08-24 20:24:59,966 INFO SenderThread:30769 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-24 20:24:59,966 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:59,966 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-24 20:24:59,966 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 6 +2024-08-24 20:24:59,966 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:24:59,966 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-24 20:24:59,966 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:24:59,966 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-24 20:24:59,969 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: status_report +2024-08-24 20:25:00,170 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 7 +2024-08-24 20:25:00,170 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:00,170 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-24 20:25:00,170 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:00,170 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-24 20:25:00,755 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:25:00,884 INFO Thread-12 :30769 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json +2024-08-24 20:25:00,885 INFO Thread-12 :30769 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202452-bsrfks94/files/config.yaml +2024-08-24 20:25:03,269 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 8 +2024-08-24 20:25:03,270 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:25:03,270 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:03,270 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-24 20:25:03,270 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:03,270 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-24 20:25:03,270 INFO SenderThread:30769 [job_builder.py:build():296] Attempting to build job artifact +2024-08-24 20:25:03,271 INFO SenderThread:30769 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-24 20:25:03,286 INFO SenderThread:30769 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-24 20:25:03,296 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 9 +2024-08-24 20:25:03,296 DEBUG SenderThread:30769 [sender.py:send():382] send: artifact +2024-08-24 20:25:03,296 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:03,297 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-24 20:25:03,756 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:25:03,886 INFO Thread-12 :30769 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202452-bsrfks94/files/output.log +2024-08-24 20:25:04,160 INFO SenderThread:30769 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MzU1Mzg0Mw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjgwMzg3NA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MzU1Mzg0Mw==', 'versionIndex': 0}}} +2024-08-24 20:25:04,160 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:04,160 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-24 20:25:04,160 INFO SenderThread:30769 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-24 20:25:04,887 INFO SenderThread:30769 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240824_202452-bsrfks94/files +2024-08-24 20:25:04,887 INFO SenderThread:30769 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202452-bsrfks94/files/requirements.txt requirements.txt +2024-08-24 20:25:04,888 INFO SenderThread:30769 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202452-bsrfks94/files/config.yaml config.yaml +2024-08-24 20:25:04,888 INFO SenderThread:30769 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202452-bsrfks94/files/wandb-metadata.json wandb-metadata.json +2024-08-24 20:25:04,889 INFO SenderThread:30769 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json wandb-summary.json +2024-08-24 20:25:04,891 INFO SenderThread:30769 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202452-bsrfks94/files/output.log output.log +2024-08-24 20:25:04,892 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 10 +2024-08-24 20:25:04,893 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:25:04,894 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:04,894 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-24 20:25:04,894 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:04,895 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-24 20:25:04,895 INFO SenderThread:30769 [file_pusher.py:finish():172] shutting down file pusher +2024-08-24 20:25:05,306 INFO wandb-upload_0:30769 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202452-bsrfks94/files/requirements.txt +2024-08-24 20:25:05,358 INFO wandb-upload_3:30769 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202452-bsrfks94/files/output.log +2024-08-24 20:25:05,363 INFO wandb-upload_1:30769 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202452-bsrfks94/files/config.yaml +2024-08-24 20:25:05,389 INFO wandb-upload_2:30769 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202452-bsrfks94/files/wandb-summary.json +2024-08-24 20:25:05,589 INFO Thread-11 (_thread_body):30769 [sender.py:transition_state():617] send defer: 11 +2024-08-24 20:25:05,589 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:05,589 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-24 20:25:05,590 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:05,590 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-24 20:25:05,590 INFO SenderThread:30769 [file_pusher.py:join():178] waiting for file pusher +2024-08-24 20:25:05,590 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 12 +2024-08-24 20:25:05,590 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:05,590 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-24 20:25:05,590 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:05,590 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-24 20:25:05,590 INFO SenderThread:30769 [file_stream.py:finish():595] file stream finish called +2024-08-24 20:25:05,757 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:25:05,781 INFO SenderThread:30769 [file_stream.py:finish():599] file stream finish is done +2024-08-24 20:25:05,781 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 13 +2024-08-24 20:25:05,781 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:25:05,781 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:05,781 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-24 20:25:05,781 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:05,781 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-24 20:25:05,781 INFO SenderThread:30769 [sender.py:transition_state():617] send defer: 14 +2024-08-24 20:25:05,782 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:25:05,782 DEBUG SenderThread:30769 [sender.py:send():382] send: final +2024-08-24 20:25:05,782 INFO HandlerThread:30769 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-24 20:25:05,782 DEBUG SenderThread:30769 [sender.py:send():382] send: footer +2024-08-24 20:25:05,782 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: defer +2024-08-24 20:25:05,782 INFO SenderThread:30769 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-24 20:25:05,782 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:25:05,783 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:25:05,783 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:25:05,783 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:25:05,783 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: server_info +2024-08-24 20:25:05,783 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: get_summary +2024-08-24 20:25:05,784 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: server_info +2024-08-24 20:25:05,784 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-24 20:25:05,785 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-24 20:25:05,785 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: job_info +2024-08-24 20:25:05,964 DEBUG SenderThread:30769 [sender.py:send_request():409] send_request: job_info +2024-08-24 20:25:05,964 INFO MainThread:30769 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-24 20:25:05,964 INFO MainThread:30769 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-24 20:25:05,964 INFO MainThread:30769 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-24 20:25:05,965 DEBUG HandlerThread:30769 [handler.py:handle_request():146] handle_request: shutdown +2024-08-24 20:25:05,965 INFO HandlerThread:30769 [handler.py:finish():869] shutting down handler +2024-08-24 20:25:06,786 INFO WriterThread:30769 [datastore.py:close():296] close: /project/wandb/run-20240824_202452-bsrfks94/run-bsrfks94.wandb +2024-08-24 20:25:06,964 INFO SenderThread:30769 [sender.py:finish():1572] shutting down sender +2024-08-24 20:25:06,964 INFO SenderThread:30769 [file_pusher.py:finish():172] shutting down file pusher +2024-08-24 20:25:06,964 INFO SenderThread:30769 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240824_202452-bsrfks94/logs/debug.log b/wandb/run-20240824_202452-bsrfks94/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6071c7e3cafe8605cff919a60b6f84c5735614da --- /dev/null +++ b/wandb/run-20240824_202452-bsrfks94/logs/debug.log @@ -0,0 +1,28 @@ +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Configure stats pid to 30143 +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train baseline'} +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-24 20:24:52,412 INFO MainThread:30143 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-24 20:24:52,413 INFO MainThread:30143 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240824_202452-bsrfks94/logs/debug.log +2024-08-24 20:24:52,413 INFO MainThread:30143 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240824_202452-bsrfks94/logs/debug-internal.log +2024-08-24 20:24:52,413 INFO MainThread:30143 [wandb_init.py:init():566] calling init triggers +2024-08-24 20:24:52,413 INFO MainThread:30143 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'test_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B_train_2024-08-24-20:24:37', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 3.5e-06, 'lr_decay_style': 'cosine', 'lr_decay_iters': 23178, 'lr_warmup_iters': 500, 'min_lr': 3.5e-07, 'train_iters': 23178, 'train_samples': None, 'global_batch_size': 1280, 'micro_batch_size': 16, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 8, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 10} +2024-08-24 20:24:52,413 INFO MainThread:30143 [wandb_init.py:init():616] starting backend +2024-08-24 20:24:52,413 INFO MainThread:30143 [wandb_init.py:init():620] setting up manager +2024-08-24 20:24:52,418 INFO MainThread:30143 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-24 20:24:52,419 INFO MainThread:30143 [wandb_init.py:init():628] backend started and connected +2024-08-24 20:24:52,426 INFO MainThread:30143 [wandb_init.py:init():720] updated telemetry +2024-08-24 20:24:52,439 INFO MainThread:30143 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-24 20:24:52,881 INFO MainThread:30143 [wandb_run.py:_on_init():2262] communicating current version +2024-08-24 20:24:52,904 INFO MainThread:30143 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-24 20:24:52,904 INFO MainThread:30143 [wandb_init.py:init():804] starting run threads in backend +2024-08-24 20:24:53,060 INFO MainThread:30143 [wandb_run.py:_console_start():2241] atexit reg +2024-08-24 20:24:53,061 INFO MainThread:30143 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-24 20:24:53,061 INFO MainThread:30143 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-24 20:24:53,061 INFO MainThread:30143 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-24 20:24:53,062 INFO MainThread:30143 [wandb_init.py:init():847] run started, returning control to user process +2024-08-24 20:25:06,965 WARNING MsgRouterThr:30143 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240824_202452-bsrfks94/run-bsrfks94.wandb b/wandb/run-20240824_202452-bsrfks94/run-bsrfks94.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0c119c511230f8586b376c7e319a8c9f25597a8a Binary files /dev/null and b/wandb/run-20240824_202452-bsrfks94/run-bsrfks94.wandb differ diff --git a/wandb/run-20240824_202743-hay4ewdq/files/config.yaml b/wandb/run-20240824_202743-hay4ewdq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2eba32ea5b361460f9d6a835afea8a74036e1e9 --- /dev/null +++ b/wandb/run-20240824_202743-hay4ewdq/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +test_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B_train_2024-08-24-20:27:28 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 3.5e-06 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 23178 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 3.5e-07 +train_iters: + desc: null + value: 23178 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1280 +micro_batch_size: + desc: null + value: 16 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 8 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724498863.855821 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240824_202743-hay4ewdq/files/output.log b/wandb/run-20240824_202743-hay4ewdq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e34a2b87cb8a450d95c39aae7ae929e4f6ce7258 --- /dev/null +++ b/wandb/run-20240824_202743-hay4ewdq/files/output.log @@ -0,0 +1,41340 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Unable to save the indexes because path_to_cache is None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 20.31493404786331, Tokens per sec: 50342.64444533221, Loss: 4.021102428436279 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 70.3606575657509, Tokens per sec: 174361.46031421292, Loss: 3.9913618564605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 69.51778029285151, Tokens per sec: 172272.71758137163, Loss: 4.004281997680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 70.56102033055834, Tokens per sec: 174857.98131718635, Loss: 4.003636360168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 68.77284487888232, Tokens per sec: 170426.68556414577, Loss: 4.010791301727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 70.59182986557522, Tokens per sec: 174934.33073890867, Loss: 4.024072170257568 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 69.54987674577595, Tokens per sec: 172352.25612743452, Loss: 4.018331527709961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 69.43400276921317, Tokens per sec: 172065.10764893968, Loss: 3.9950156211853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 70.6177615804778, Tokens per sec: 174998.59238505148, Loss: 4.005020618438721 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 69.57237975911528, Tokens per sec: 172408.02107340092, Loss: 4.030388832092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 69.41239651018958, Tokens per sec: 172011.56495895333, Loss: 4.024319648742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 70.60895126877233, Tokens per sec: 174976.7594621094, Loss: 4.013145446777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 70.54588592757862, Tokens per sec: 174820.47659941053, Loss: 4.0108113288879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 69.58279189340445, Tokens per sec: 172433.8234891608, Loss: 4.021638870239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 69.40855504938322, Tokens per sec: 172002.04539590335, Loss: 4.005174160003662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 70.59606184124202, Tokens per sec: 174944.81804080162, Loss: 4.01125431060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 70.55848849415776, Tokens per sec: 174851.70714768092, Loss: 4.017087936401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 68.38534519334374, Tokens per sec: 169466.4186567669, Loss: 4.022975921630859 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 70.59256052155754, Tokens per sec: 174936.14138492124, Loss: 3.9988558292388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 70.56769303651309, Tokens per sec: 174874.5170176015, Loss: 4.028948783874512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 69.47287015173978, Tokens per sec: 172161.42530443627, Loss: 4.014774799346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 69.45277021405852, Tokens per sec: 172111.6154446712, Loss: 4.007519245147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 70.61326071696523, Tokens per sec: 174987.43874945628, Loss: 3.980170488357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 70.5872309306839, Tokens per sec: 174922.93407163347, Loss: 3.995248317718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 68.41238590925663, Tokens per sec: 169533.4285295798, Loss: 4.0038347244262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 70.6269022457454, Tokens per sec: 175021.243960512, Loss: 4.0229620933532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 70.61151866290155, Tokens per sec: 174983.12174758338, Loss: 4.015242576599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 69.46380679752008, Tokens per sec: 172138.96531426895, Loss: 4.010164737701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 69.64800575758372, Tokens per sec: 172595.4306284972, Loss: 4.027735233306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 70.53135653754995, Tokens per sec: 174784.471170375, Loss: 4.005396842956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 70.54166156316181, Tokens per sec: 174810.0081590335, Loss: 4.017270565032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 69.7123909376263, Tokens per sec: 172754.9841972558, Loss: 3.9913690090179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 69.48866115349215, Tokens per sec: 172200.55714630106, Loss: 3.997417449951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 70.5486352297008, Tokens per sec: 174827.28967292994, Loss: 3.974970579147339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 69.48643399184222, Tokens per sec: 172195.03799439047, Loss: 4.026078224182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 69.57464357470982, Tokens per sec: 172413.63105782412, Loss: 4.012585163116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 70.58592116912912, Tokens per sec: 174919.68833821858, Loss: 3.9978582859039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 69.57019692282235, Tokens per sec: 172402.61176460778, Loss: 4.003357410430908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 70.5864043001022, Tokens per sec: 174920.88558999114, Loss: 4.003114223480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 69.59394874382517, Tokens per sec: 172461.47139353186, Loss: 4.008548736572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 70.56903376418151, Tokens per sec: 174877.83948845716, Loss: 4.004150390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 69.47916538004189, Tokens per sec: 172177.02557364554, Loss: 4.019111633300781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 70.59019476147053, Tokens per sec: 174930.27877081567, Loss: 3.970932722091675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 69.61497636610513, Tokens per sec: 172513.58015792543, Loss: 4.021045207977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 69.49882242719224, Tokens per sec: 172225.73790188663, Loss: 4.0026116371154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 70.61406366825518, Tokens per sec: 174989.42855120517, Loss: 4.034980773925781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 70.59749144955272, Tokens per sec: 174948.36076767987, Loss: 4.017622947692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 68.4420514845621, Tokens per sec: 169606.94309311965, Loss: 3.991218090057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 70.49349535288168, Tokens per sec: 174690.64698401326, Loss: 4.026672840118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 70.56691566936712, Tokens per sec: 174872.59061617972, Loss: 4.01682710647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 69.63367828476866, Tokens per sec: 172559.92557256008, Loss: 3.968195676803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 69.46791142716381, Tokens per sec: 172149.1370386884, Loss: 4.009128093719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 70.59951587159352, Tokens per sec: 174953.37750850548, Loss: 3.9860217571258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 70.52738223448303, Tokens per sec: 174774.62240956957, Loss: 3.9982218742370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 68.37659926103865, Tokens per sec: 169444.74527307093, Loss: 4.009627819061279 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 70.59595537553935, Tokens per sec: 174944.5542070622, Loss: 4.013144493103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 70.58427204449619, Tokens per sec: 174915.60162571212, Loss: 3.9918715953826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 68.53366029421855, Tokens per sec: 169833.9598149918, Loss: 4.008973598480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 70.59142262603883, Tokens per sec: 174933.32155447637, Loss: 3.987338066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 70.51490622951177, Tokens per sec: 174743.70549490536, Loss: 3.974925994873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 70.59001040238743, Tokens per sec: 174929.8219086989, Loss: 4.009181976318359 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 68.50716126651228, Tokens per sec: 169768.29230522682, Loss: 4.005714416503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 70.60110510570826, Tokens per sec: 174957.3158057104, Loss: 3.993781566619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 70.57462474670865, Tokens per sec: 174891.694559057, Loss: 3.971559524536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 69.52502200987348, Tokens per sec: 172290.66335389303, Loss: 3.9642550945281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 69.46269516643385, Tokens per sec: 172136.21056998102, Loss: 3.9960176944732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 70.58822247624401, Tokens per sec: 174925.39123075918, Loss: 3.9949052333831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 70.53273826323756, Tokens per sec: 174787.8952388386, Loss: 4.004136562347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 69.63055709924508, Tokens per sec: 172552.1909310057, Loss: 4.0013957023620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 69.67397594091233, Tokens per sec: 172659.78760363738, Loss: 3.973294734954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 70.61922385899868, Tokens per sec: 175002.2160723095, Loss: 3.9885082244873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 69.51175525375857, Tokens per sec: 172257.78687078643, Loss: 3.996652603149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 69.59775948261765, Tokens per sec: 172470.91482404713, Loss: 3.995777130126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 70.54405744907885, Tokens per sec: 174815.94542826453, Loss: 3.971942663192749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 69.43002808615216, Tokens per sec: 172055.25794646706, Loss: 3.966489315032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 70.54211448142212, Tokens per sec: 174811.13054037557, Loss: 3.9846043586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 69.56250425449532, Tokens per sec: 172383.54848507605, Loss: 3.9816927909851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 69.2932320548575, Tokens per sec: 171716.262311592, Loss: 3.994716167449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 70.58948587656928, Tokens per sec: 174928.52207594024, Loss: 3.9858930110931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 70.64937929409223, Tokens per sec: 175076.9445623667, Loss: 3.982977867126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 69.63413580180276, Tokens per sec: 172561.05935017025, Loss: 3.987461566925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 69.49106940078987, Tokens per sec: 172206.52504839518, Loss: 3.977959394454956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 70.56445607284913, Tokens per sec: 174866.49546506122, Loss: 3.988520622253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 69.56543255368207, Tokens per sec: 172390.8051330396, Loss: 3.987250328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 69.49498978472184, Tokens per sec: 172216.24019164455, Loss: 3.9773612022399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 70.58286244057993, Tokens per sec: 174912.10847192694, Loss: 3.9984710216522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 70.57282967897282, Tokens per sec: 174887.2461834646, Loss: 3.969658613204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 69.67242069923157, Tokens per sec: 172655.9335434284, Loss: 3.974128484725952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 69.42805342486994, Tokens per sec: 172050.3645182828, Loss: 3.9841015338897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 70.6044704631032, Tokens per sec: 174965.65553205978, Loss: 3.952324628829956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 70.6214775523615, Tokens per sec: 175007.80097272736, Loss: 3.9992311000823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 68.49076163055784, Tokens per sec: 169727.65220076265, Loss: 3.97517991065979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 70.54528386892623, Tokens per sec: 174818.98463174768, Loss: 3.9739856719970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 70.51284904484444, Tokens per sec: 174738.60756468217, Loss: 3.990055561065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 68.95543061511054, Tokens per sec: 170879.15313199852, Loss: 3.979052782058716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 70.63103878196362, Tokens per sec: 175031.4947529378, Loss: 3.967052459716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 70.67136018343187, Tokens per sec: 175131.4156275452, Loss: 3.9920458793640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 69.52812424512094, Tokens per sec: 172298.35103457567, Loss: 3.943685293197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 69.68695427647634, Tokens per sec: 172691.94934884668, Loss: 3.9897377490997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 70.54250363220446, Tokens per sec: 174812.09489888273, Loss: 3.968249559402466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 70.63192807519904, Tokens per sec: 175033.69851953894, Loss: 3.9683380126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 69.4634397191836, Tokens per sec: 172138.05565371417, Loss: 3.983828067779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 69.54053179000472, Tokens per sec: 172329.09829760154, Loss: 3.9892184734344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 70.59247129126062, Tokens per sec: 174935.92026241595, Loss: 3.971158027648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 69.44794315141336, Tokens per sec: 172099.65345169164, Loss: 3.9688918590545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 70.58233933835378, Tokens per sec: 174910.81216698603, Loss: 3.983124017715454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 69.65679799523636, Tokens per sec: 172617.21876194538, Loss: 3.968071460723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 70.59283347550243, Tokens per sec: 174936.81779486506, Loss: 3.946690797805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 69.41953399310454, Tokens per sec: 172029.2524278752, Loss: 3.965829372406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 69.49050764907425, Tokens per sec: 172205.1329657618, Loss: 3.9423227310180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 70.60658323291874, Tokens per sec: 174970.89120840348, Loss: 3.956374168395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 69.28461419665639, Tokens per sec: 171694.9063096342, Loss: 3.954908609390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 70.55338347087608, Tokens per sec: 174839.0563376242, Loss: 3.9590044021606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 69.5602178415361, Tokens per sec: 172377.88250189394, Loss: 3.937296152114868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 69.48749971524606, Tokens per sec: 172197.67897438427, Loss: 3.9456331729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 70.6240669980021, Tokens per sec: 175014.21790427578, Loss: 3.954662799835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 70.60249123200472, Tokens per sec: 174960.75077936743, Loss: 3.961935043334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 69.65598439853325, Tokens per sec: 172615.20257969, Loss: 3.9554898738861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 69.76205219906748, Tokens per sec: 172878.05027374663, Loss: 3.955028533935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 70.60200223182811, Tokens per sec: 174959.53898306194, Loss: 3.9778521060943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 70.57912152353195, Tokens per sec: 174902.838067386, Loss: 3.9629299640655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 68.36512442201561, Tokens per sec: 169416.30935791397, Loss: 3.9512126445770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 70.48721730085153, Tokens per sec: 174675.08927949803, Loss: 3.9597489833831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 70.56798875806425, Tokens per sec: 174875.24984818167, Loss: 3.924793243408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 68.50448892696723, Tokens per sec: 169761.66995345164, Loss: 3.9609711170196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 70.59822743811964, Tokens per sec: 174950.18462842453, Loss: 3.9414010047912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 70.56715400005427, Tokens per sec: 174873.18122587676, Loss: 3.945077657699585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 70.54229359946443, Tokens per sec: 174811.57441461593, Loss: 3.9409873485565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 69.34600957865352, Tokens per sec: 171847.05082948273, Loss: 3.9114596843719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 70.65462417227384, Tokens per sec: 175089.94194827176, Loss: 3.9381778240203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 70.59825544478257, Tokens per sec: 174950.25403202043, Loss: 3.91644287109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 68.39724920588887, Tokens per sec: 169495.91811118898, Loss: 3.918898105621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 70.58166218978167, Tokens per sec: 174909.1341182303, Loss: 3.9532487392425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 70.60316973124009, Tokens per sec: 174962.4321752155, Loss: 3.943495988845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 70.5400457841297, Tokens per sec: 174806.00408059917, Loss: 3.9465041160583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 68.77358591916294, Tokens per sec: 170428.52194359343, Loss: 3.9313855171203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 70.56474906801245, Tokens per sec: 174867.22153935197, Loss: 3.9258570671081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 70.62905707173388, Tokens per sec: 175026.58385668515, Loss: 3.931851387023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 69.40953889960288, Tokens per sec: 172004.48348512768, Loss: 3.9434869289398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 69.47517460285468, Tokens per sec: 172167.13598815523, Loss: 3.9357452392578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 70.57590093507623, Tokens per sec: 174894.8570944156, Loss: 3.9457449913024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 69.40186465036884, Tokens per sec: 171985.46585013738, Loss: 3.904700756072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 70.5932364391323, Tokens per sec: 174937.81638313, Loss: 3.905057430267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 69.57288938798196, Tokens per sec: 172409.28398987284, Loss: 3.9159905910491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 68.80521116266232, Tokens per sec: 170506.89278079304, Loss: 3.9270589351654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 70.56703011755, Tokens per sec: 174872.87423138443, Loss: 3.9242756366729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 69.6379244716307, Tokens per sec: 172570.4480913603, Loss: 3.9125149250030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 70.48607809743604, Tokens per sec: 174672.26620794082, Loss: 3.9053235054016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 69.43115425115562, Tokens per sec: 172058.04870740292, Loss: 3.90889310836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 70.58253504322325, Tokens per sec: 174911.29714521125, Loss: 3.949573516845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 69.56409345705477, Tokens per sec: 172387.48670408345, Loss: 3.928894281387329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 69.45052886259424, Tokens per sec: 172106.06112883746, Loss: 3.9052534103393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 70.55962035360584, Tokens per sec: 174854.51202007788, Loss: 3.908541440963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 70.56509809217552, Tokens per sec: 174868.08645967627, Loss: 3.914411783218384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 69.59008597303115, Tokens per sec: 172451.89902198495, Loss: 3.900752544403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 69.24009118414664, Tokens per sec: 171584.57338002033, Loss: 3.9074149131774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 70.49321480365937, Tokens per sec: 174689.95175213533, Loss: 3.9081900119781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 69.55389982179841, Tokens per sec: 172362.22575299657, Loss: 3.906296491622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 69.48404528910343, Tokens per sec: 172189.11852586642, Loss: 3.9099628925323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 70.5871236352066, Tokens per sec: 174922.66818162115, Loss: 3.8904993534088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 70.53762714595435, Tokens per sec: 174800.01042876564, Loss: 3.8872876167297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 68.47649691713487, Tokens per sec: 169692.30266951263, Loss: 3.9234673976898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 70.58663040590936, Tokens per sec: 174921.4459050889, Loss: 3.930980682373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 70.54193351268523, Tokens per sec: 174810.68207991027, Loss: 3.8911328315734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 69.39731673478195, Tokens per sec: 171974.19561431886, Loss: 3.8840203285217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 69.65016855636954, Tokens per sec: 172600.79028214043, Loss: 3.941223621368408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 70.56105255834719, Tokens per sec: 174858.0611811973, Loss: 3.892932891845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 70.51727743711793, Tokens per sec: 174749.58160856352, Loss: 3.912043571472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 68.4442349841021, Tokens per sec: 169612.35404551207, Loss: 3.8991494178771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 70.62403612573527, Tokens per sec: 175014.14139939772, Loss: 3.911205768585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 70.59907376719129, Tokens per sec: 174952.2819250949, Loss: 3.896237850189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 69.44703119733374, Tokens per sec: 172097.39352902232, Loss: 3.887273073196411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 69.67828358775165, Tokens per sec: 172670.46242703169, Loss: 3.9055185317993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 70.6058442547459, Tokens per sec: 174969.05993908603, Loss: 3.9158666133880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 70.59944403077525, Tokens per sec: 174953.19947904386, Loss: 3.893519401550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 69.1816355332498, Tokens per sec: 171439.71383767645, Loss: 3.9038586616516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 69.55479463697155, Tokens per sec: 172364.44320356767, Loss: 3.8971123695373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 70.5376352093014, Tokens per sec: 174800.03041062792, Loss: 3.8855457305908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 69.41220588447277, Tokens per sec: 172011.09256742793, Loss: 3.903794050216675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 70.59731788762717, Tokens per sec: 174947.93066210893, Loss: 3.8565001487731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 69.51197488320422, Tokens per sec: 172258.33113674694, Loss: 3.8874192237854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 69.42576059255356, Tokens per sec: 172044.6826272265, Loss: 3.887808322906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 70.57826842805463, Tokens per sec: 174900.72400281698, Loss: 3.8966903686523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 69.64521699573991, Tokens per sec: 172588.51976944078, Loss: 3.886871337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 70.5852193621091, Tokens per sec: 174917.94918311256, Loss: 3.892449140548706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 69.72667032483385, Tokens per sec: 172790.37009176047, Loss: 3.8820812702178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 70.55248487219211, Tokens per sec: 174836.82951109653, Loss: 3.8892412185668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 69.56389949281768, Tokens per sec: 172387.00603933705, Loss: 3.893630027770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 69.39093199354578, Tokens per sec: 171958.3735221983, Loss: 3.8652052879333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 70.55982549614227, Tokens per sec: 174855.02038588148, Loss: 3.879337787628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 70.54802194677741, Tokens per sec: 174825.76989028708, Loss: 3.8869967460632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 68.61414531216181, Tokens per sec: 170033.41055561186, Loss: 3.8934528827667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 70.6285810545737, Tokens per sec: 175025.40423372458, Loss: 3.8839519023895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 70.6257807948199, Tokens per sec: 175018.4648815806, Loss: 3.8976075649261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 70.58438473280914, Tokens per sec: 174915.88087976523, Loss: 3.8701508045196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 68.46598690044041, Tokens per sec: 169666.25769036982, Loss: 3.9006752967834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 70.5276197626994, Tokens per sec: 174775.21103065566, Loss: 3.8863394260406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 70.57996445892773, Tokens per sec: 174904.92695415372, Loss: 3.8620200157165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 69.43243017674921, Tokens per sec: 172061.21059157865, Loss: 3.887754201889038 +------------------------------------------------------------------ +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 70.54307306542393, Tokens per sec: 174813.50601712882, Loss: 3.8694040775299072 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/model.pt +[rank0]:[2024-08-24 20:53:39,602] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0065002789997379296, 'preprocessing_with_comm': 0.006895844999235123, 'state_converting': 0.9594847819989809, : 0.9742748469998332}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200, took 4.40s + eval ppl=37.881309509277344, eval loss=3.63445782661438 +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 68.11294800342382, Tokens per sec: 168791.38841311773, Loss: 3.865222454071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 67.38516551688711, Tokens per sec: 166987.86323962052, Loss: 3.8698275089263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 70.5902526632776, Tokens per sec: 174930.42225787198, Loss: 3.889863967895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 70.54232706025489, Tokens per sec: 174811.65733414073, Loss: 3.8844470977783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 70.51886330900423, Tokens per sec: 174753.511573796, Loss: 3.8772122859954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 68.75989144793596, Tokens per sec: 170394.5855353224, Loss: 3.8512027263641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 70.50255125206154, Tokens per sec: 174713.0884997706, Loss: 3.8560383319854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 70.59149944841027, Tokens per sec: 174933.51192877546, Loss: 3.8763463497161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 69.33612906389791, Tokens per sec: 171822.56582548982, Loss: 3.876448631286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 69.5278878500909, Tokens per sec: 172297.76522165016, Loss: 3.890062093734741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 70.5447966290177, Tokens per sec: 174817.7771975804, Loss: 3.8698744773864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 69.49971833470103, Tokens per sec: 172227.95805938577, Loss: 3.86995267868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 70.57989268613414, Tokens per sec: 174904.74909326484, Loss: 3.8889455795288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 69.61711592209613, Tokens per sec: 172518.8822133627, Loss: 3.876286745071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 70.49925638818789, Tokens per sec: 174704.92346413247, Loss: 3.828260898590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 69.45667911566287, Tokens per sec: 172121.3021334478, Loss: 3.8566737174987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 69.59900053131385, Tokens per sec: 172473.99027942913, Loss: 3.872551441192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 70.63280093789938, Tokens per sec: 175035.86156946415, Loss: 3.869687080383301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 69.49923251223257, Tokens per sec: 172226.75413779117, Loss: 3.8748226165771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 70.64944590050884, Tokens per sec: 175077.10962040437, Loss: 3.857649564743042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 69.62374207326813, Tokens per sec: 172535.30254589772, Loss: 3.848055362701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 69.42793831669074, Tokens per sec: 172050.07926753451, Loss: 3.8501243591308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 70.55754283423192, Tokens per sec: 174849.36369821167, Loss: 3.872023105621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 70.48654750352578, Tokens per sec: 174673.42944794055, Loss: 3.8792762756347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 68.50481080327242, Tokens per sec: 169762.4675983935, Loss: 3.871894121170044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 70.55574616242973, Tokens per sec: 174844.91134756606, Loss: 3.881235122680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 70.5284034423828, Tokens per sec: 174777.15307524055, Loss: 3.8330352306365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 70.55691181284453, Tokens per sec: 174847.799957701, Loss: 3.8448150157928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 68.51305701961533, Tokens per sec: 169782.90263088697, Loss: 3.8304178714752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 70.49587241421148, Tokens per sec: 174696.5376038434, Loss: 3.8634257316589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 70.59497395774648, Tokens per sec: 174942.12214565938, Loss: 3.8548641204833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 68.23479552283302, Tokens per sec: 169093.33998882605, Loss: 3.8692078590393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 70.54554545940913, Tokens per sec: 174819.63288178082, Loss: 3.857196569442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 70.60470496112116, Tokens per sec: 174966.23664398034, Loss: 3.8569493293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 70.57649604630431, Tokens per sec: 174896.3318455957, Loss: 3.859393835067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 68.5066814206813, Tokens per sec: 169767.10319439825, Loss: 3.869546413421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 70.5212325095451, Tokens per sec: 174759.38271372527, Loss: 3.837867259979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 70.56579673396004, Tokens per sec: 174869.81777098007, Loss: 3.8640477657318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 68.52731932869705, Tokens per sec: 169818.2462039145, Loss: 3.8135979175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 70.64797330112513, Tokens per sec: 175073.46035691135, Loss: 3.853105068206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 70.6177926733353, Tokens per sec: 174998.66943657753, Loss: 3.849566698074341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 69.51813684588369, Tokens per sec: 172273.6011590629, Loss: 3.8466262817382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 69.44273204760687, Tokens per sec: 172086.7397624087, Loss: 3.8436601161956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 70.60985815266498, Tokens per sec: 174979.00682029687, Loss: 3.8341307640075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 70.56170191690104, Tokens per sec: 174859.67036322653, Loss: 3.845560073852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 69.8193233264747, Tokens per sec: 173019.9744363972, Loss: 3.852722644805908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 69.74903473280334, Tokens per sec: 172845.7915583518, Loss: 3.851975202560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 70.61094959327886, Tokens per sec: 174981.71153037017, Loss: 3.840780258178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 69.50830178298249, Tokens per sec: 172249.2287897721, Loss: 3.8545758724212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 70.58498765667294, Tokens per sec: 174917.37499151728, Loss: 3.856403112411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 69.59955989374872, Tokens per sec: 172475.37644116164, Loss: 3.858766555786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 69.55210180869209, Tokens per sec: 172357.7700784516, Loss: 3.8515000343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 70.59835166553655, Tokens per sec: 174950.49247764843, Loss: 3.8191158771514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 69.45131565558196, Tokens per sec: 172108.01088853343, Loss: 3.8598062992095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 70.54948379212865, Tokens per sec: 174829.392504103, Loss: 3.868398427963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 69.49756432197434, Tokens per sec: 172222.62017856492, Loss: 3.8369879722595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 70.60708019691636, Tokens per sec: 174972.12273993468, Loss: 3.846816301345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 69.6201326620571, Tokens per sec: 172526.3580273073, Loss: 3.839768409729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 69.51385594992215, Tokens per sec: 172262.99262729057, Loss: 3.8409595489501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 70.5433389256301, Tokens per sec: 174814.16484800665, Loss: 3.8261992931365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 70.55307198240271, Tokens per sec: 174838.2844348739, Loss: 3.818662166595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 69.58887806986243, Tokens per sec: 172448.90570487137, Loss: 3.825167179107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 69.75166744989758, Tokens per sec: 172852.31572132907, Loss: 3.827608585357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 70.56317704434437, Tokens per sec: 174863.32589152982, Loss: 3.8224267959594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 70.49708156732363, Tokens per sec: 174699.53401845452, Loss: 3.825491428375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 69.51255315451878, Tokens per sec: 172259.76415676626, Loss: 3.8187708854675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 70.52173460407224, Tokens per sec: 174760.6269592738, Loss: 3.798099994659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 70.59439941282803, Tokens per sec: 174940.6983600601, Loss: 3.8149170875549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 68.48714750824196, Tokens per sec: 169718.69600753544, Loss: 3.8105087280273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 70.59634299316428, Tokens per sec: 174945.51476623642, Loss: 3.842785358428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 70.55458048471742, Tokens per sec: 174842.022669785, Loss: 3.8312244415283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 69.55023202008825, Tokens per sec: 172353.1365363171, Loss: 3.8103222846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 69.64714377978831, Tokens per sec: 172593.29455256546, Loss: 3.8261117935180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 70.60252839359919, Tokens per sec: 174960.84286989222, Loss: 3.814885377883911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 70.59845135090852, Tokens per sec: 174950.7395089814, Loss: 3.817495346069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 68.5890475057205, Tokens per sec: 169971.21542649865, Loss: 3.805943727493286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 70.60045916415422, Tokens per sec: 174955.71509138314, Loss: 3.812714099884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 70.57420941755962, Tokens per sec: 174890.6653276734, Loss: 3.802793502807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 69.14862272360422, Tokens per sec: 171357.90445871348, Loss: 3.8185319900512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 69.55114476787222, Tokens per sec: 172355.39842587322, Loss: 3.774855375289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 70.54516467852521, Tokens per sec: 174818.689264804, Loss: 3.8045005798339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 70.57495760246356, Tokens per sec: 174892.51941228478, Loss: 3.7950119972229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 69.47161826407914, Tokens per sec: 172158.32298890644, Loss: 3.8081789016723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 69.59949254394826, Tokens per sec: 172475.2095409369, Loss: 3.7945868968963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 70.54299583145017, Tokens per sec: 174813.31462283383, Loss: 3.8022961616516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 69.51750796732519, Tokens per sec: 172272.04272871825, Loss: 3.7923953533172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 70.57629801348696, Tokens per sec: 174895.84109845923, Loss: 3.764394760131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 69.62427378846102, Tokens per sec: 172536.62019471894, Loss: 3.790356159210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 69.42912407755465, Tokens per sec: 172053.0177135776, Loss: 3.7830350399017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 70.61774754587152, Tokens per sec: 174998.5576057511, Loss: 3.799055337905884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 69.55598248703711, Tokens per sec: 172367.3868268826, Loss: 3.7833266258239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 70.60358397959477, Tokens per sec: 174963.45872827168, Loss: 3.7916691303253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 69.74708178941977, Tokens per sec: 172840.9519495128, Loss: 3.7854621410369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 70.54634097752361, Tokens per sec: 174821.60426330208, Loss: 3.7899892330169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 69.62561352740157, Tokens per sec: 172539.94021539803, Loss: 3.8073625564575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 69.40710805647977, Tokens per sec: 171998.45958808914, Loss: 3.777606248855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 70.59476502437971, Tokens per sec: 174941.60438576064, Loss: 3.768867015838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 70.59214840740172, Tokens per sec: 174935.12012064504, Loss: 3.771031618118286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 68.57781817872439, Tokens per sec: 169943.3879171898, Loss: 3.773970603942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 70.61326224269301, Tokens per sec: 174987.4425303778, Loss: 3.7735695838928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 70.65366095133768, Tokens per sec: 175087.55498068445, Loss: 3.767451763153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 70.60033470337315, Tokens per sec: 174955.40666385714, Loss: 3.763669967651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 68.47555895683334, Tokens per sec: 169689.9783005606, Loss: 3.7561943531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 70.55493957235689, Tokens per sec: 174842.9125285124, Loss: 3.771334648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 70.52763265289275, Tokens per sec: 174775.24297397473, Loss: 3.7506494522094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 67.83402153487809, Tokens per sec: 168100.17789777438, Loss: 3.75077748298645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 70.5957601139658, Tokens per sec: 174944.0703273731, Loss: 3.748176097869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 70.64359430154447, Tokens per sec: 175062.60871356382, Loss: 3.775681257247925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 69.50734862998104, Tokens per sec: 172246.86677163604, Loss: 3.7663931846618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 69.570490769969, Tokens per sec: 172403.3399502071, Loss: 3.7577686309814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 70.59363108646833, Tokens per sec: 174938.79436270267, Loss: 3.756230592727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 70.62535756190216, Tokens per sec: 175017.41606378747, Loss: 3.7520318031311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 68.5386763220913, Tokens per sec: 169846.39008462158, Loss: 3.7701921463012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 70.60017924100411, Tokens per sec: 174955.0214109811, Loss: 3.761392116546631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 70.5471747919492, Tokens per sec: 174823.67054730095, Loss: 3.755807638168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 69.45153630628354, Tokens per sec: 172108.55768527862, Loss: 3.7312450408935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 69.6636441978121, Tokens per sec: 172634.18440609955, Loss: 3.758359909057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 70.58864637476731, Tokens per sec: 174926.44169799617, Loss: 3.7584831714630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 69.46880158833302, Tokens per sec: 172151.34295612507, Loss: 3.7479307651519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 70.55884842085469, Tokens per sec: 174852.59908568513, Loss: 3.756796360015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 69.57860916586044, Tokens per sec: 172423.45822954128, Loss: 3.7451813220977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 70.55054396921231, Tokens per sec: 174832.01973970447, Loss: 3.7313692569732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 69.46908710504817, Tokens per sec: 172152.0504979975, Loss: 3.7493720054626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 70.56296256235836, Tokens per sec: 174862.79438154135, Loss: 3.7514538764953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 69.59054987841166, Tokens per sec: 172453.04863062126, Loss: 3.7257070541381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 69.32539892991667, Tokens per sec: 171795.97537146247, Loss: 3.745544672012329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 70.60212687736826, Tokens per sec: 174959.84786844128, Loss: 3.7310562133789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 69.62425528409833, Tokens per sec: 172536.57433886974, Loss: 3.712556838989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 70.58819688684964, Tokens per sec: 174925.32781742103, Loss: 3.732401132583618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 69.7468004883808, Tokens per sec: 172840.2548545504, Loss: 3.7153491973876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 70.66115368581352, Tokens per sec: 175106.12280210838, Loss: 3.7291057109832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 69.62948827446226, Tokens per sec: 172549.54226545308, Loss: 3.7343952655792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 69.48372468670705, Tokens per sec: 172188.32403781093, Loss: 3.7394285202026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 70.57386224121086, Tokens per sec: 174889.80498643656, Loss: 3.714618444442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 70.63736184784287, Tokens per sec: 175047.16400672984, Loss: 3.7202184200286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 68.60376523847361, Tokens per sec: 170007.68758955298, Loss: 3.7073256969451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 70.54931987652843, Tokens per sec: 174828.98630318887, Loss: 3.7248175144195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 70.52036398960094, Tokens per sec: 174757.23042562825, Loss: 3.722233772277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 69.52548607076582, Tokens per sec: 172291.81334790477, Loss: 3.7176685333251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 69.40479379763735, Tokens per sec: 171992.72459974096, Loss: 3.727051019668579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 70.58322061165819, Tokens per sec: 174912.99605931467, Loss: 3.7272675037384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 70.56768164600363, Tokens per sec: 174874.4887906643, Loss: 3.7106785774230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 68.50446062455217, Tokens per sec: 169761.59981694934, Loss: 3.7250027656555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 70.57466982846977, Tokens per sec: 174891.8062766272, Loss: 3.738286018371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 70.54240533605152, Tokens per sec: 174811.8513101875, Loss: 3.7351157665252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 69.49936280988881, Tokens per sec: 172227.0770297368, Loss: 3.678900957107544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 69.67106533251108, Tokens per sec: 172652.57479538908, Loss: 3.7407288551330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 70.56252273069086, Tokens per sec: 174861.70443021783, Loss: 3.7140097618103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 70.56103018603764, Tokens per sec: 174858.00574014976, Loss: 3.7169206142425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 68.48160705834297, Tokens per sec: 169704.96616235483, Loss: 3.724804401397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 70.55912298546316, Tokens per sec: 174853.2794870307, Loss: 3.709878921508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 70.56548588828376, Tokens per sec: 174869.04746115176, Loss: 3.715708017349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 69.50603741889383, Tokens per sec: 172243.6174461201, Loss: 3.720360279083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 69.6608838894508, Tokens per sec: 172627.34405790718, Loss: 3.7095463275909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 70.63222092422235, Tokens per sec: 175034.42423167868, Loss: 3.7151527404785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 69.49456862094878, Tokens per sec: 172215.1965014199, Loss: 3.7111051082611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 70.57978227976788, Tokens per sec: 174904.47549412693, Loss: 3.6941020488739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 69.58253936950805, Tokens per sec: 172433.19770712772, Loss: 3.6915388107299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 69.13894021556706, Tokens per sec: 171333.91013718108, Loss: 3.7003326416015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 70.59088805540128, Tokens per sec: 174931.99682954868, Loss: 3.720935106277466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 70.62363835924235, Tokens per sec: 175013.15569021067, Loss: 3.7025437355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 69.6207633987656, Tokens per sec: 172527.92106235237, Loss: 3.698112726211548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 68.89288951132824, Tokens per sec: 170724.16938735466, Loss: 3.699671506881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 70.59975003544335, Tokens per sec: 174953.95779232064, Loss: 3.7033610343933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 69.65230666575084, Tokens per sec: 172606.0887527195, Loss: 3.711460590362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 69.46671986576884, Tokens per sec: 172146.18421829035, Loss: 3.6866118907928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 70.5137006628683, Tokens per sec: 174740.71796795874, Loss: 3.698519706726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 70.59392399450178, Tokens per sec: 174939.520221076, Loss: 3.6934702396392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 69.61108581658965, Tokens per sec: 172503.93894764667, Loss: 3.6962475776672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 69.51212656657283, Tokens per sec: 172258.70702484366, Loss: 3.7013039588928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 70.61649324015876, Tokens per sec: 174995.44929802165, Loss: 3.6894335746765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 70.62068266553776, Tokens per sec: 175005.83115561414, Loss: 3.69254469871521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 68.54337575866025, Tokens per sec: 169858.0358061251, Loss: 3.7005298137664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 70.56607111749237, Tokens per sec: 174870.49772359902, Loss: 3.7121896743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 70.56419524496623, Tokens per sec: 174865.8491048348, Loss: 3.6998813152313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 69.5410416648264, Tokens per sec: 172330.36182357703, Loss: 3.67203426361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 69.526989307913, Tokens per sec: 172295.53853515073, Loss: 3.6947414875030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 70.45298355598392, Tokens per sec: 174590.25428855777, Loss: 3.6704225540161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 70.60344555417925, Tokens per sec: 174963.11569484317, Loss: 3.6879117488861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 68.86122629465326, Tokens per sec: 170645.70444844285, Loss: 3.6760811805725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 70.56945098891124, Tokens per sec: 174878.87341729857, Loss: 3.6841073036193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 70.62652391189583, Tokens per sec: 175020.30640755585, Loss: 3.689802646636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 69.39883833201394, Tokens per sec: 171977.96629987747, Loss: 3.679446220397949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 69.52138259522182, Tokens per sec: 172281.6444834716, Loss: 3.6906259059906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 70.59012933965813, Tokens per sec: 174930.1166483577, Loss: 3.6696982383728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 69.47131050208942, Tokens per sec: 172157.5603207931, Loss: 3.6931252479553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 69.70746311720926, Tokens per sec: 172742.77251541815, Loss: 3.6669256687164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 70.61077150132034, Tokens per sec: 174981.2701988785, Loss: 3.6896164417266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 70.62939601915492, Tokens per sec: 175027.4238057326, Loss: 3.6627278327941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 69.47994252762415, Tokens per sec: 172178.95143096425, Loss: 3.6891653537750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 69.60835044638048, Tokens per sec: 172497.16039894242, Loss: 3.6751224994659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 70.57746156756451, Tokens per sec: 174898.72451363946, Loss: 3.6690492630004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 69.4101811230346, Tokens per sec: 172006.0749855383, Loss: 3.6909360885620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 70.60675660501299, Tokens per sec: 174971.32084355157, Loss: 3.6533396244049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 69.72584147702204, Tokens per sec: 172788.3161155777, Loss: 3.6896004676818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 70.60322140258306, Tokens per sec: 174962.56022249546, Loss: 3.6560611724853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 69.3880258873066, Tokens per sec: 171951.17187080355, Loss: 3.7163631916046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 70.59927021447153, Tokens per sec: 174952.76874309583, Loss: 3.692112445831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 69.5771625991816, Tokens per sec: 172419.8734779586, Loss: 3.6843550205230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 69.53278565560676, Tokens per sec: 172309.90252325629, Loss: 3.65867018699646 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/model.pt +[rank0]:[2024-08-24 21:19:07,766] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006405303000065032, 'preprocessing_with_comm': 0.0012950890013598837, 'state_converting': 0.9920383799999399, : 1.001122124000176}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400, took 4.29s + eval ppl=30.15152359008789, eval loss=3.406235456466675 +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 66.49200868860581, Tokens per sec: 164774.52222979255, Loss: 3.680385112762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 70.54797206711663, Tokens per sec: 174825.64628299294, Loss: 3.6859235763549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 68.06799000672301, Tokens per sec: 168679.97754476094, Loss: 3.655238628387451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 70.58140289075992, Tokens per sec: 174908.49154669026, Loss: 3.6720974445343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 70.55583912683151, Tokens per sec: 174845.1417235946, Loss: 3.6665658950805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 68.39596610325978, Tokens per sec: 169492.73844152867, Loss: 3.6799685955047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 70.56864364336829, Tokens per sec: 174876.87272610667, Loss: 3.6565864086151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 70.56851858929187, Tokens per sec: 174876.56282832945, Loss: 3.6538960933685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 69.38763238225768, Tokens per sec: 171950.1967219442, Loss: 3.69547438621521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 69.22213685107779, Tokens per sec: 171540.08056484265, Loss: 3.6597328186035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 70.52245597532212, Tokens per sec: 174762.41459669656, Loss: 3.6538383960723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 70.6522497437516, Tokens per sec: 175084.05785282873, Loss: 3.6844980716705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 68.56683653294351, Tokens per sec: 169916.17418921666, Loss: 3.6798434257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 70.57571808066552, Tokens per sec: 174894.4039610428, Loss: 3.68729567527771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 70.52480608353608, Tokens per sec: 174768.2384237366, Loss: 3.6586015224456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 69.46837333135606, Tokens per sec: 172150.28168815977, Loss: 3.6903958320617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 69.5714597252871, Tokens per sec: 172405.74112822462, Loss: 3.6627774238586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 70.60977417102102, Tokens per sec: 174978.7987045318, Loss: 3.691706418991089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 69.5367284934804, Tokens per sec: 172319.67330984658, Loss: 3.6448099613189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 70.58945473335822, Tokens per sec: 174928.4448996325, Loss: 3.6904234886169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 69.50919284270105, Tokens per sec: 172251.43693391304, Loss: 3.641116142272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 70.58141183921579, Tokens per sec: 174908.51372194968, Loss: 3.668766498565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 69.76264967759835, Tokens per sec: 172879.5308913638, Loss: 3.6464591026306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 69.59605711887833, Tokens per sec: 172466.69617917028, Loss: 3.6677045822143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 70.60966303077929, Tokens per sec: 174978.52328676966, Loss: 3.6417956352233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 69.39570808236766, Tokens per sec: 171970.20919642854, Loss: 3.635606527328491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 70.60576937850017, Tokens per sec: 174968.87438750078, Loss: 3.6589674949645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 69.74700560866805, Tokens per sec: 172840.7631652181, Loss: 3.622581958770752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 69.5051627551135, Tokens per sec: 172241.4499329203, Loss: 3.6267337799072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 70.6113554571632, Tokens per sec: 174982.71730578516, Loss: 3.6292898654937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 70.56814662772217, Tokens per sec: 174875.6410665848, Loss: 3.669858694076538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 69.56621265608439, Tokens per sec: 172392.73831272792, Loss: 3.6691627502441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 69.15565019012361, Tokens per sec: 171375.3192948879, Loss: 3.659156322479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 70.59830573447391, Tokens per sec: 174950.37865541546, Loss: 3.655545949935913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 70.6231288681915, Tokens per sec: 175011.89311526157, Loss: 3.6760058403015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 68.49916532797509, Tokens per sec: 169748.4774887063, Loss: 3.634709596633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 70.63413506581125, Tokens per sec: 175039.1676853961, Loss: 3.63329815864563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 70.61879380014288, Tokens per sec: 175001.15033908995, Loss: 3.6499292850494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 69.568429897279, Tokens per sec: 172398.23288065745, Loss: 3.639646053314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 69.743147658123, Tokens per sec: 172831.20273877907, Loss: 3.6630494594573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 70.60769254844142, Tokens per sec: 174973.6402144699, Loss: 3.6517014503479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 70.59153447126988, Tokens per sec: 174933.59871927966, Loss: 3.668663740158081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 68.50212973063071, Tokens per sec: 169755.82360504285, Loss: 3.6683902740478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 70.59549369846059, Tokens per sec: 174943.41012040313, Loss: 3.658668279647827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 70.58619484340122, Tokens per sec: 174920.36653321254, Loss: 3.6090385913848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 69.47406074810633, Tokens per sec: 172164.3757333876, Loss: 3.6418144702911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 69.6611832602011, Tokens per sec: 172628.08593160482, Loss: 3.657088279724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 70.6100638134345, Tokens per sec: 174979.51647033912, Loss: 3.635735511779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 69.48792345643596, Tokens per sec: 172198.72905173185, Loss: 3.6311886310577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 70.64876191093853, Tokens per sec: 175075.41461890147, Loss: 3.6689956188201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 69.57991372519405, Tokens per sec: 172426.691071279, Loss: 3.6427853107452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 70.64741656835007, Tokens per sec: 175072.08071176553, Loss: 3.6310853958129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 69.44406660340563, Tokens per sec: 172090.0469386915, Loss: 3.6357717514038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 69.68974063028102, Tokens per sec: 172698.85424051687, Loss: 3.659834623336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 70.6033452521259, Tokens per sec: 174962.86713530577, Loss: 3.6641228199005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 69.53321320947845, Tokens per sec: 172310.96204884927, Loss: 3.657824993133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 70.59511797900625, Tokens per sec: 174942.4790462063, Loss: 3.637890338897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 69.58082176542592, Tokens per sec: 172428.9412950025, Loss: 3.6325275897979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 70.5722558330949, Tokens per sec: 174885.82413016475, Loss: 3.6214399337768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 69.48361926071428, Tokens per sec: 172188.06278058718, Loss: 3.6568260192871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 69.71653844764208, Tokens per sec: 172765.26218396085, Loss: 3.641014337539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 70.58897332061244, Tokens per sec: 174927.2519058165, Loss: 3.6315393447875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 69.3833830376458, Tokens per sec: 171939.666377892, Loss: 3.6270315647125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 70.58194795068118, Tokens per sec: 174909.84226521847, Loss: 3.6270461082458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 69.5989432835456, Tokens per sec: 172473.84841315306, Loss: 3.66219425201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 69.48823432061673, Tokens per sec: 172199.49940741627, Loss: 3.623962640762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 70.61612433297947, Tokens per sec: 174994.53510539292, Loss: 3.6201109886169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 70.61439970278613, Tokens per sec: 174990.26128179012, Loss: 3.6280527114868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 68.84291538819365, Tokens per sec: 170600.328005123, Loss: 3.6377038955688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 69.78812684619453, Tokens per sec: 172942.66612168693, Loss: 3.640937089920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 70.59447778541205, Tokens per sec: 174940.8925759568, Loss: 3.621453285217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 70.58590698717404, Tokens per sec: 174919.6531937716, Loss: 3.6448140144348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 68.48459738583753, Tokens per sec: 169712.37652329812, Loss: 3.6112451553344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 70.54943883655605, Tokens per sec: 174829.28109924178, Loss: 3.6150155067443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 70.5597720451831, Tokens per sec: 174854.88792851655, Loss: 3.6334714889526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 69.43911238122938, Tokens per sec: 172077.7698303861, Loss: 3.6311442852020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 70.55920282193127, Tokens per sec: 174853.4773305934, Loss: 3.6450355052948 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 70.53756759182819, Tokens per sec: 174799.86284708063, Loss: 3.633481979370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 70.65137002686545, Tokens per sec: 175081.87781747608, Loss: 3.6275384426116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 68.07325722731416, Tokens per sec: 168693.03029761807, Loss: 3.6520211696624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 70.64250841190425, Tokens per sec: 175059.91775941473, Loss: 3.623793363571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 70.61345840161165, Tokens per sec: 174987.92863378685, Loss: 3.619168281555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 69.25696977250728, Tokens per sec: 171626.400381308, Loss: 3.6246213912963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 69.60263753604232, Tokens per sec: 172483.00317779544, Loss: 3.6266305446624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 70.61067796832505, Tokens per sec: 174981.03841381267, Loss: 3.634089469909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 69.43546391507103, Tokens per sec: 172068.72852933398, Loss: 3.613579273223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 69.6607220514055, Tokens per sec: 172626.943005403, Loss: 3.6159212589263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 70.58548228401185, Tokens per sec: 174918.60073255078, Loss: 3.631824016571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 70.59397073876971, Tokens per sec: 174939.63605852128, Loss: 3.642808675765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 69.52744017394915, Tokens per sec: 172296.65583085528, Loss: 3.6442248821258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 69.54619392929011, Tokens per sec: 172343.1297024869, Loss: 3.6182384490966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 70.56857143269902, Tokens per sec: 174876.6937801135, Loss: 3.62553334236145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 69.50305198201156, Tokens per sec: 172236.21920465282, Loss: 3.618220806121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 70.61598607502798, Tokens per sec: 174994.19248695872, Loss: 3.5946643352508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 69.65099426825891, Tokens per sec: 172602.83648716199, Loss: 3.6084821224212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 69.38884127568683, Tokens per sec: 171953.19249303223, Loss: 3.627741575241089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 70.57527730912739, Tokens per sec: 174893.31168061635, Loss: 3.625331401824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 69.48847659323256, Tokens per sec: 172200.09978564686, Loss: 3.6161081790924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 70.5817870552207, Tokens per sec: 174909.44354854326, Loss: 3.6197140216827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 69.46184185219022, Tokens per sec: 172134.09596328498, Loss: 3.608295202255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 70.59672301824962, Tokens per sec: 174946.45651026108, Loss: 3.602508783340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 69.70446169215181, Tokens per sec: 172735.3346534915, Loss: 3.6015782356262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 69.45205483502477, Tokens per sec: 172109.84265662005, Loss: 3.6278135776519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 70.61871690894642, Tokens per sec: 175000.95979423483, Loss: 3.627638816833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 70.58405268208286, Tokens per sec: 174915.0580214871, Loss: 3.6116561889648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 69.55670262219223, Tokens per sec: 172369.17140112643, Loss: 3.6106905937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 69.77207430928718, Tokens per sec: 172902.88616689763, Loss: 3.6338043212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 70.51090033025642, Tokens per sec: 174733.77843526436, Loss: 3.6104934215545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 70.47507672980427, Tokens per sec: 174645.00360704862, Loss: 3.615079641342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 66.3809815061684, Tokens per sec: 164499.3846410588, Loss: 3.6104674339294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 70.59975676675718, Tokens per sec: 174953.9744732578, Loss: 3.6152584552764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 70.54754423875501, Tokens per sec: 174824.5860771837, Loss: 3.574331283569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 68.61531906244183, Tokens per sec: 170036.31923810497, Loss: 3.6155858039855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 70.55442083288149, Tokens per sec: 174841.62703494847, Loss: 3.6177568435668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 70.55964241471484, Tokens per sec: 174854.5666899362, Loss: 3.5965077877044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 69.53752850203429, Tokens per sec: 172321.65581917134, Loss: 3.610945224761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 66.8940974264949, Tokens per sec: 165770.94241601272, Loss: 3.627354145050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 70.5807709344162, Tokens per sec: 174906.92548925476, Loss: 3.609254837036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 70.5707751711688, Tokens per sec: 174882.1548868035, Loss: 3.5967655181884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 69.4540741224699, Tokens per sec: 172114.84667335197, Loss: 3.602682113647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 69.60319991255139, Tokens per sec: 172484.3968087358, Loss: 3.6292457580566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 70.60302301259877, Tokens per sec: 174962.06859025997, Loss: 3.6308035850524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 69.44448214543459, Tokens per sec: 172091.07669761498, Loss: 3.610365867614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 70.59227023207131, Tokens per sec: 174935.42201559636, Loss: 3.5916366577148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 69.62555649222072, Tokens per sec: 172539.79887593698, Loss: 3.6111526489257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 70.57035889023287, Tokens per sec: 174881.12329678715, Loss: 3.5836522579193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 69.45226709931697, Tokens per sec: 172110.36867091898, Loss: 3.5965962409973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 69.55574404748681, Tokens per sec: 172366.7959474107, Loss: 3.6267969608306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 70.60319510556018, Tokens per sec: 174962.4950555759, Loss: 3.6097965240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 69.56292573025861, Tokens per sec: 172384.59294844687, Loss: 3.600773572921753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 70.59403854914196, Tokens per sec: 174939.8041000936, Loss: 3.600058078765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 69.658881053279, Tokens per sec: 172622.38080924284, Loss: 3.57088041305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 69.4566857036651, Tokens per sec: 172121.31845924308, Loss: 3.6169188022613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 70.55526433490127, Tokens per sec: 174843.7173258729, Loss: 3.6098644733428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 69.6473855400568, Tokens per sec: 172593.89366114294, Loss: 3.5987930297851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 69.61118698021619, Tokens per sec: 172504.18964225732, Loss: 3.60115385055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 70.60661855322282, Tokens per sec: 174970.97873600785, Loss: 3.6185498237609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 70.6089744625232, Tokens per sec: 174976.81693877903, Loss: 3.5969533920288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 69.65826266749391, Tokens per sec: 172620.84838114658, Loss: 3.589298963546753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 69.505793426955, Tokens per sec: 172243.01280721775, Loss: 3.6030750274658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 70.64082359675054, Tokens per sec: 175055.74260186614, Loss: 3.593569278717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 70.63003211614613, Tokens per sec: 175029.00012414868, Loss: 3.5880067348480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 68.51034304021412, Tokens per sec: 169776.17709096128, Loss: 3.607905626296997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 70.55870372003059, Tokens per sec: 174852.2405011029, Loss: 3.6156883239746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 70.59928415701954, Tokens per sec: 174952.8032942657, Loss: 3.603943109512329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 70.621062980969, Tokens per sec: 175006.77361914795, Loss: 3.5930261611938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 68.49251084963434, Tokens per sec: 169731.9869583266, Loss: 3.6017816066741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 70.52943553185, Tokens per sec: 174779.7107066343, Loss: 3.5715279579162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 70.56750080942308, Tokens per sec: 174874.04065769687, Loss: 3.5953166484832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 68.09102185782929, Tokens per sec: 168737.05300897016, Loss: 3.587254047393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 70.59050501542954, Tokens per sec: 174931.04761430336, Loss: 3.577481985092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 70.6289714332043, Tokens per sec: 175026.37163497496, Loss: 3.6015753746032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 69.49473528881748, Tokens per sec: 172215.6095227585, Loss: 3.5763309001922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 69.72316506493672, Tokens per sec: 172781.68367159914, Loss: 3.5839481353759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 70.5467032713822, Tokens per sec: 174822.50206739362, Loss: 3.576525926589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 70.59686467441654, Tokens per sec: 174946.80754984744, Loss: 3.6074562072753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 69.8477891758498, Tokens per sec: 173090.51594692087, Loss: 3.5979254245758057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 69.60110636683858, Tokens per sec: 172479.20877183636, Loss: 3.586439609527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 70.61036056824575, Tokens per sec: 174980.25186145166, Loss: 3.5752432346343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 69.53388825282282, Tokens per sec: 172312.63488061877, Loss: 3.5700833797454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 69.68177994325322, Tokens per sec: 172679.1267811199, Loss: 3.583024501800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 70.60639535133116, Tokens per sec: 174970.42561713257, Loss: 3.58660626411438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 69.52066770997541, Tokens per sec: 172279.8729190799, Loss: 3.5962939262390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 70.63491317957875, Tokens per sec: 175041.09593702824, Loss: 3.6073009967803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 69.55413238234543, Tokens per sec: 172362.80206365077, Loss: 3.5779318809509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 70.5887275391921, Tokens per sec: 174926.64283238183, Loss: 3.6000397205352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 69.45664560165686, Tokens per sec: 172121.21908204904, Loss: 3.6091415882110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 70.57464562242107, Tokens per sec: 174891.74629137202, Loss: 3.5629770755767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 69.5502812937289, Tokens per sec: 172353.2586418266, Loss: 3.588047981262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 69.49363329441519, Tokens per sec: 172212.8786592349, Loss: 3.5939993858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 70.59037066958273, Tokens per sec: 174930.71469049554, Loss: 3.609055995941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 69.64421612816994, Tokens per sec: 172586.03950934773, Loss: 3.5642051696777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 70.58811928110751, Tokens per sec: 174925.13550184306, Loss: 3.564007043838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 69.78595736613063, Tokens per sec: 172937.28991110067, Loss: 3.573174476623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 70.61803821446841, Tokens per sec: 174999.27791455394, Loss: 3.566051483154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 69.60282261170859, Tokens per sec: 172483.46181568425, Loss: 3.5733723640441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 69.53542710552944, Tokens per sec: 172316.4483271444, Loss: 3.5615978240966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 70.61176489499834, Tokens per sec: 174983.73193784393, Loss: 3.564650535583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 70.5925612181003, Tokens per sec: 174936.14311103095, Loss: 3.583205461502075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 68.6094233102838, Tokens per sec: 170021.7089147867, Loss: 3.571272373199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 70.58092971752846, Tokens per sec: 174907.31897129837, Loss: 3.5807690620422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 70.54638599496168, Tokens per sec: 174821.7158214727, Loss: 3.5720722675323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 69.5177285367714, Tokens per sec: 172272.5893241037, Loss: 3.594545364379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 69.59937934612778, Tokens per sec: 172474.92902426797, Loss: 3.5826332569122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 70.62145663010284, Tokens per sec: 175007.74912506566, Loss: 3.6018571853637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 70.57665809368379, Tokens per sec: 174896.73341685307, Loss: 3.587627649307251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 68.57989530915428, Tokens per sec: 169948.5352752099, Loss: 3.5697290897369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 70.5982074051317, Tokens per sec: 174950.1349844736, Loss: 3.5812504291534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 70.55413054670944, Tokens per sec: 174840.90767383648, Loss: 3.574244499206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 69.46349771033384, Tokens per sec: 172138.1993621727, Loss: 3.574878692626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 69.56648862660558, Tokens per sec: 172393.42219808008, Loss: 3.5667333602905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 70.60713688347836, Tokens per sec: 174972.26321547985, Loss: 3.584354877471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 70.56601835307228, Tokens per sec: 174870.36696755356, Loss: 3.5644476413726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 69.36088673204401, Tokens per sec: 171883.9180544381, Loss: 3.583005905151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 69.48080388224398, Tokens per sec: 172181.08596259824, Loss: 3.597978353500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 70.55435229942412, Tokens per sec: 174841.45720149152, Loss: 3.569149971008301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 69.50996961265736, Tokens per sec: 172253.36185543303, Loss: 3.5948047637939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 70.60444903303438, Tokens per sec: 174965.60242598853, Loss: 3.5886197090148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 69.51833762936998, Tokens per sec: 172274.09872266007, Loss: 3.5688843727111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 69.53063353749455, Tokens per sec: 172304.56933749886, Loss: 3.5764760971069336 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/model.pt +[rank0]:[2024-08-24 21:44:27,853] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063475890001427615, 'preprocessing_with_comm': 0.0011817310005426407, 'state_converting': 0.9907033420004154, : 0.9996117179998691}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600, took 4.30s + eval ppl=26.09284782409668, eval loss=3.2616612911224365 +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 65.32162771669216, Tokens per sec: 161874.18925328317, Loss: 3.565812587738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 69.12423975903324, Tokens per sec: 171297.48078650134, Loss: 3.585853099822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 69.52249862519304, Tokens per sec: 172284.4101286811, Loss: 3.5779707431793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 70.56880865523657, Tokens per sec: 174877.2816436938, Loss: 3.5767858028411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 69.2972365232919, Tokens per sec: 171726.18582550104, Loss: 3.5610997676849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 70.5717410815109, Tokens per sec: 174884.54851903502, Loss: 3.5549886226654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 68.951765088321, Tokens per sec: 170870.06955282294, Loss: 3.5771536827087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 70.58948587656928, Tokens per sec: 174928.52207594024, Loss: 3.580589771270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 70.55091205989896, Tokens per sec: 174832.93190897448, Loss: 3.579137086868286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 68.5656039254243, Tokens per sec: 169913.11965200113, Loss: 3.575138568878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 70.59323196811324, Tokens per sec: 174937.80530345222, Loss: 3.5725722312927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 70.57659315798011, Tokens per sec: 174896.57249902608, Loss: 3.5683138370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 68.51300630545082, Tokens per sec: 169782.77695560068, Loss: 3.573479652404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 70.57108160464344, Tokens per sec: 174882.9142627101, Loss: 3.577430486679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 70.59684915133285, Tokens per sec: 174946.769081936, Loss: 3.5609092712402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 70.59714995594024, Tokens per sec: 174947.51450888952, Loss: 3.5651001930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 68.74312756291572, Tokens per sec: 170353.04278154788, Loss: 3.5811495780944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 70.5635692772084, Tokens per sec: 174864.29788777485, Loss: 3.569563150405884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 70.62220012719062, Tokens per sec: 175009.5915927521, Loss: 3.5700769424438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 68.52217770006217, Tokens per sec: 169805.50468176013, Loss: 3.572430372238159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 70.60358281209353, Tokens per sec: 174963.45583507497, Loss: 3.594179630279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 70.54144176998422, Tokens per sec: 174809.4634873272, Loss: 3.5760741233825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 69.49694087859662, Tokens per sec: 172221.07521719698, Loss: 3.575888156890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 69.4259667057463, Tokens per sec: 172045.19339842335, Loss: 3.5622920989990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 70.59048655859695, Tokens per sec: 174931.00187623902, Loss: 3.5531466007232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 70.52657197112498, Tokens per sec: 174772.61448771044, Loss: 3.574744939804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 69.4843052147667, Tokens per sec: 172189.76265029318, Loss: 3.5712625980377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 69.57039773288135, Tokens per sec: 172403.10939405503, Loss: 3.5535454750061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 70.57574545797085, Tokens per sec: 174894.47180502128, Loss: 3.549238681793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 69.54263022488469, Tokens per sec: 172334.2984503958, Loss: 3.548917531967163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 70.59972421173003, Tokens per sec: 174953.89379831433, Loss: 3.5488553047180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 69.57336598270005, Tokens per sec: 172410.46504408555, Loss: 3.552384376525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 70.53117328437273, Tokens per sec: 174784.01704881486, Loss: 3.5678279399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 69.7256475647504, Tokens per sec: 172787.83557960737, Loss: 3.5709314346313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 69.56253275244221, Tokens per sec: 172383.61910612773, Loss: 3.5588796138763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 70.61934994061588, Tokens per sec: 175002.52851644592, Loss: 3.5755698680877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 69.5598117157232, Tokens per sec: 172376.87607739112, Loss: 3.582479476928711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 70.64613578442794, Tokens per sec: 175068.90678811652, Loss: 3.5805318355560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 69.61385019807095, Tokens per sec: 172510.78938947906, Loss: 3.578111171722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 69.46652119415347, Tokens per sec: 172145.691888142, Loss: 3.5591230392456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 70.58177790906552, Tokens per sec: 174909.4208833632, Loss: 3.569155216217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 70.55331670699694, Tokens per sec: 174838.89088937704, Loss: 3.5906691551208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 68.61560079018321, Tokens per sec: 170037.017390483, Loss: 3.553964138031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 70.62593273331379, Tokens per sec: 175018.84140190587, Loss: 3.5736773014068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 70.60571393704996, Tokens per sec: 174968.73699747978, Loss: 3.569722890853882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 70.60695557153358, Tokens per sec: 174971.81390450755, Loss: 3.559689521789551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 68.4414245387866, Tokens per sec: 169605.3894524242, Loss: 3.550999879837036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 70.5703164760377, Tokens per sec: 174881.0181897392, Loss: 3.5729916095733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 70.6162008794572, Tokens per sec: 174994.724795997, Loss: 3.5684046745300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 68.5498889366725, Tokens per sec: 169874.17617872526, Loss: 3.580770492553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 70.5996839369265, Tokens per sec: 174953.79399291452, Loss: 3.5539493560791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 70.59171455569611, Tokens per sec: 174934.04498832583, Loss: 3.555567502975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 70.63288445604125, Tokens per sec: 175036.06853661998, Loss: 3.5514025688171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 68.50761247718219, Tokens per sec: 169769.41045497177, Loss: 3.5550739765167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 70.59637017935266, Tokens per sec: 174945.58213660604, Loss: 3.581902265548706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 70.6408272632184, Tokens per sec: 175055.75168777737, Loss: 3.5416977405548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 68.53761437697084, Tokens per sec: 169843.7584676301, Loss: 3.5700876712799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 70.60658133083427, Tokens per sec: 174970.88649482868, Loss: 3.5347917079925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 70.60926835484271, Tokens per sec: 174977.54523632166, Loss: 3.585970878601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 69.47093656231911, Tokens per sec: 172156.6336568494, Loss: 3.5651872158050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 69.68064616533547, Tokens per sec: 172676.3171545438, Loss: 3.5499393939971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 70.4806385251967, Tokens per sec: 174658.78634871336, Loss: 3.5780832767486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 69.45739619371223, Tokens per sec: 172123.07913184678, Loss: 3.5546774864196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 70.59413228280195, Tokens per sec: 174940.0363824287, Loss: 3.5659353733062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 69.61739932710003, Tokens per sec: 172519.58452218198, Loss: 3.529914140701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 70.59218633934228, Tokens per sec: 174935.21412017226, Loss: 3.5507376194000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 69.49716059685488, Tokens per sec: 172221.61970324494, Loss: 3.558537006378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 70.58332907151596, Tokens per sec: 174913.26483479008, Loss: 3.547065258026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 69.72687276467767, Tokens per sec: 172790.871759994, Loss: 3.5329267978668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 69.48712678677164, Tokens per sec: 172196.75481654322, Loss: 3.569331645965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 70.63108760130007, Tokens per sec: 175031.6157326314, Loss: 3.5651073455810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 69.58573840930292, Tokens per sec: 172441.1252801427, Loss: 3.558903217315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 70.58763556201009, Tokens per sec: 174923.93679263358, Loss: 3.5474777221679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 69.4799493571008, Tokens per sec: 172178.9683551597, Loss: 3.5597341060638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 70.54317564953683, Tokens per sec: 174813.7602318612, Loss: 3.5732340812683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 69.66128302963779, Tokens per sec: 172628.33317125944, Loss: 3.5401604175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 69.49329093965301, Tokens per sec: 172212.03026642074, Loss: 3.5754897594451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 70.6073208611883, Tokens per sec: 174972.71913251193, Loss: 3.546344041824341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 70.55885598127877, Tokens per sec: 174852.61782124883, Loss: 3.5423457622528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 68.4412169484713, Tokens per sec: 169604.8750207549, Loss: 3.5483858585357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 70.6270396043036, Tokens per sec: 175021.58435015052, Loss: 3.5518884658813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 70.59283196949083, Tokens per sec: 174936.81406280238, Loss: 3.5509560108184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 70.63536969919502, Tokens per sec: 175042.22724292718, Loss: 3.542140007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 69.40937107501573, Tokens per sec: 172004.06759731332, Loss: 3.544715642929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 70.58815496880538, Tokens per sec: 174925.2239398897, Loss: 3.555255889892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 70.57044052056766, Tokens per sec: 174881.32558574906, Loss: 3.5461182594299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 68.58723332305723, Tokens per sec: 169966.71968201039, Loss: 3.538705587387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 70.5698820366528, Tokens per sec: 174879.94160108594, Loss: 3.5470874309539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 70.601622422921, Tokens per sec: 174958.59777475073, Loss: 3.561490058898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 69.47334045426459, Tokens per sec: 172162.59076590085, Loss: 3.5652592182159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 69.56789017683255, Tokens per sec: 172396.89539393602, Loss: 3.5570363998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 70.57854918423891, Tokens per sec: 174901.41974756945, Loss: 3.536306858062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 70.58220234182018, Tokens per sec: 174910.4726744844, Loss: 3.570559024810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 68.04738308251527, Tokens per sec: 168628.91131653404, Loss: 3.5479016304016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 70.63281629776232, Tokens per sec: 175035.89963289653, Loss: 3.512148857116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 70.56614076477263, Tokens per sec: 174870.67031723176, Loss: 3.549591541290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 69.46249814004165, Tokens per sec: 172135.7223168769, Loss: 3.550360679626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 69.58287717196262, Tokens per sec: 172434.03481882214, Loss: 3.546182632446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 70.53591862078846, Tokens per sec: 174795.776515195, Loss: 3.573906898498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 70.55721080094078, Tokens per sec: 174848.54088313974, Loss: 3.543303966522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 69.76557207111996, Tokens per sec: 172886.77290443858, Loss: 3.5291943550109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 69.70291343087658, Tokens per sec: 172731.49789149646, Loss: 3.5496277809143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 70.57749129834184, Tokens per sec: 174898.79818978076, Loss: 3.561312675476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 69.48609220961929, Tokens per sec: 172194.19102039162, Loss: 3.553074359893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 70.61344782513827, Tokens per sec: 174987.90242412072, Loss: 3.5576415061950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 69.56166170214105, Tokens per sec: 172381.46054752736, Loss: 3.570141315460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 69.53404003185837, Tokens per sec: 172313.0110057887, Loss: 3.5145230293273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 70.6167854337332, Tokens per sec: 174996.1733858869, Loss: 3.5424416065216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 69.70758746019875, Tokens per sec: 172743.08065104362, Loss: 3.5492656230926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 69.51858160898104, Tokens per sec: 172274.70333101283, Loss: 3.533473491668701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 70.59322463563907, Tokens per sec: 174937.78713277352, Loss: 3.548675298690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 70.58583969118885, Tokens per sec: 174919.48642690707, Loss: 3.5352842807769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 69.62100454562264, Tokens per sec: 172528.5186508286, Loss: 3.55464506149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 69.49013163284738, Tokens per sec: 172204.20115613003, Loss: 3.558427572250366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 70.6379173100587, Tokens per sec: 175048.54050328984, Loss: 3.5294888019561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 70.6462074280625, Tokens per sec: 175069.08432893516, Loss: 3.5437567234039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 68.66736337126787, Tokens per sec: 170165.29076852984, Loss: 3.5209996700286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 70.59762572837624, Tokens per sec: 174948.6935253966, Loss: 3.543109178543091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 70.61807595758287, Tokens per sec: 174999.37144614937, Loss: 3.5260066986083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 70.63201800837327, Tokens per sec: 175033.92138385162, Loss: 3.523345708847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 69.29889915564577, Tokens per sec: 171730.30601162807, Loss: 3.5436501502990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 70.57317753187716, Tokens per sec: 174888.1082012807, Loss: 3.5499305725097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 70.59006473683434, Tokens per sec: 174929.9565554438, Loss: 3.54270601272583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 68.58161964294372, Tokens per sec: 169952.80836429118, Loss: 3.5591278076171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 70.54798666628133, Tokens per sec: 174825.6824613313, Loss: 3.521151304244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 70.59120606165432, Tokens per sec: 174932.784884075, Loss: 3.5218725204467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 69.53526845268749, Tokens per sec: 172316.055167925, Loss: 3.5500996112823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 69.6182987429489, Tokens per sec: 172521.8133737343, Loss: 3.547717571258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 70.52816406742294, Tokens per sec: 174776.55987772066, Loss: 3.535306692123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 70.60447032187395, Tokens per sec: 174965.65518207813, Loss: 3.535468578338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 68.72454938771658, Tokens per sec: 170307.00401684947, Loss: 3.5306754112243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 70.59516309638704, Tokens per sec: 174942.59085204604, Loss: 3.552039623260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 70.46975815649783, Tokens per sec: 174631.82359650603, Loss: 3.5288286209106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 69.4485548752336, Tokens per sec: 172101.16937070515, Loss: 3.5347177982330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 69.65947704083437, Tokens per sec: 172623.85773205847, Loss: 3.5399091243743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 70.59412961894627, Tokens per sec: 174940.02978110086, Loss: 3.5559935569763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 69.47258751200904, Tokens per sec: 172160.72489204817, Loss: 3.5378928184509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 70.57471151411112, Tokens per sec: 174891.90957823853, Loss: 3.5267953872680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 69.56235888611738, Tokens per sec: 172383.18824622186, Loss: 3.5447797775268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 70.56001006431525, Tokens per sec: 174855.47776614598, Loss: 3.529141902923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 69.53398873557656, Tokens per sec: 172312.8838879516, Loss: 3.530775547027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 70.60421112827957, Tokens per sec: 174965.01287179874, Loss: 3.5279932022094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 69.693336960281, Tokens per sec: 172707.76634242528, Loss: 3.535144567489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 69.5465766185349, Tokens per sec: 172344.07804859217, Loss: 3.5209760665893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 70.62178324877169, Tokens per sec: 175008.55852210705, Loss: 3.5444698333740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 69.58156968487876, Tokens per sec: 172430.79472179676, Loss: 3.538193702697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 69.52093769585461, Tokens per sec: 172280.54197382994, Loss: 3.5272483825683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 70.63165091778875, Tokens per sec: 175033.0116929448, Loss: 3.5362255573272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 70.52035834426255, Tokens per sec: 174757.21643585784, Loss: 3.5630273818969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 69.60245884828535, Tokens per sec: 172482.56036984958, Loss: 3.532918930053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 69.46555483155657, Tokens per sec: 172143.29713517323, Loss: 3.5306715965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 70.57638259289992, Tokens per sec: 174896.0506955617, Loss: 3.518582344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 70.57396978798526, Tokens per sec: 174890.0714991908, Loss: 3.5269687175750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 68.56670953970252, Tokens per sec: 169915.85948597602, Loss: 3.520782470703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 70.56701096759383, Tokens per sec: 174872.8267756835, Loss: 3.5303072929382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 70.63413470771033, Tokens per sec: 175039.16679798256, Loss: 3.5535168647766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 69.49180835038446, Tokens per sec: 172208.35624689257, Loss: 3.5192790031433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 69.58967576640846, Tokens per sec: 172450.88248478595, Loss: 3.539088010787964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 70.60482033224885, Tokens per sec: 174966.52254634397, Loss: 3.518996238708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 70.62270479093527, Tokens per sec: 175010.8422051047, Loss: 3.525501251220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 68.59227341447219, Tokens per sec: 169979.20958374854, Loss: 3.5244734287261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 70.54949564681831, Tokens per sec: 174829.42188132985, Loss: 3.516637086868286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 70.59571432748655, Tokens per sec: 174943.95686343373, Loss: 3.554564952850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 69.37926730693785, Tokens per sec: 171929.4671438125, Loss: 3.5212008953094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 69.47517993622526, Tokens per sec: 172167.14920483506, Loss: 3.5475780963897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 70.57204717529642, Tokens per sec: 174885.30705315457, Loss: 3.5250513553619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 70.63386428019504, Tokens per sec: 175038.49664880958, Loss: 3.529165744781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 68.66791722822538, Tokens per sec: 170166.66328708292, Loss: 3.5371580123901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 70.57074417616604, Tokens per sec: 174882.07807777228, Loss: 3.5218167304992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 70.58998628933892, Tokens per sec: 174929.7621539084, Loss: 3.519075870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 69.48683137255507, Tokens per sec: 172196.02274757205, Loss: 3.5280587673187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 69.55496740550865, Tokens per sec: 172364.87134303464, Loss: 3.5496857166290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 70.63506078472334, Tokens per sec: 175041.4617188366, Loss: 3.531729221343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 69.44785179004325, Tokens per sec: 172099.42704815223, Loss: 3.5106301307678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 70.62216355722505, Tokens per sec: 175009.5009683489, Loss: 3.528944492340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 69.63304842220228, Tokens per sec: 172558.36470373548, Loss: 3.545968532562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 69.52279181390907, Tokens per sec: 172285.13668261684, Loss: 3.5269789695739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 70.56231382011316, Tokens per sec: 174861.18672679286, Loss: 3.544867992401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 69.53980379655415, Tokens per sec: 172327.29424963598, Loss: 3.5290985107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 70.61645539513394, Tokens per sec: 174995.3555138812, Loss: 3.540426254272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 69.50397813658344, Tokens per sec: 172238.5143177063, Loss: 3.529829502105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 70.62399641672766, Tokens per sec: 175014.04299610254, Loss: 3.5214123725891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 69.49953207351375, Tokens per sec: 172227.49648364534, Loss: 3.5371408462524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 69.48413080910959, Tokens per sec: 172189.33045386247, Loss: 3.5323119163513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 70.62724992529974, Tokens per sec: 175022.10554874767, Loss: 3.5173726081848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 70.49146422329056, Tokens per sec: 174685.61362113908, Loss: 3.533946990966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 69.64762721037444, Tokens per sec: 172594.4925468122, Loss: 3.533360242843628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 69.46893800675025, Tokens per sec: 172151.6810159907, Loss: 3.535343885421753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 70.61956539663623, Tokens per sec: 175003.06244019882, Loss: 3.508563756942749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 70.55417828196292, Tokens per sec: 174841.02596705305, Loss: 3.515542984008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 68.5095700202162, Tokens per sec: 169774.26146225122, Loss: 3.53082275390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 70.58558584787669, Tokens per sec: 174918.85737521635, Loss: 3.5286147594451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 70.61180588047498, Tokens per sec: 174983.83350436998, Loss: 3.5479326248168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 68.60324692481757, Tokens per sec: 170006.40315121633, Loss: 3.532099962234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 70.54633983070633, Tokens per sec: 174821.60142136255, Loss: 3.5382490158081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 70.5234175666856, Tokens per sec: 174764.79752602332, Loss: 3.5255303382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 69.50891159559387, Tokens per sec: 172250.7399725995, Loss: 3.5262129306793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 69.54809084488173, Tokens per sec: 172347.83046828542, Loss: 3.5170607566833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 70.60655897671224, Tokens per sec: 174970.83109885172, Loss: 3.5292162895202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 70.5578621950903, Tokens per sec: 174850.1551095992, Loss: 3.5145602226257324 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/model.pt +[rank0]:[2024-08-24 22:09:47,373] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00634098000045924, 'preprocessing_with_comm': 0.001300169000387541, 'state_converting': 0.9707384900011675, : 0.979767085000276}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800, took 4.36s + eval ppl=26.391780853271484, eval loss=3.27305269241333 +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 64.80042793509139, Tokens per sec: 160582.5987795808, Loss: 3.5162458419799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 69.11486939080402, Tokens per sec: 171274.259981798, Loss: 3.5338618755340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 70.63669706328584, Tokens per sec: 175045.51659736544, Loss: 3.538292407989502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 68.65593886311797, Tokens per sec: 170136.9795788209, Loss: 3.5140252113342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 69.50501744378668, Tokens per sec: 172241.08983544508, Loss: 3.566688060760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 70.544508001182, Tokens per sec: 174817.06194600853, Loss: 3.5143990516662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 70.57943203770793, Tokens per sec: 174903.60755572002, Loss: 3.5339882373809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 69.45076208075962, Tokens per sec: 172106.63906914237, Loss: 3.5346755981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 69.51768207505582, Tokens per sec: 172272.47418685426, Loss: 3.50443172454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 70.52959632023469, Tokens per sec: 174780.10915796386, Loss: 3.544829845428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 68.62301158247021, Tokens per sec: 170055.38215013637, Loss: 3.53165340423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 70.60078661483959, Tokens per sec: 174956.52655025313, Loss: 3.5295519828796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 69.58020172883043, Tokens per sec: 172427.4047760162, Loss: 3.5251142978668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 68.74010681582023, Tokens per sec: 170345.5570374825, Loss: 3.5248641967773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 70.56531175947562, Tokens per sec: 174868.6159507833, Loss: 3.5348691940307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 70.60176128368123, Tokens per sec: 174958.94188701137, Loss: 3.518557548522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 68.51209517466815, Tokens per sec: 169780.5190731519, Loss: 3.5260846614837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 69.6784724039184, Tokens per sec: 172670.9303342927, Loss: 3.5260961055755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 70.61945026930056, Tokens per sec: 175002.77714197873, Loss: 3.5300655364990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 69.54092821862281, Tokens per sec: 172330.0806913874, Loss: 3.5370097160339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 68.6929211829733, Tokens per sec: 170228.62584136127, Loss: 3.545257568359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 70.64761263426792, Tokens per sec: 175072.56658470843, Loss: 3.5201632976531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 70.59501023599323, Tokens per sec: 174942.2120471512, Loss: 3.540147542953491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 68.53663199520908, Tokens per sec: 169841.32401740624, Loss: 3.5330536365509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 69.938111320086, Tokens per sec: 173314.3441701437, Loss: 3.503901958465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 70.64051069476899, Tokens per sec: 175054.9671962869, Loss: 3.5345451831817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 70.59446679102177, Tokens per sec: 174940.86533064654, Loss: 3.5514307022094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 68.05772738073526, Tokens per sec: 168654.5456270417, Loss: 3.522395133972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 70.5809806788365, Tokens per sec: 174907.4452590335, Loss: 3.53149151802063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 70.57918285084763, Tokens per sec: 174902.99004322998, Loss: 3.52742600440979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 68.436680888635, Tokens per sec: 169593.6341647923, Loss: 3.524219036102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 69.60002223274051, Tokens per sec: 172476.52216811414, Loss: 3.5136890411376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 70.54863984548355, Tokens per sec: 174827.30111134803, Loss: 3.5418925285339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 68.66122203425101, Tokens per sec: 170150.07185887796, Loss: 3.5248169898986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 69.67855622984094, Tokens per sec: 172671.13806416292, Loss: 3.5279438495635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 70.61918406118173, Tokens per sec: 175002.117448935, Loss: 3.5287671089172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 70.57060361249462, Tokens per sec: 174881.72974551015, Loss: 3.5243170261383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 68.68135233164091, Tokens per sec: 170199.95695334542, Loss: 3.5121185779571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 69.58879678285737, Tokens per sec: 172448.70426671833, Loss: 3.5280745029449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 70.61804331972878, Tokens per sec: 174999.29056595155, Loss: 3.524355888366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 68.72676015674777, Tokens per sec: 170312.48254604512, Loss: 3.5286550521850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 69.60976232358738, Tokens per sec: 172500.65918619224, Loss: 3.5127978324890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 70.64670886217552, Tokens per sec: 175070.32693790473, Loss: 3.5364315509796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 69.54968523627741, Tokens per sec: 172351.78154580065, Loss: 3.532315254211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 69.7120230264423, Tokens per sec: 172754.0724728131, Loss: 3.5176615715026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 69.56585560056752, Tokens per sec: 172391.8534898242, Loss: 3.5155673027038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 70.59394267903821, Tokens per sec: 174939.56652341556, Loss: 3.538290500640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 69.41652600215087, Tokens per sec: 172021.7982949345, Loss: 3.5201504230499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 70.66566101538996, Tokens per sec: 175117.29246132215, Loss: 3.509290933609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 69.53635244265328, Tokens per sec: 172318.7414144716, Loss: 3.5019726753234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 68.44662289109974, Tokens per sec: 169618.27154210192, Loss: 3.544461488723755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 70.59930769276643, Tokens per sec: 174952.86161843914, Loss: 3.5245914459228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 70.56979086009353, Tokens per sec: 174879.71565552818, Loss: 3.5380611419677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 68.5978137968572, Tokens per sec: 169992.9392616221, Loss: 3.5155692100524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 69.72214806368252, Tokens per sec: 172779.1634304593, Loss: 3.5059194564819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 70.58474921490688, Tokens per sec: 174916.78410655446, Loss: 3.5106420516967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 69.71136454846342, Tokens per sec: 172752.4406918441, Loss: 3.5386321544647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 68.60561143777001, Tokens per sec: 170012.2626747851, Loss: 3.530672311782837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 70.52515165942054, Tokens per sec: 174769.0947988466, Loss: 3.52081036567688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 70.54852314434133, Tokens per sec: 174827.01191306193, Loss: 3.510941505432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 68.18360368972789, Tokens per sec: 168966.4809871456, Loss: 3.495396375656128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 70.61684080779794, Tokens per sec: 174996.3106089193, Loss: 3.5218586921691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 70.58170991524516, Tokens per sec: 174909.2523871862, Loss: 3.5125961303710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 69.51864069633034, Tokens per sec: 172274.84975597338, Loss: 3.5101020336151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 68.68943814889043, Tokens per sec: 170219.99449921603, Loss: 3.5363917350769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 70.55621030045279, Tokens per sec: 174846.06153271627, Loss: 3.503994941711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 70.57210393732586, Tokens per sec: 174885.44771571638, Loss: 3.5166051387786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 68.53078644628381, Tokens per sec: 169826.83810322903, Loss: 3.5076682567596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 70.59247279723971, Tokens per sec: 174935.92399439804, Loss: 3.5103049278259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 70.59903854858499, Tokens per sec: 174952.19464950883, Loss: 3.500185489654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 68.65429150203742, Tokens per sec: 170132.89723659153, Loss: 3.5016274452209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 69.57256428991614, Tokens per sec: 172408.47836105325, Loss: 3.51393985748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 70.5325311451813, Tokens per sec: 174787.3819774792, Loss: 3.5073509216308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 70.58752558313182, Tokens per sec: 174923.66425285806, Loss: 3.5096793174743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 68.81557832705482, Tokens per sec: 170532.58375619512, Loss: 3.530416250228882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 69.29942340813157, Tokens per sec: 171731.60516703906, Loss: 3.5359718799591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 70.57816464104718, Tokens per sec: 174900.46680717941, Loss: 3.5188164710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 68.70497564387621, Tokens per sec: 170258.4981233872, Loss: 3.533796787261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 69.64280630664837, Tokens per sec: 172582.5458163126, Loss: 3.5455362796783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 70.58210952436686, Tokens per sec: 174910.24266261037, Loss: 3.519458770751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 68.57843702834047, Tokens per sec: 169944.92149471038, Loss: 3.5365686416625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 70.62426337713104, Tokens per sec: 175014.70455339027, Loss: 3.5360097885131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 69.59177463442069, Tokens per sec: 172456.08371092964, Loss: 3.5412256717681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 70.59062546968342, Tokens per sec: 174931.34611321363, Loss: 3.539470672607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 68.95271367937728, Tokens per sec: 170872.42026595635, Loss: 3.51948618888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 70.54224237445048, Tokens per sec: 174811.44747338849, Loss: 3.5203845500946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 69.61548111781016, Tokens per sec: 172514.83098825352, Loss: 3.51729679107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 69.45842833097892, Tokens per sec: 172125.63688169298, Loss: 3.529223918914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 70.5979241672837, Tokens per sec: 174949.43308988502, Loss: 3.5214929580688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 70.62838662064115, Tokens per sec: 175024.92240502106, Loss: 3.5192103385925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 68.52566400512787, Tokens per sec: 169814.14412976115, Loss: 3.513958215713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 69.99299849692639, Tokens per sec: 173450.3606406759, Loss: 3.517821788787842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 70.57727897792951, Tokens per sec: 174898.27203640997, Loss: 3.5125691890716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 69.6732374673003, Tokens per sec: 172657.95758467718, Loss: 3.535736322402954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 68.68090570770434, Tokens per sec: 170198.8501700309, Loss: 3.549076557159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 70.5291548656141, Tokens per sec: 174779.01518478378, Loss: 3.5435612201690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 70.58432824275177, Tokens per sec: 174915.74089118018, Loss: 3.5284507274627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 66.96401159426011, Tokens per sec: 165944.1974254163, Loss: 3.5362696647644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 70.53384116253588, Tokens per sec: 174790.6283447942, Loss: 3.5224246978759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 70.62983136118241, Tokens per sec: 175028.5026312336, Loss: 3.5321273803710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 68.72018745024586, Tokens per sec: 170296.19465528996, Loss: 3.5212244987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 69.69007009922935, Tokens per sec: 172699.67070086487, Loss: 3.5003719329833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 70.57681153321076, Tokens per sec: 174897.11365690365, Loss: 3.5249505043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 70.49058093186898, Tokens per sec: 174683.42472769378, Loss: 3.5086824893951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 68.65204830646184, Tokens per sec: 170127.3383508468, Loss: 3.5187430381774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 69.19353175039335, Tokens per sec: 171469.19397423463, Loss: 3.536067485809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 70.59369485808583, Tokens per sec: 174938.95239579584, Loss: 3.5347847938537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 69.48573079610755, Tokens per sec: 172193.29539789652, Loss: 3.505861282348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 69.56016161715992, Tokens per sec: 172377.74317169611, Loss: 3.529419422149658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 70.59768801982844, Tokens per sec: 174948.8478904771, Loss: 3.5328922271728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 69.53457753768879, Tokens per sec: 172314.34300444764, Loss: 3.5283398628234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 69.6844247094635, Tokens per sec: 172685.68080313405, Loss: 3.5297765731811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 69.66045300428297, Tokens per sec: 172626.27627699537, Loss: 3.5051634311676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 70.63607042110758, Tokens per sec: 175043.96370901735, Loss: 3.512932300567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 68.61325698292487, Tokens per sec: 170031.20917790514, Loss: 3.5047686100006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 69.60207791368885, Tokens per sec: 172481.616371956, Loss: 3.536350727081299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 70.57219686017181, Tokens per sec: 174885.67798876495, Loss: 3.510643243789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 68.66468896499924, Tokens per sec: 170158.66329518598, Loss: 3.5217316150665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 70.61646107471456, Tokens per sec: 174995.36958850763, Loss: 3.5237815380096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 69.43151120910265, Tokens per sec: 172058.93328851784, Loss: 3.5410637855529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 69.7031178893325, Tokens per sec: 172732.00456207318, Loss: 3.499735116958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 69.81294692211048, Tokens per sec: 173004.17300396375, Loss: 3.5406556129455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 70.64280949260872, Tokens per sec: 175060.6638705672, Loss: 3.5116958618164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 69.55320459829323, Tokens per sec: 172360.5029125656, Loss: 3.5287063121795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 68.59348273748417, Tokens per sec: 169982.20641939013, Loss: 3.499375343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 70.59211681005209, Tokens per sec: 174935.04181893187, Loss: 3.5149309635162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 70.6076510027375, Tokens per sec: 174973.5372596389, Loss: 3.515517234802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 68.15035885302571, Tokens per sec: 168884.0966195765, Loss: 3.522679328918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 70.57473463833068, Tokens per sec: 174891.96688260196, Loss: 3.5299758911132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 70.5977258918133, Tokens per sec: 174948.9417414275, Loss: 3.501840829849243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 68.67533102384631, Tokens per sec: 170185.0354893286, Loss: 3.5347867012023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 69.68307995483327, Tokens per sec: 172682.3483530234, Loss: 3.503740072250366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 70.56101169759083, Tokens per sec: 174857.95992374187, Loss: 3.5368709564208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 70.59604222365212, Tokens per sec: 174944.76942625275, Loss: 3.534701347351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 67.66307911859266, Tokens per sec: 167676.5637593574, Loss: 3.527289390563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 70.57414966053749, Tokens per sec: 174890.5172431898, Loss: 3.549696922302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 70.60624923133051, Tokens per sec: 174970.06351567453, Loss: 3.5120532512664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 68.62761526377231, Tokens per sec: 170066.79057954033, Loss: 3.5222792625427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 69.59182660143193, Tokens per sec: 172456.21249090816, Loss: 3.5155670642852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 70.56571460647676, Tokens per sec: 174869.6142500294, Loss: 3.4990856647491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 70.56772566546562, Tokens per sec: 174874.59787574023, Loss: 3.5178093910217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 68.03892229788609, Tokens per sec: 168607.94456019058, Loss: 3.506990671157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 70.6043641415253, Tokens per sec: 174965.39205547745, Loss: 3.499390125274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 70.54817531820959, Tokens per sec: 174826.14996159126, Loss: 3.4935100078582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 68.66846873391376, Tokens per sec: 170168.02997893209, Loss: 3.5282700061798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 69.6590795802316, Tokens per sec: 172622.87278090094, Loss: 3.5159454345703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 70.56296995437451, Tokens per sec: 174862.81269977166, Loss: 3.5191681385040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 69.44237115788798, Tokens per sec: 172085.84543792997, Loss: 3.510125160217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 70.5502549371115, Tokens per sec: 174831.3034863191, Loss: 3.518876791000366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 69.58619725382871, Tokens per sec: 172442.26234742353, Loss: 3.4654412269592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 69.49925562127638, Tokens per sec: 172226.8114045474, Loss: 3.519712448120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 69.63385612541836, Tokens per sec: 172560.36628128082, Loss: 3.523179292678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 69.75279759435644, Tokens per sec: 172855.11634379378, Loss: 3.510204553604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 70.63892927125909, Tokens per sec: 175051.04825462116, Loss: 3.5426437854766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 68.81748718174039, Tokens per sec: 170537.31410838367, Loss: 3.505354642868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 70.58031812181532, Tokens per sec: 174905.80336974838, Loss: 3.5303471088409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 69.57416650535735, Tokens per sec: 172412.44882741518, Loss: 3.5252621173858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 69.47089648943566, Tokens per sec: 172156.5343518298, Loss: 3.5138673782348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 69.9437927251815, Tokens per sec: 173328.42331782883, Loss: 3.52551531791687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 70.59152544495015, Tokens per sec: 174933.57635106504, Loss: 3.5267317295074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 69.40477657433928, Tokens per sec: 171992.68191851102, Loss: 3.516201972961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 68.78098431539253, Tokens per sec: 170446.85598445102, Loss: 3.5172083377838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 70.6071488328644, Tokens per sec: 174972.29282737477, Loss: 3.501258611679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 70.586146453116, Tokens per sec: 174920.24661675515, Loss: 3.5140366554260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 67.65061478078984, Tokens per sec: 167645.67575722185, Loss: 3.5358664989471436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 70.48774713989486, Tokens per sec: 174676.40227901397, Loss: 3.5123651027679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 70.59327314858065, Tokens per sec: 174937.9073531869, Loss: 3.5129220485687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 68.46658621994901, Tokens per sec: 169667.7428701326, Loss: 3.4856584072113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 70.60447339137532, Tokens per sec: 174965.66278864068, Loss: 3.5403106212615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 70.65222874667745, Tokens per sec: 175084.005819766, Loss: 3.516066551208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 70.60519795340937, Tokens per sec: 174967.45833317813, Loss: 3.52927827835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 68.02443203431089, Tokens per sec: 168572.03609081692, Loss: 3.5168657302856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 70.56381193031895, Tokens per sec: 174864.89920891327, Loss: 3.5125479698181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 70.57517431177345, Tokens per sec: 174893.05644182715, Loss: 3.5087876319885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 68.76373876642201, Tokens per sec: 170404.119614348, Loss: 3.5104658603668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 69.48428806143002, Tokens per sec: 172189.72014243528, Loss: 3.506789445877075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 70.57130852478727, Tokens per sec: 174883.47659582383, Loss: 3.511784553527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 69.46723707173479, Tokens per sec: 172147.4659116489, Loss: 3.5128910541534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 69.70702791107418, Tokens per sec: 172741.6940266735, Loss: 3.505648612976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 69.6735672386787, Tokens per sec: 172658.77479448015, Loss: 3.500514030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 70.59436369058145, Tokens per sec: 174940.60983639798, Loss: 3.511840343475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 68.76929971113934, Tokens per sec: 170417.90024794618, Loss: 3.5312345027923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 69.56036087046971, Tokens per sec: 172378.23694334735, Loss: 3.5117692947387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 70.6172522213316, Tokens per sec: 174997.33013697667, Loss: 3.4946539402008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 68.52263485539821, Tokens per sec: 169806.63756304263, Loss: 3.537461280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 70.6201481980067, Tokens per sec: 175004.50668619573, Loss: 3.5144150257110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 69.73719095686339, Tokens per sec: 172816.44137687297, Loss: 3.4996941089630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 70.60417230778745, Tokens per sec: 174964.9166703428, Loss: 3.5194990634918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 69.77754836297582, Tokens per sec: 172916.45147495487, Loss: 3.5078728199005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 69.45279300978052, Tokens per sec: 172111.67193498145, Loss: 3.5132687091827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 70.57089361093352, Tokens per sec: 174882.4483935877, Loss: 3.4997169971466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 68.63504331511841, Tokens per sec: 170085.19810904335, Loss: 3.528658151626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 70.65859659051037, Tokens per sec: 175099.78603825986, Loss: 3.5263383388519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 69.64166426958026, Tokens per sec: 172579.71572265206, Loss: 3.5242080688476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 68.73821280905207, Tokens per sec: 170340.86348006895, Loss: 3.516240358352661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 70.58092721466554, Tokens per sec: 174907.31276892833, Loss: 3.5167932510375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 70.5852807094541, Tokens per sec: 174918.10120859143, Loss: 3.5149612426757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 68.59873231979407, Tokens per sec: 169995.2154626433, Loss: 3.4931750297546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 69.65852121287894, Tokens per sec: 172621.48908509177, Loss: 3.5145604610443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 70.59953742104024, Tokens per sec: 174953.43091040832, Loss: 3.5295658111572266 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/optimizer.pt +[rank0]:[2024-08-24 22:35:11,595] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006363355998473708, 'preprocessing_with_comm': 0.0014622210001107305, 'state_converting': 0.9682152320019668, : 0.9774286960018799}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000, took 4.24s + eval ppl=25.523460388183594, eval loss=3.239598035812378 +------------------------------------------------------------------ +iteration: 1001 , TFLOPS: 64.27366063342997, Tokens per sec: 159277.21137785513, Loss: 3.50211763381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1002 , TFLOPS: 68.34477824567092, Tokens per sec: 169365.8892916155, Loss: 3.5029213428497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1003 , TFLOPS: 70.53697491306309, Tokens per sec: 174798.39412381174, Loss: 3.513390064239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1004 , TFLOPS: 70.53884284446964, Tokens per sec: 174803.0230636068, Loss: 3.519376516342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1005 , TFLOPS: 68.4655726713972, Tokens per sec: 169665.23118516966, Loss: 3.5023345947265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1006 , TFLOPS: 70.01793548010345, Tokens per sec: 173512.15723202412, Loss: 3.5085368156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1007 , TFLOPS: 70.57881824745542, Tokens per sec: 174902.08651585976, Loss: 3.4947400093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1008 , TFLOPS: 70.58031078262734, Tokens per sec: 174905.78518243207, Loss: 3.503169536590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1009 , TFLOPS: 67.65831498732699, Tokens per sec: 167664.7577172687, Loss: 3.5126137733459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1010 , TFLOPS: 70.55466120413661, Tokens per sec: 174842.2227013977, Loss: 3.5184214115142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1011 , TFLOPS: 70.55537787084204, Tokens per sec: 174843.9986804411, Loss: 3.4998068809509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1012 , TFLOPS: 68.71283869900091, Tokens per sec: 170277.98364017322, Loss: 3.5095372200012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1013 , TFLOPS: 69.54711951645267, Tokens per sec: 172345.42340943753, Loss: 3.526327610015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1014 , TFLOPS: 70.59541730980929, Tokens per sec: 174943.2208209103, Loss: 3.534637928009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1015 , TFLOPS: 69.71955552919198, Tokens per sec: 172772.73884440048, Loss: 3.514427423477173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1016 , TFLOPS: 68.47456288148905, Tokens per sec: 169687.5099161307, Loss: 3.4947917461395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1017 , TFLOPS: 70.58492889773277, Tokens per sec: 174917.2293803907, Loss: 3.498927593231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1018 , TFLOPS: 70.57465858628117, Tokens per sec: 174891.77841724546, Loss: 3.5116937160491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1019 , TFLOPS: 68.6618902705136, Tokens per sec: 170151.72782194894, Loss: 3.519881010055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1020 , TFLOPS: 69.464242325862, Tokens per sec: 172140.04460147768, Loss: 3.5229766368865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1021 , TFLOPS: 70.50324631811407, Tokens per sec: 174714.8109500166, Loss: 3.516493320465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1022 , TFLOPS: 68.59372501422831, Tokens per sec: 169982.80680785113, Loss: 3.525451898574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1023 , TFLOPS: 70.59010424770759, Tokens per sec: 174930.05446774018, Loss: 3.5071322917938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1024 , TFLOPS: 68.99713723560875, Tokens per sec: 170982.50673195662, Loss: 3.5023984909057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1025 , TFLOPS: 70.52585689469163, Tokens per sec: 174770.84244953643, Loss: 3.5089221000671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1026 , TFLOPS: 68.82971064425459, Tokens per sec: 170567.60519501942, Loss: 3.503302574157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1027 , TFLOPS: 70.5688191243071, Tokens per sec: 174877.30758720386, Loss: 3.522813081741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1028 , TFLOPS: 69.52321362111348, Tokens per sec: 172286.1819673353, Loss: 3.51902437210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1029 , TFLOPS: 68.24706210255128, Tokens per sec: 169123.73792464324, Loss: 3.5110557079315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1030 , TFLOPS: 70.50288192617009, Tokens per sec: 174713.90794663836, Loss: 3.5367684364318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1031 , TFLOPS: 66.02773278338364, Tokens per sec: 163623.99539243896, Loss: 3.520771026611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1032 , TFLOPS: 68.64828291339936, Tokens per sec: 170118.0072920457, Loss: 3.5290119647979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1033 , TFLOPS: 70.58080076206573, Tokens per sec: 174906.9994054561, Loss: 3.505086660385132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1034 , TFLOPS: 70.56142998421858, Tokens per sec: 174858.9964840835, Loss: 3.50140380859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1035 , TFLOPS: 69.57266175774504, Tokens per sec: 172408.7198970702, Loss: 3.5226798057556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1036 , TFLOPS: 68.88477416320676, Tokens per sec: 170704.05866073025, Loss: 3.5151216983795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1037 , TFLOPS: 70.60797366115436, Tokens per sec: 174974.33684273966, Loss: 3.4942994117736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1038 , TFLOPS: 69.54465753234878, Tokens per sec: 172339.3223416182, Loss: 3.508476734161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1039 , TFLOPS: 68.64815935673042, Tokens per sec: 170117.7011050092, Loss: 3.5227243900299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1040 , TFLOPS: 70.57954834243708, Tokens per sec: 174903.8957716509, Loss: 3.500058650970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1041 , TFLOPS: 70.5433386624334, Tokens per sec: 174814.16419577622, Loss: 3.5232954025268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1042 , TFLOPS: 68.66693526665262, Tokens per sec: 170164.22987813363, Loss: 3.5288021564483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1043 , TFLOPS: 70.57448638778256, Tokens per sec: 174891.35169039664, Loss: 3.478847026824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1044 , TFLOPS: 70.56766657781725, Tokens per sec: 174874.45145003853, Loss: 3.477879524230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1045 , TFLOPS: 70.58893334040823, Tokens per sec: 174927.15283046628, Loss: 3.530040979385376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1046 , TFLOPS: 67.62015896081826, Tokens per sec: 167570.20288034674, Loss: 3.511979818344116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1047 , TFLOPS: 70.61904458552407, Tokens per sec: 175001.7718128908, Loss: 3.514559745788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1048 , TFLOPS: 70.5853265294399, Tokens per sec: 174918.21475556373, Loss: 3.5177149772644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1049 , TFLOPS: 68.01448142158198, Tokens per sec: 168547.3773763242, Loss: 3.510009765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1050 , TFLOPS: 69.61980560262715, Tokens per sec: 172525.54753801133, Loss: 3.5030786991119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1051 , TFLOPS: 70.59783491399156, Tokens per sec: 174949.21191039504, Loss: 3.515129804611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1052 , TFLOPS: 68.6117097265996, Tokens per sec: 170027.3749062868, Loss: 3.5217199325561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1053 , TFLOPS: 69.64006728882272, Tokens per sec: 172575.75822841295, Loss: 3.512850046157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1054 , TFLOPS: 70.59224401852117, Tokens per sec: 174935.35705553147, Loss: 3.511826992034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1055 , TFLOPS: 70.58840546131334, Tokens per sec: 174925.84468791846, Loss: 3.5204503536224365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1056 , TFLOPS: 68.53046769666842, Tokens per sec: 169826.04820656896, Loss: 3.5158913135528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1057 , TFLOPS: 69.56144060702653, Tokens per sec: 172380.91264947792, Loss: 3.503150224685669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1058 , TFLOPS: 70.62501110690722, Tokens per sec: 175016.55751014483, Loss: 3.5172410011291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1059 , TFLOPS: 68.65622772658186, Tokens per sec: 170137.69541430537, Loss: 3.512099027633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1060 , TFLOPS: 70.61682577511515, Tokens per sec: 174996.27335627534, Loss: 3.4733662605285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1061 , TFLOPS: 69.65384812520347, Tokens per sec: 172609.90865904884, Loss: 3.5068063735961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1062 , TFLOPS: 69.65936501342692, Tokens per sec: 172623.580115802, Loss: 3.51108455657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1063 , TFLOPS: 69.71549249301164, Tokens per sec: 172762.67019316522, Loss: 3.522035598754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1064 , TFLOPS: 70.61966415287762, Tokens per sec: 175003.3071690439, Loss: 3.5320181846618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1065 , TFLOPS: 69.60996366416086, Tokens per sec: 172501.1581303128, Loss: 3.536428213119507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1066 , TFLOPS: 68.66819861318935, Tokens per sec: 170167.36059002075, Loss: 3.5091311931610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1067 , TFLOPS: 70.54483140815356, Tokens per sec: 174817.86338411033, Loss: 3.513726234436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1068 , TFLOPS: 69.7038035680893, Tokens per sec: 172733.70374956628, Loss: 3.5471549034118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1069 , TFLOPS: 68.64267489032933, Tokens per sec: 170104.10999310343, Loss: 3.5085926055908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1070 , TFLOPS: 70.57474237150569, Tokens per sec: 174891.9860462615, Loss: 3.5091753005981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1071 , TFLOPS: 70.55354746176313, Tokens per sec: 174839.46272510738, Loss: 3.502882957458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1072 , TFLOPS: 68.5157436648566, Tokens per sec: 169789.56043375324, Loss: 3.515831708908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1073 , TFLOPS: 69.93928892391271, Tokens per sec: 173317.26240215014, Loss: 3.4971001148223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1074 , TFLOPS: 70.51312942164702, Tokens per sec: 174739.302369285, Loss: 3.509401559829712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1075 , TFLOPS: 69.69192756209632, Tokens per sec: 172704.27369846564, Loss: 3.50651216506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1076 , TFLOPS: 68.67884344389093, Tokens per sec: 170193.73965312418, Loss: 3.5147368907928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1077 , TFLOPS: 70.57006397661387, Tokens per sec: 174880.3924683518, Loss: 3.495600938796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1078 , TFLOPS: 70.59116045050705, Tokens per sec: 174932.67185462758, Loss: 3.4965786933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1079 , TFLOPS: 67.60982519947603, Tokens per sec: 167544.59468138294, Loss: 3.527540683746338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1080 , TFLOPS: 70.57900574711819, Tokens per sec: 174902.55116067888, Loss: 3.4859671592712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1081 , TFLOPS: 70.61602467304184, Tokens per sec: 174994.288137089, Loss: 3.5203819274902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1082 , TFLOPS: 69.79805881877452, Tokens per sec: 172967.27864383726, Loss: 3.4913065433502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1083 , TFLOPS: 68.39712332602112, Tokens per sec: 169495.60616700986, Loss: 3.483367443084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1084 , TFLOPS: 70.56528879194673, Tokens per sec: 174868.55903471663, Loss: 3.5097756385803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1085 , TFLOPS: 70.59231095022552, Tokens per sec: 174935.522919668, Loss: 3.504500389099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1086 , TFLOPS: 68.49847788674164, Tokens per sec: 169746.7739336019, Loss: 3.500823497772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1087 , TFLOPS: 69.56141369113276, Tokens per sec: 172380.84594892815, Loss: 3.5020458698272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1088 , TFLOPS: 70.54104733942378, Tokens per sec: 174808.48604494837, Loss: 3.5091257095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1089 , TFLOPS: 68.63761102541862, Tokens per sec: 170091.56117802396, Loss: 3.539452075958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1090 , TFLOPS: 69.54772294832, Tokens per sec: 172346.91878007833, Loss: 3.5092244148254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1091 , TFLOPS: 70.52019264774756, Tokens per sec: 174756.8058216407, Loss: 3.503349781036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1092 , TFLOPS: 69.50989086439947, Tokens per sec: 172253.16670857515, Loss: 3.534360885620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1093 , TFLOPS: 69.72771790057845, Tokens per sec: 172792.9660998556, Loss: 3.517510175704956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1094 , TFLOPS: 69.63667075933144, Tokens per sec: 172567.3412541749, Loss: 3.4867775440216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1095 , TFLOPS: 70.55583640945795, Tokens per sec: 174845.1349896436, Loss: 3.5367071628570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1096 , TFLOPS: 68.63674808000657, Tokens per sec: 170089.4227042316, Loss: 3.5202128887176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1097 , TFLOPS: 70.51358075269427, Tokens per sec: 174740.42081733828, Loss: 3.5178263187408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1098 , TFLOPS: 69.52344009710006, Tokens per sec: 172286.74319977843, Loss: 3.515908718109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1099 , TFLOPS: 68.6855100227959, Tokens per sec: 170210.26017003498, Loss: 3.506880283355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1100 , TFLOPS: 70.65892662706413, Tokens per sec: 175100.60390519662, Loss: 3.519430637359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1101 , TFLOPS: 70.61105851506453, Tokens per sec: 174981.98145055378, Loss: 3.4928274154663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1102 , TFLOPS: 68.77555247362997, Tokens per sec: 170433.39528219658, Loss: 3.5120697021484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1103 , TFLOPS: 69.3211684044967, Tokens per sec: 171785.49166344092, Loss: 3.529432773590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1104 , TFLOPS: 70.60785543593788, Tokens per sec: 174974.04386762963, Loss: 3.4886631965637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1105 , TFLOPS: 69.56460664810407, Tokens per sec: 172388.7584480363, Loss: 3.499685287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1106 , TFLOPS: 68.58370217441066, Tokens per sec: 169957.96910667606, Loss: 3.5065441131591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1107 , TFLOPS: 70.57858201140483, Tokens per sec: 174901.50109690282, Loss: 3.4860122203826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1108 , TFLOPS: 69.70818061022976, Tokens per sec: 172744.55054216133, Loss: 3.481391191482544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1109 , TFLOPS: 68.63794090786394, Tokens per sec: 170092.37866306296, Loss: 3.524446725845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1110 , TFLOPS: 70.52322185131167, Tokens per sec: 174764.3125217669, Loss: 3.505707263946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1111 , TFLOPS: 70.5871194849595, Tokens per sec: 174922.65789685168, Loss: 3.5455658435821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1112 , TFLOPS: 70.5933496078755, Tokens per sec: 174938.09682774218, Loss: 3.483812093734741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1113 , TFLOPS: 68.62032606159934, Tokens per sec: 170048.7271335838, Loss: 3.5241525173187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1114 , TFLOPS: 70.6062598903963, Tokens per sec: 174970.08993001372, Loss: 3.501697063446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1115 , TFLOPS: 70.59726783468865, Tokens per sec: 174947.8066254135, Loss: 3.5270748138427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1116 , TFLOPS: 67.8459567983301, Tokens per sec: 168129.75479538628, Loss: 3.496164083480835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1117 , TFLOPS: 70.594434683309, Tokens per sec: 174940.78576419735, Loss: 3.527188539505005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1118 , TFLOPS: 70.62588605976397, Tokens per sec: 175018.72573970788, Loss: 3.5259511470794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1119 , TFLOPS: 68.5716211579087, Tokens per sec: 169928.03101695032, Loss: 3.5077781677246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1120 , TFLOPS: 69.535548835467, Tokens per sec: 172316.74998733934, Loss: 3.5231804847717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1121 , TFLOPS: 70.58180071801787, Tokens per sec: 174909.47740645974, Loss: 3.4916741847991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1122 , TFLOPS: 70.58731935722697, Tokens per sec: 174923.15320234819, Loss: 3.482426881790161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1123 , TFLOPS: 68.09318008289364, Tokens per sec: 168742.40132842772, Loss: 3.512786626815796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1124 , TFLOPS: 70.51239439868029, Tokens per sec: 174737.48090140393, Loss: 3.5103282928466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1125 , TFLOPS: 70.63948437168364, Tokens per sec: 175052.42385462255, Loss: 3.5231447219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1126 , TFLOPS: 68.59608375157823, Tokens per sec: 169988.6520188406, Loss: 3.4988605976104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1127 , TFLOPS: 69.54465741360592, Tokens per sec: 172339.3220473603, Loss: 3.47664475440979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1128 , TFLOPS: 70.5938134971656, Tokens per sec: 174939.2463965047, Loss: 3.5035200119018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1129 , TFLOPS: 69.83030043705486, Tokens per sec: 173047.1769256434, Loss: 3.4948132038116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1130 , TFLOPS: 69.778668567723, Tokens per sec: 172919.2274657191, Loss: 3.498528003692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1131 , TFLOPS: 69.65755417506185, Tokens per sec: 172619.092658853, Loss: 3.4973032474517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1132 , TFLOPS: 70.59542412501962, Tokens per sec: 174943.23770975225, Loss: 3.5057573318481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1133 , TFLOPS: 68.33463203047356, Tokens per sec: 169340.74585265855, Loss: 3.486375093460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1134 , TFLOPS: 70.58351157013972, Tokens per sec: 174913.7170864836, Loss: 3.477621555328369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1135 , TFLOPS: 69.56756000646791, Tokens per sec: 172396.0771954012, Loss: 3.5216825008392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1136 , TFLOPS: 68.62577786058337, Tokens per sec: 170062.23729203147, Loss: 3.5022952556610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1137 , TFLOPS: 70.53635068740893, Tokens per sec: 174796.8472238766, Loss: 3.532620906829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1138 , TFLOPS: 69.56196946089291, Tokens per sec: 172382.22320761692, Loss: 3.5145909786224365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1139 , TFLOPS: 69.53729022553735, Tokens per sec: 172321.06534376362, Loss: 3.4809763431549072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1140 , TFLOPS: 69.25570954607194, Tokens per sec: 171623.27740137585, Loss: 3.50846529006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1141 , TFLOPS: 70.59970448853451, Tokens per sec: 174953.84492206312, Loss: 3.4902682304382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1142 , TFLOPS: 69.70028740249776, Tokens per sec: 172724.99030389282, Loss: 3.50187087059021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1143 , TFLOPS: 68.26317323622675, Tokens per sec: 169163.6630886789, Loss: 3.531893730163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1144 , TFLOPS: 70.62214681720019, Tokens per sec: 175009.45948472325, Loss: 3.4996726512908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1145 , TFLOPS: 70.61660950695622, Tokens per sec: 174995.73741995366, Loss: 3.527693748474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1146 , TFLOPS: 67.67967562874568, Tokens per sec: 167717.69173977312, Loss: 3.5026018619537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1147 , TFLOPS: 70.6198495523267, Tokens per sec: 175003.7666093021, Loss: 3.4922542572021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1148 , TFLOPS: 70.63049729799326, Tokens per sec: 175030.15289600988, Loss: 3.5091705322265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1149 , TFLOPS: 69.67674737375195, Tokens per sec: 172666.65551951248, Loss: 3.5218420028686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1150 , TFLOPS: 69.40221121787337, Tokens per sec: 171986.3246825912, Loss: 3.495352268218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1151 , TFLOPS: 70.6147150388849, Tokens per sec: 174991.04271938003, Loss: 3.505892753601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1152 , TFLOPS: 70.63271284967675, Tokens per sec: 175035.64327714473, Loss: 3.4762210845947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1153 , TFLOPS: 67.73150853793203, Tokens per sec: 167846.1393986032, Loss: 3.513460397720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1154 , TFLOPS: 70.59615470479488, Tokens per sec: 174945.04816691534, Loss: 3.513078212738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1155 , TFLOPS: 70.5880154369141, Tokens per sec: 174924.8781644924, Loss: 3.518364906311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1156 , TFLOPS: 68.60466398088042, Tokens per sec: 170009.9147722417, Loss: 3.529566764831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1157 , TFLOPS: 69.55945589569089, Tokens per sec: 172375.9943161542, Loss: 3.5191650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1158 , TFLOPS: 70.60422887671145, Tokens per sec: 174965.05685436804, Loss: 3.5028271675109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1159 , TFLOPS: 69.4946590836238, Tokens per sec: 172215.42067789385, Loss: 3.5091171264648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1160 , TFLOPS: 68.84642774704518, Tokens per sec: 170609.0320172754, Loss: 3.540431022644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 70.63762264200813, Tokens per sec: 175047.81028340018, Loss: 3.472507953643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1162 , TFLOPS: 70.61186568247382, Tokens per sec: 174983.98170031083, Loss: 3.504579544067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1163 , TFLOPS: 68.701662688572, Tokens per sec: 170250.28825519118, Loss: 3.486525774002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1164 , TFLOPS: 69.53465464324125, Tokens per sec: 172314.5340805005, Loss: 3.504969596862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1165 , TFLOPS: 70.57747695979316, Tokens per sec: 174898.7626572776, Loss: 3.5132837295532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1166 , TFLOPS: 68.65451172901804, Tokens per sec: 170133.4429833095, Loss: 3.4958865642547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1167 , TFLOPS: 70.60087034915509, Tokens per sec: 174956.73405311105, Loss: 3.4988245964050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1168 , TFLOPS: 69.66217129728254, Tokens per sec: 172630.5343963339, Loss: 3.506962537765503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1169 , TFLOPS: 69.75966997964967, Tokens per sec: 172872.14687160615, Loss: 3.5010387897491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1170 , TFLOPS: 69.45399129140033, Tokens per sec: 172114.64140883696, Loss: 3.5172529220581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1171 , TFLOPS: 70.59110838268805, Tokens per sec: 174932.54282483636, Loss: 3.50984263420105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1172 , TFLOPS: 69.58515446566814, Tokens per sec: 172439.67820348905, Loss: 3.513199806213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1173 , TFLOPS: 68.65362677883589, Tokens per sec: 170131.2499792728, Loss: 3.5071489810943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1174 , TFLOPS: 70.54844579506462, Tokens per sec: 174826.8202330335, Loss: 3.553081750869751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1175 , TFLOPS: 69.52189500105453, Tokens per sec: 172282.91428157123, Loss: 3.5253796577453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1176 , TFLOPS: 68.66872487360838, Tokens per sec: 170168.66472131148, Loss: 3.5077571868896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1177 , TFLOPS: 70.6040476456223, Tokens per sec: 174964.60774376494, Loss: 3.496852159500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1178 , TFLOPS: 70.56233755686954, Tokens per sec: 174861.24554908997, Loss: 3.4800901412963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1179 , TFLOPS: 69.68535828143895, Tokens per sec: 172687.99429733065, Loss: 3.477583885192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1180 , TFLOPS: 68.83121048607207, Tokens per sec: 170571.32196826432, Loss: 3.5107014179229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1181 , TFLOPS: 70.52961861628465, Tokens per sec: 174780.16441003178, Loss: 3.5026984214782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1182 , TFLOPS: 70.55953796779022, Tokens per sec: 174854.30785895127, Loss: 3.501807451248169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1183 , TFLOPS: 67.651373016191, Tokens per sec: 167647.55474807246, Loss: 3.5036122798919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1184 , TFLOPS: 70.57983379456275, Tokens per sec: 174904.60315346345, Loss: 3.5320065021514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1185 , TFLOPS: 70.56985435351703, Tokens per sec: 174879.87299922598, Loss: 3.5081746578216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1186 , TFLOPS: 68.6599089296799, Tokens per sec: 170146.8178411003, Loss: 3.5024631023406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1187 , TFLOPS: 69.64512452822537, Tokens per sec: 172588.29062475357, Loss: 3.534403085708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1188 , TFLOPS: 70.56304358308523, Tokens per sec: 174862.99515982767, Loss: 3.511974334716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1189 , TFLOPS: 70.6076312939884, Tokens per sec: 174973.48841918758, Loss: 3.498518705368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1190 , TFLOPS: 67.84592883747779, Tokens per sec: 168129.6855053141, Loss: 3.511228084564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1191 , TFLOPS: 70.55142229578182, Tokens per sec: 174834.19632969933, Loss: 3.5359933376312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1192 , TFLOPS: 70.60655701812377, Tokens per sec: 174970.82624525373, Loss: 3.5007855892181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1193 , TFLOPS: 68.72105808794768, Tokens per sec: 170298.35219142374, Loss: 3.518159866333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1194 , TFLOPS: 69.62954445530335, Tokens per sec: 172549.68148776612, Loss: 3.4913861751556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1195 , TFLOPS: 70.62309864678339, Tokens per sec: 175011.81822328313, Loss: 3.508281946182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1196 , TFLOPS: 69.63996616948671, Tokens per sec: 172575.50764355907, Loss: 3.5039889812469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1197 , TFLOPS: 68.78994030722286, Tokens per sec: 170469.0499187905, Loss: 3.525700569152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1198 , TFLOPS: 70.56533370187269, Tokens per sec: 174868.67032646041, Loss: 3.5139763355255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1199 , TFLOPS: 70.56115843918408, Tokens per sec: 174858.3235655749, Loss: 3.4975473880767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1200 , TFLOPS: 68.7319028124532, Tokens per sec: 170325.22661339346, Loss: 3.5337460041046143 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/optimizer.pt +[rank0]:[2024-08-24 23:00:37,426] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006419741999707185, 'preprocessing_with_comm': 0.0013333059978322126, 'state_converting': 0.9589731619998929, : 0.9681563670019386}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200, took 4.24s + eval ppl=23.951292037963867, eval loss=3.1760222911834717 +------------------------------------------------------------------ +iteration: 1201 , TFLOPS: 65.00745250159511, Tokens per sec: 161095.62846101567, Loss: 3.5100626945495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1202 , TFLOPS: 70.56345082440885, Tokens per sec: 174864.00434868893, Loss: 3.5016984939575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1203 , TFLOPS: 68.72868048581053, Tokens per sec: 170317.24133300432, Loss: 3.511089324951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1204 , TFLOPS: 70.5352594937087, Tokens per sec: 174794.14312568252, Loss: 3.5146725177764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1205 , TFLOPS: 69.54166333293928, Tokens per sec: 172331.90238564313, Loss: 3.5136306285858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1206 , TFLOPS: 69.743020009438, Tokens per sec: 172830.88641127598, Loss: 3.516854763031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1207 , TFLOPS: 68.85401568621518, Tokens per sec: 170627.83576641916, Loss: 3.496772050857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1208 , TFLOPS: 70.51876261780323, Tokens per sec: 174753.26204990785, Loss: 3.5229904651641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1209 , TFLOPS: 69.43103492603386, Tokens per sec: 172057.75300660645, Loss: 3.5297536849975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1210 , TFLOPS: 68.51659779781811, Tokens per sec: 169791.67706932238, Loss: 3.518131971359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1211 , TFLOPS: 70.59325864363488, Tokens per sec: 174937.8714083334, Loss: 3.5267820358276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1212 , TFLOPS: 70.57953936624834, Tokens per sec: 174903.87352766638, Loss: 3.493590831756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1213 , TFLOPS: 67.69125821153838, Tokens per sec: 167746.39465586023, Loss: 3.486229658126831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1214 , TFLOPS: 70.55859883391525, Tokens per sec: 174851.98058175493, Loss: 3.500009298324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1215 , TFLOPS: 70.62990481854997, Tokens per sec: 175028.68466668244, Loss: 3.500664710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1216 , TFLOPS: 69.50942254911335, Tokens per sec: 172252.00617170698, Loss: 3.499060869216919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1217 , TFLOPS: 69.45489028028852, Tokens per sec: 172116.8692023337, Loss: 3.501054286956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1218 , TFLOPS: 70.5805742984096, Tokens per sec: 174906.4382035692, Loss: 3.5019075870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1219 , TFLOPS: 70.60761282819894, Tokens per sec: 174973.4426589271, Loss: 3.513930082321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1220 , TFLOPS: 68.44888015980393, Tokens per sec: 169623.8653026096, Loss: 3.4976015090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1221 , TFLOPS: 70.5841859216839, Tokens per sec: 174915.38820389603, Loss: 3.5231804847717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1222 , TFLOPS: 70.64023143415803, Tokens per sec: 175054.27515772977, Loss: 3.531857967376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1223 , TFLOPS: 68.62718962055754, Tokens per sec: 170065.73578876568, Loss: 3.503324508666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1224 , TFLOPS: 69.55797112168338, Tokens per sec: 172372.31488260222, Loss: 3.4912338256835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1225 , TFLOPS: 70.58440327111306, Tokens per sec: 174915.92681972453, Loss: 3.5040998458862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1226 , TFLOPS: 70.55129434126502, Tokens per sec: 174833.87924431136, Loss: 3.5082831382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1227 , TFLOPS: 69.76612004831078, Tokens per sec: 172888.1308522821, Loss: 3.520754814147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1228 , TFLOPS: 69.6217676756611, Tokens per sec: 172530.4097711296, Loss: 3.512875556945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1229 , TFLOPS: 70.56573002181058, Tokens per sec: 174869.65245092477, Loss: 3.492124319076538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1230 , TFLOPS: 68.55436245089265, Tokens per sec: 169885.26203976103, Loss: 3.488983631134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1231 , TFLOPS: 69.54512773850414, Tokens per sec: 172340.48756426864, Loss: 3.505021095275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1232 , TFLOPS: 70.61643801733928, Tokens per sec: 174995.3124497917, Loss: 3.517219066619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1233 , TFLOPS: 68.7287326617417, Tokens per sec: 170317.37063070943, Loss: 3.5109305381774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1234 , TFLOPS: 70.55954801094438, Tokens per sec: 174854.3327469936, Loss: 3.527961015701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1235 , TFLOPS: 69.50380709943649, Tokens per sec: 172238.09046881483, Loss: 3.537576913833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1236 , TFLOPS: 69.68555911482423, Tokens per sec: 172688.491984583, Loss: 3.494208574295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1237 , TFLOPS: 69.4618893329232, Tokens per sec: 172134.21362577175, Loss: 3.5050575733184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1238 , TFLOPS: 69.57426088736628, Tokens per sec: 172412.6827164303, Loss: 3.501044750213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1239 , TFLOPS: 70.61923211996083, Tokens per sec: 175002.23654388372, Loss: 3.5276055335998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1240 , TFLOPS: 69.5090036104652, Tokens per sec: 172250.96799558698, Loss: 3.5052316188812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1241 , TFLOPS: 70.60702399065272, Tokens per sec: 174971.98345462178, Loss: 3.511500358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1242 , TFLOPS: 69.6051139311059, Tokens per sec: 172489.13995756043, Loss: 3.5060815811157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1243 , TFLOPS: 69.65320785433066, Tokens per sec: 172608.32199729438, Loss: 3.523315906524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1244 , TFLOPS: 69.76315102989538, Tokens per sec: 172880.77329758438, Loss: 3.511237144470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1245 , TFLOPS: 70.5391581733614, Tokens per sec: 174803.80448333692, Loss: 3.5128228664398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1246 , TFLOPS: 69.58956681722567, Tokens per sec: 172450.61249670922, Loss: 3.5207319259643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1247 , TFLOPS: 69.56657850877491, Tokens per sec: 172393.644935997, Loss: 3.497293710708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1248 , TFLOPS: 70.57938862465188, Tokens per sec: 174903.49997338484, Loss: 3.5151076316833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1249 , TFLOPS: 70.59887707624684, Tokens per sec: 174951.79450326724, Loss: 3.5325310230255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1250 , TFLOPS: 68.47145534386617, Tokens per sec: 169679.80909557766, Loss: 3.509160041809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1251 , TFLOPS: 70.55209061894509, Tokens per sec: 174835.85250811977, Loss: 3.5174429416656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1252 , TFLOPS: 70.55008294231041, Tokens per sec: 174830.87726425525, Loss: 3.4996755123138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1253 , TFLOPS: 67.83186176887006, Tokens per sec: 168094.8257596886, Loss: 3.5107533931732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1254 , TFLOPS: 70.62663272156608, Tokens per sec: 175020.5760499051, Loss: 3.5065407752990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1255 , TFLOPS: 70.55867189872744, Tokens per sec: 174852.16164440825, Loss: 3.501263380050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1256 , TFLOPS: 69.6557022005771, Tokens per sec: 172614.50326206797, Loss: 3.4755940437316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1257 , TFLOPS: 68.4995987435263, Tokens per sec: 169749.5515401869, Loss: 3.493368148803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1258 , TFLOPS: 70.6071406688661, Tokens per sec: 174972.27259608769, Loss: 3.5025038719177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1259 , TFLOPS: 70.66535159067615, Tokens per sec: 175116.5256727954, Loss: 3.5089340209960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1260 , TFLOPS: 69.48548052653082, Tokens per sec: 172192.67520231596, Loss: 3.5049550533294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1261 , TFLOPS: 69.726089973219, Tokens per sec: 172788.93191652786, Loss: 3.51104474067688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1262 , TFLOPS: 70.57058416898354, Tokens per sec: 174881.6815623478, Loss: 3.517508029937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1263 , TFLOPS: 69.72865457066405, Tokens per sec: 172795.28727151043, Loss: 3.5232791900634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1264 , TFLOPS: 69.51639583993285, Tokens per sec: 172269.2867545291, Loss: 3.4979748725891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1265 , TFLOPS: 69.53851187454228, Tokens per sec: 172324.0927245736, Loss: 3.4888553619384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1266 , TFLOPS: 70.61352256694862, Tokens per sec: 174988.08764256028, Loss: 3.499532699584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1267 , TFLOPS: 69.52022757583282, Tokens per sec: 172278.78221818976, Loss: 3.4930758476257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1268 , TFLOPS: 69.69998526394022, Tokens per sec: 172724.24157126393, Loss: 3.501237630844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1269 , TFLOPS: 70.55180186561456, Tokens per sec: 174835.13694555793, Loss: 3.50417160987854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1270 , TFLOPS: 68.68608414197732, Tokens per sec: 170211.682900611, Loss: 3.4931578636169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1271 , TFLOPS: 70.49642847889622, Tokens per sec: 174697.9155933878, Loss: 3.5013444423675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1272 , TFLOPS: 69.55592095114221, Tokens per sec: 172367.23433415632, Loss: 3.493736982345581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1273 , TFLOPS: 69.80833556515347, Tokens per sec: 172992.74555344137, Loss: 3.513672351837158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1274 , TFLOPS: 69.77380835710504, Tokens per sec: 172907.18332841143, Loss: 3.487732410430908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1275 , TFLOPS: 69.7260941330263, Tokens per sec: 172788.94222498857, Loss: 3.51106333732605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1276 , TFLOPS: 70.59822852071436, Tokens per sec: 174950.1873112135, Loss: 3.515573501586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1277 , TFLOPS: 68.87355288212112, Tokens per sec: 170676.25109007384, Loss: 3.523798942565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1278 , TFLOPS: 70.59228535781544, Tokens per sec: 174935.45949885654, Loss: 3.5007457733154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1279 , TFLOPS: 69.55941355469736, Tokens per sec: 172375.889390508, Loss: 3.5016632080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1280 , TFLOPS: 69.48128424042518, Tokens per sec: 172182.27634308708, Loss: 3.4930877685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1281 , TFLOPS: 70.64684432756523, Tokens per sec: 175070.66263606318, Loss: 3.4925246238708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1282 , TFLOPS: 70.56493509150836, Tokens per sec: 174867.68252606704, Loss: 3.5040688514709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1283 , TFLOPS: 68.68398983717185, Tokens per sec: 170206.49298259633, Loss: 3.532796859741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1284 , TFLOPS: 69.510173597863, Tokens per sec: 172253.86735324343, Loss: 3.5146777629852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1285 , TFLOPS: 70.60961508860555, Tokens per sec: 174978.40448078187, Loss: 3.5002100467681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1286 , TFLOPS: 70.57962371823677, Tokens per sec: 174904.08256118587, Loss: 3.481116771697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1287 , TFLOPS: 68.59634502190748, Tokens per sec: 169989.29947549777, Loss: 3.508246898651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1288 , TFLOPS: 70.5463390505047, Tokens per sec: 174821.5994879369, Loss: 3.479374408721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1289 , TFLOPS: 70.55703578205052, Tokens per sec: 174848.10716704972, Loss: 3.5190131664276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1290 , TFLOPS: 67.18268348693057, Tokens per sec: 166486.09046415126, Loss: 3.505798578262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1291 , TFLOPS: 70.63675130942951, Tokens per sec: 175045.65102528516, Loss: 3.494499921798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1292 , TFLOPS: 70.53955150199957, Tokens per sec: 174804.77919503103, Loss: 3.5029056072235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1293 , TFLOPS: 70.56923998673545, Tokens per sec: 174878.35053066316, Loss: 3.507708787918091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1294 , TFLOPS: 67.91314680470973, Tokens per sec: 168296.2590327852, Loss: 3.5180983543395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1295 , TFLOPS: 70.57264205221561, Tokens per sec: 174886.78122369145, Loss: 3.4955554008483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1296 , TFLOPS: 70.61467668716972, Tokens per sec: 174990.947679605, Loss: 3.4932973384857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1297 , TFLOPS: 68.59582128340426, Tokens per sec: 169988.00159379226, Loss: 3.5215446949005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1298 , TFLOPS: 69.64454494693886, Tokens per sec: 172586.8543584795, Loss: 3.495499610900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1299 , TFLOPS: 70.59391055295102, Tokens per sec: 174939.4869114325, Loss: 3.538667917251587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1300 , TFLOPS: 69.51772174549309, Tokens per sec: 172272.57249456792, Loss: 3.522200584411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1301 , TFLOPS: 70.54486573612205, Tokens per sec: 174817.94845259778, Loss: 3.5086419582366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1302 , TFLOPS: 69.6012961283198, Tokens per sec: 172479.6790216908, Loss: 3.4984934329986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1303 , TFLOPS: 69.79777060997156, Tokens per sec: 172966.5644306747, Loss: 3.5062849521636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1304 , TFLOPS: 69.52423279737462, Tokens per sec: 172288.7075983819, Loss: 3.5060956478118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1305 , TFLOPS: 69.65352451527728, Tokens per sec: 172609.1067180032, Loss: 3.5117111206054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1306 , TFLOPS: 70.59290947262876, Tokens per sec: 174937.00612411587, Loss: 3.485924243927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1307 , TFLOPS: 69.57041703141348, Tokens per sec: 172403.1572179436, Loss: 3.515101909637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1308 , TFLOPS: 70.64923958600548, Tokens per sec: 175076.59835033765, Loss: 3.5075128078460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1309 , TFLOPS: 69.5737997682624, Tokens per sec: 172411.54001249443, Loss: 3.517552375793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1310 , TFLOPS: 69.71527253031341, Tokens per sec: 172762.12510136812, Loss: 3.5128345489501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1311 , TFLOPS: 69.71876269282684, Tokens per sec: 172770.77410854952, Loss: 3.492785692214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1312 , TFLOPS: 69.67182898277736, Tokens per sec: 172654.46720487194, Loss: 3.5280158519744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1313 , TFLOPS: 70.60243427045971, Tokens per sec: 174960.60962238407, Loss: 3.51002836227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1314 , TFLOPS: 68.612553654115, Tokens per sec: 170029.46625163627, Loss: 3.5033884048461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1315 , TFLOPS: 70.59670083038604, Tokens per sec: 174946.40152629095, Loss: 3.513963460922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1316 , TFLOPS: 69.59517327183082, Tokens per sec: 172464.50590882273, Loss: 3.488410234451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1317 , TFLOPS: 68.61979072404054, Tokens per sec: 170047.4005081408, Loss: 3.5033345222473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1318 , TFLOPS: 70.61538282868267, Tokens per sec: 174992.697576062, Loss: 3.483046054840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1319 , TFLOPS: 70.54530071355912, Tokens per sec: 174819.02637460353, Loss: 3.4892959594726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1320 , TFLOPS: 68.64887138237631, Tokens per sec: 170119.46558299582, Loss: 3.49188232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1321 , TFLOPS: 70.62358859871115, Tokens per sec: 175013.03237813278, Loss: 3.5146656036376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1322 , TFLOPS: 70.56706386518525, Tokens per sec: 174872.95786174212, Loss: 3.515535831451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1323 , TFLOPS: 69.61557378121155, Tokens per sec: 172515.06061836984, Loss: 3.5125691890716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1324 , TFLOPS: 68.46862065015564, Tokens per sec: 169672.78441229567, Loss: 3.52854061126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1325 , TFLOPS: 70.59649727135479, Tokens per sec: 174945.89708458763, Loss: 3.494873523712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1326 , TFLOPS: 70.5942831250518, Tokens per sec: 174940.41018614048, Loss: 3.507352352142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1327 , TFLOPS: 68.49925894291353, Tokens per sec: 169748.70947683626, Loss: 3.4944281578063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1328 , TFLOPS: 70.60188892234535, Tokens per sec: 174959.25818968166, Loss: 3.5112476348876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1329 , TFLOPS: 70.56418922580689, Tokens per sec: 174865.8341886949, Loss: 3.5094807147979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1330 , TFLOPS: 69.69504457269807, Tokens per sec: 172711.9979941038, Loss: 3.5017101764678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1331 , TFLOPS: 68.57354606973404, Tokens per sec: 169932.80116050006, Loss: 3.522063970565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1332 , TFLOPS: 70.55411559715242, Tokens per sec: 174840.87062718734, Loss: 3.528442144393921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1333 , TFLOPS: 70.59056644724042, Tokens per sec: 174931.19984909802, Loss: 3.4867022037506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1334 , TFLOPS: 68.65730212051085, Tokens per sec: 170140.3578808157, Loss: 3.5155045986175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1335 , TFLOPS: 69.57505927286648, Tokens per sec: 172414.66120364922, Loss: 3.489041805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1336 , TFLOPS: 70.58564410913016, Tokens per sec: 174919.00175302048, Loss: 3.4936490058898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1337 , TFLOPS: 68.61609525326837, Tokens per sec: 170038.24272447766, Loss: 3.5236687660217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1338 , TFLOPS: 70.58783269853063, Tokens per sec: 174924.42531864787, Loss: 3.4791741371154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1339 , TFLOPS: 69.56332702994187, Tokens per sec: 172385.5874132688, Loss: 3.497037649154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1340 , TFLOPS: 69.15635814810125, Tokens per sec: 171377.07369274445, Loss: 3.4980709552764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1341 , TFLOPS: 70.56350813688962, Tokens per sec: 174864.1463753298, Loss: 3.499610662460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1342 , TFLOPS: 69.54262534699147, Tokens per sec: 172334.2863624391, Loss: 3.5101492404937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1343 , TFLOPS: 70.57225200442466, Tokens per sec: 174885.81464229815, Loss: 3.5058369636535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1344 , TFLOPS: 68.86590667535722, Tokens per sec: 170657.30294741373, Loss: 3.486377239227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1345 , TFLOPS: 70.55710796913066, Tokens per sec: 174848.28605458638, Loss: 3.5134518146514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1346 , TFLOPS: 69.68050730012388, Tokens per sec: 172675.9730312522, Loss: 3.4850425720214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1347 , TFLOPS: 69.52306779612489, Tokens per sec: 172285.8205969496, Loss: 3.50763201713562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1348 , TFLOPS: 70.59302569003037, Tokens per sec: 174937.29412363944, Loss: 3.498721122741699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1349 , TFLOPS: 69.58213026694213, Tokens per sec: 172432.183905903, Loss: 3.4915084838867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1350 , TFLOPS: 69.76870069290402, Tokens per sec: 172894.52597386565, Loss: 3.52152681350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1351 , TFLOPS: 69.51190073013528, Tokens per sec: 172258.1473772735, Loss: 3.506326913833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1352 , TFLOPS: 70.57808927781095, Tokens per sec: 174900.28004877808, Loss: 3.490055799484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1353 , TFLOPS: 69.59838526680532, Tokens per sec: 172472.46558620004, Loss: 3.5044474601745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1354 , TFLOPS: 68.67667240604223, Tokens per sec: 170188.35958217553, Loss: 3.508852481842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1355 , TFLOPS: 70.63353899154257, Tokens per sec: 175037.69054769527, Loss: 3.493344306945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1356 , TFLOPS: 70.59465253823092, Tokens per sec: 174941.32563269252, Loss: 3.4884400367736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1357 , TFLOPS: 67.8042917020836, Tokens per sec: 168026.5040970982, Loss: 3.500162124633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1358 , TFLOPS: 70.57898847241651, Tokens per sec: 174902.50835206517, Loss: 3.4977753162384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1359 , TFLOPS: 70.62174956185514, Tokens per sec: 175008.47504221695, Loss: 3.4976601600646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1360 , TFLOPS: 70.5661860520146, Tokens per sec: 174870.78254400616, Loss: 3.5061404705047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1361 , TFLOPS: 68.8535998558521, Tokens per sec: 170626.80529297196, Loss: 3.5123867988586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1362 , TFLOPS: 70.60446442765105, Tokens per sec: 174965.6405755445, Loss: 3.5185444355010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1363 , TFLOPS: 70.56927225026458, Tokens per sec: 174878.43048324238, Loss: 3.5146360397338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1364 , TFLOPS: 67.86161331861156, Tokens per sec: 168168.55337735178, Loss: 3.514261484146118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1365 , TFLOPS: 70.63252798575662, Tokens per sec: 175035.18516398623, Loss: 3.506194829940796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1366 , TFLOPS: 70.55479909041698, Tokens per sec: 174842.5643987899, Loss: 3.481536865234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1367 , TFLOPS: 69.57163697400097, Tokens per sec: 172406.18037006314, Loss: 3.4853928089141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1368 , TFLOPS: 69.33227927926666, Tokens per sec: 171813.02563508478, Loss: 3.525660991668701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1369 , TFLOPS: 70.61777652879981, Tokens per sec: 174998.62942864012, Loss: 3.494515895843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1370 , TFLOPS: 69.62561485506997, Tokens per sec: 172539.9435055066, Loss: 3.5024003982543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1371 , TFLOPS: 69.50373581972879, Tokens per sec: 172237.91382984706, Loss: 3.4958131313323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1372 , TFLOPS: 69.6704818354093, Tokens per sec: 172651.1288252934, Loss: 3.4994966983795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1373 , TFLOPS: 70.6092471384615, Tokens per sec: 174977.49265979184, Loss: 3.5333011150360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1374 , TFLOPS: 68.69198915583074, Tokens per sec: 170226.31617543075, Loss: 3.5005669593811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1375 , TFLOPS: 70.54878241808096, Tokens per sec: 174827.65442195008, Loss: 3.5110909938812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1376 , TFLOPS: 69.53150702524135, Tokens per sec: 172306.733936358, Loss: 3.533381700515747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1377 , TFLOPS: 69.70695246940772, Tokens per sec: 172741.50707391344, Loss: 3.5038743019104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1378 , TFLOPS: 69.82781852896444, Tokens per sec: 173041.0264839906, Loss: 3.4847631454467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1379 , TFLOPS: 69.62471568052284, Tokens per sec: 172537.71525192657, Loss: 3.5293381214141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1380 , TFLOPS: 70.56590002374988, Tokens per sec: 174870.07373445758, Loss: 3.5070860385894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1381 , TFLOPS: 69.53977351789244, Tokens per sec: 172327.21921577677, Loss: 3.5040712356567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1382 , TFLOPS: 70.55892005684615, Tokens per sec: 174852.7766075633, Loss: 3.5299131870269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1383 , TFLOPS: 69.54246410362178, Tokens per sec: 172333.88678360658, Loss: 3.481013059616089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1384 , TFLOPS: 68.6867317963407, Tokens per sec: 170213.28785946846, Loss: 3.4967424869537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1385 , TFLOPS: 70.5728399986881, Tokens per sec: 174887.27175685583, Loss: 3.5343785285949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1386 , TFLOPS: 69.52575280723296, Tokens per sec: 172292.47435025376, Loss: 3.5192532539367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1387 , TFLOPS: 69.52372981536931, Tokens per sec: 172287.46115356474, Loss: 3.4903438091278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1388 , TFLOPS: 70.60597147503026, Tokens per sec: 174969.37520496512, Loss: 3.4781973361968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1389 , TFLOPS: 70.57415248279992, Tokens per sec: 174890.524237067, Loss: 3.4978795051574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1390 , TFLOPS: 69.39241104552164, Tokens per sec: 171962.0387759255, Loss: 3.50893235206604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1391 , TFLOPS: 68.88757789912833, Tokens per sec: 170711.0066272009, Loss: 3.508613109588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1392 , TFLOPS: 70.61272526930716, Tokens per sec: 174986.11185117514, Loss: 3.500880479812622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1393 , TFLOPS: 70.60807502162365, Tokens per sec: 174974.58802514832, Loss: 3.475954294204712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1394 , TFLOPS: 67.83302998283762, Tokens per sec: 168097.7207225896, Loss: 3.5165257453918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1395 , TFLOPS: 70.62085152874701, Tokens per sec: 175006.24961724848, Loss: 3.5101869106292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1396 , TFLOPS: 70.56685295249424, Tokens per sec: 174872.4351968599, Loss: 3.5181353092193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1397 , TFLOPS: 69.75769298479408, Tokens per sec: 172867.24766057005, Loss: 3.504704713821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1398 , TFLOPS: 68.51610545040671, Tokens per sec: 169790.45697820038, Loss: 3.512939929962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1399 , TFLOPS: 70.6077880227294, Tokens per sec: 174973.87681027284, Loss: 3.490349292755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1400 , TFLOPS: 70.58784764356585, Tokens per sec: 174924.46235409146, Loss: 3.5247254371643066 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/model.pt +[rank0]:[2024-08-24 23:26:00,734] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063495130016235635, 'preprocessing_with_comm': 0.001938981000421336, 'state_converting': 0.964750325998466, : 0.9744206750001467}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400, took 4.24s + eval ppl=26.471790313720703, eval loss=3.2760796546936035 +------------------------------------------------------------------ +iteration: 1401 , TFLOPS: 63.669443018478646, Tokens per sec: 157779.8935679408, Loss: 3.5290186405181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1402 , TFLOPS: 70.57257544004155, Tokens per sec: 174886.61615138617, Loss: 3.5225119590759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1403 , TFLOPS: 70.52286890128862, Tokens per sec: 174763.4378727291, Loss: 3.4889779090881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1404 , TFLOPS: 67.43702414102943, Tokens per sec: 167116.37462294422, Loss: 3.4888205528259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1405 , TFLOPS: 70.53019814874959, Tokens per sec: 174781.60055532068, Loss: 3.5200748443603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1406 , TFLOPS: 70.53335752924748, Tokens per sec: 174789.42984822893, Loss: 3.5336098670959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1407 , TFLOPS: 68.45158807689835, Tokens per sec: 169630.57581947127, Loss: 3.527747631072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1408 , TFLOPS: 69.50594705434989, Tokens per sec: 172243.3935128257, Loss: 3.501718282699585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1409 , TFLOPS: 70.45536104561035, Tokens per sec: 174596.14596975414, Loss: 3.5136594772338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1410 , TFLOPS: 69.54315297390649, Tokens per sec: 172335.5938800574, Loss: 3.5178356170654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1411 , TFLOPS: 68.73400384160288, Tokens per sec: 170330.43319507383, Loss: 3.530369281768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1412 , TFLOPS: 69.48073566787657, Tokens per sec: 172180.9169198814, Loss: 3.4942874908447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1413 , TFLOPS: 70.52941826260543, Tokens per sec: 174779.66791154392, Loss: 3.483916997909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1414 , TFLOPS: 68.14579509334871, Tokens per sec: 168872.78712035727, Loss: 3.514873504638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1415 , TFLOPS: 70.54203956172567, Tokens per sec: 174810.94488111473, Loss: 3.4996299743652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1416 , TFLOPS: 69.57050373320301, Tokens per sec: 172403.37207452903, Loss: 3.493879795074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1417 , TFLOPS: 68.58505683390453, Tokens per sec: 169961.3261021299, Loss: 3.481656789779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1418 , TFLOPS: 70.00399247918416, Tokens per sec: 173477.6049397978, Loss: 3.5063276290893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1419 , TFLOPS: 69.48505000288424, Tokens per sec: 172191.6083172936, Loss: 3.5085909366607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1420 , TFLOPS: 70.58300426698165, Tokens per sec: 174912.45993337384, Loss: 3.5193347930908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1421 , TFLOPS: 68.20721789518647, Tokens per sec: 169024.99958959143, Loss: 3.491957664489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1422 , TFLOPS: 70.56641409759023, Tokens per sec: 174871.3476660639, Loss: 3.5176119804382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1423 , TFLOPS: 69.54331206467157, Tokens per sec: 172335.98812449857, Loss: 3.5070741176605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1424 , TFLOPS: 67.77102351002613, Tokens per sec: 167944.06185238587, Loss: 3.5004353523254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1425 , TFLOPS: 70.56361681830623, Tokens per sec: 174864.4156998524, Loss: 3.4980499744415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1426 , TFLOPS: 70.63608436880061, Tokens per sec: 175043.99827293714, Loss: 3.5069518089294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1427 , TFLOPS: 67.72066289809003, Tokens per sec: 167819.26270832884, Loss: 3.4957566261291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1428 , TFLOPS: 70.60014937813614, Tokens per sec: 174954.94740750457, Loss: 3.5135436058044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1429 , TFLOPS: 70.62020309695875, Tokens per sec: 175004.6427318465, Loss: 3.503427505493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1430 , TFLOPS: 69.53352683510279, Tokens per sec: 172311.7392476951, Loss: 3.4980459213256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1431 , TFLOPS: 67.72867536801643, Tokens per sec: 167839.11849145291, Loss: 3.501800298690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1432 , TFLOPS: 70.54902349354226, Tokens per sec: 174828.25183349985, Loss: 3.49995756149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1433 , TFLOPS: 70.5984332006024, Tokens per sec: 174950.6945305234, Loss: 3.516007423400879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1434 , TFLOPS: 68.33796075405417, Tokens per sec: 169348.99479638043, Loss: 3.479757785797119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1435 , TFLOPS: 70.58371204304632, Tokens per sec: 174914.21388043006, Loss: 3.527554988861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1436 , TFLOPS: 70.58189201970033, Tokens per sec: 174909.70366208657, Loss: 3.5095582008361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1437 , TFLOPS: 69.53893279474129, Tokens per sec: 172325.13581119492, Loss: 3.5101852416992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1438 , TFLOPS: 68.00333046084616, Tokens per sec: 168519.74406724112, Loss: 3.504056215286255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1439 , TFLOPS: 70.61660451494596, Tokens per sec: 174995.7250492023, Loss: 3.4959819316864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1440 , TFLOPS: 70.54450711761262, Tokens per sec: 174817.05975642632, Loss: 3.471128463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1441 , TFLOPS: 67.79194691844143, Tokens per sec: 167995.91236334268, Loss: 3.4866654872894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1442 , TFLOPS: 70.53852650336293, Tokens per sec: 174802.2391354965, Loss: 3.4960577487945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1443 , TFLOPS: 70.587484804642, Tokens per sec: 174923.5631992682, Loss: 3.4937639236450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1444 , TFLOPS: 67.36913674967762, Tokens per sec: 166948.14218876092, Loss: 3.508089542388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1445 , TFLOPS: 69.59253468137652, Tokens per sec: 172457.96719101229, Loss: 3.522158622741699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1446 , TFLOPS: 70.55623358182429, Tokens per sec: 174846.11922651948, Loss: 3.500188112258911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1447 , TFLOPS: 69.44698810944834, Tokens per sec: 172097.2867524958, Loss: 3.5150275230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1448 , TFLOPS: 69.91787783798539, Tokens per sec: 173264.20337259638, Loss: 3.4818530082702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1449 , TFLOPS: 69.51884588256581, Tokens per sec: 172275.358230068, Loss: 3.491199493408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1450 , TFLOPS: 70.60203183262871, Tokens per sec: 174959.61233710666, Loss: 3.5125410556793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1451 , TFLOPS: 68.01474814503855, Tokens per sec: 168548.0383464316, Loss: 3.4982306957244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1452 , TFLOPS: 70.53049003264064, Tokens per sec: 174782.32387575635, Loss: 3.489976406097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1453 , TFLOPS: 69.71358338720664, Tokens per sec: 172757.93921867505, Loss: 3.475465774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1454 , TFLOPS: 68.51442253527317, Tokens per sec: 169786.28652911106, Loss: 3.4956793785095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1455 , TFLOPS: 70.55124278353998, Tokens per sec: 174833.75147858917, Loss: 3.4727015495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1456 , TFLOPS: 69.52843756009342, Tokens per sec: 172299.12746359193, Loss: 3.492544174194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1457 , TFLOPS: 68.95175712304633, Tokens per sec: 170870.04981399485, Loss: 3.5081655979156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1458 , TFLOPS: 68.6956250907871, Tokens per sec: 170235.32642278392, Loss: 3.5141515731811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1459 , TFLOPS: 70.59337171831461, Tokens per sec: 174938.15161984597, Loss: 3.5026121139526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1460 , TFLOPS: 69.57078749932244, Tokens per sec: 172404.07527823237, Loss: 3.5383734703063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1461 , TFLOPS: 68.721482898231, Tokens per sec: 170299.40491810255, Loss: 3.5167124271392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1462 , TFLOPS: 70.63163520978418, Tokens per sec: 175032.97276677913, Loss: 3.5166378021240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1463 , TFLOPS: 70.61903858527495, Tokens per sec: 175001.7569436125, Loss: 3.500628709793091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1464 , TFLOPS: 66.7672789198566, Tokens per sec: 165456.67218635124, Loss: 3.516756057739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1465 , TFLOPS: 70.58928076857512, Tokens per sec: 174928.0137957362, Loss: 3.4825007915496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1466 , TFLOPS: 70.60803909720502, Tokens per sec: 174974.4990004816, Loss: 3.518615484237671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1467 , TFLOPS: 69.31375286951551, Tokens per sec: 171767.1151508689, Loss: 3.505514621734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1468 , TFLOPS: 69.72154998409171, Tokens per sec: 172777.68132334933, Loss: 3.5144684314727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1469 , TFLOPS: 70.55536125653099, Tokens per sec: 174843.95750834813, Loss: 3.4946937561035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1470 , TFLOPS: 70.56261754684098, Tokens per sec: 174861.93939508265, Loss: 3.488720178604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1471 , TFLOPS: 66.82128500162055, Tokens per sec: 165590.50520622337, Loss: 3.4896223545074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1472 , TFLOPS: 70.56084005473707, Tokens per sec: 174857.53457384225, Loss: 3.5037753582000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1473 , TFLOPS: 70.5994444073444, Tokens per sec: 174953.2004122237, Loss: 3.5013036727905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1474 , TFLOPS: 68.48406785799105, Tokens per sec: 169711.0642949622, Loss: 3.5011789798736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1475 , TFLOPS: 69.60313646298651, Tokens per sec: 172484.2395737245, Loss: 3.504002809524536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1476 , TFLOPS: 70.59476914730837, Tokens per sec: 174941.614602832, Loss: 3.4802327156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1477 , TFLOPS: 69.49896201065587, Tokens per sec: 172226.0838050859, Loss: 3.499505043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1478 , TFLOPS: 67.61002444427697, Tokens per sec: 167545.08843194827, Loss: 3.520742654800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1479 , TFLOPS: 70.57082771696153, Tokens per sec: 174882.28510106629, Loss: 3.500988245010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1480 , TFLOPS: 70.62120237040786, Tokens per sec: 175007.11904153248, Loss: 3.489234209060669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1481 , TFLOPS: 68.76207217124097, Tokens per sec: 170399.98960789878, Loss: 3.4862983226776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1482 , TFLOPS: 69.66986944127255, Tokens per sec: 172649.61124516174, Loss: 3.501331090927124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1483 , TFLOPS: 70.5737472534643, Tokens per sec: 174889.52003413363, Loss: 3.5247983932495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1484 , TFLOPS: 68.55017171611982, Tokens per sec: 169874.87693734642, Loss: 3.5244221687316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1485 , TFLOPS: 69.89006776396089, Tokens per sec: 173195.2869456311, Loss: 3.520512104034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1486 , TFLOPS: 69.53194648037346, Tokens per sec: 172307.82295458537, Loss: 3.494317054748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1487 , TFLOPS: 70.57887165200461, Tokens per sec: 174902.2188582156, Loss: 3.4882519245147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1488 , TFLOPS: 68.86764967706064, Tokens per sec: 170661.6222976423, Loss: 3.5120599269866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1489 , TFLOPS: 70.51303018367707, Tokens per sec: 174739.05644666348, Loss: 3.5016355514526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1490 , TFLOPS: 69.63351489711145, Tokens per sec: 172559.52067994687, Loss: 3.5075197219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1491 , TFLOPS: 67.62188232008846, Tokens per sec: 167574.473554462, Loss: 3.491299629211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1492 , TFLOPS: 70.64359102127705, Tokens per sec: 175062.60058469983, Loss: 3.487114667892456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1493 , TFLOPS: 69.54693185004261, Tokens per sec: 172344.95835140024, Loss: 3.5283000469207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1494 , TFLOPS: 68.53155039114768, Tokens per sec: 169828.73124275717, Loss: 3.5021393299102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1495 , TFLOPS: 70.61723764999056, Tokens per sec: 174997.29402758836, Loss: 3.5009140968322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1496 , TFLOPS: 70.58701288625429, Tokens per sec: 174922.39373351733, Loss: 3.492823839187622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1497 , TFLOPS: 68.66316214049665, Tokens per sec: 170154.879655875, Loss: 3.508660078048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1498 , TFLOPS: 68.67390720776747, Tokens per sec: 170181.50711623894, Loss: 3.4793074131011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1499 , TFLOPS: 70.59880197005481, Tokens per sec: 174951.6083818498, Loss: 3.500844717025757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1500 , TFLOPS: 70.63908880622576, Tokens per sec: 175051.44359984278, Loss: 3.493840217590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1501 , TFLOPS: 67.73267045560263, Tokens per sec: 167849.01875858675, Loss: 3.5080511569976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1502 , TFLOPS: 70.59158104292935, Tokens per sec: 174933.7141289821, Loss: 3.5049619674682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1503 , TFLOPS: 70.60862424143464, Tokens per sec: 174975.94905234152, Loss: 3.50309681892395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1504 , TFLOPS: 69.47694412423905, Tokens per sec: 172171.5210570746, Loss: 3.498669147491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1505 , TFLOPS: 68.80248530791792, Tokens per sec: 170500.13781245876, Loss: 3.5036168098449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1506 , TFLOPS: 70.62719847319404, Tokens per sec: 175021.97804476187, Loss: 3.516061305999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1507 , TFLOPS: 70.54387641395074, Tokens per sec: 174815.49680327444, Loss: 3.4909400939941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1508 , TFLOPS: 67.67549287625005, Tokens per sec: 167707.3264183207, Loss: 3.483462333679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1509 , TFLOPS: 70.59818831354451, Tokens per sec: 174950.0876734174, Loss: 3.4896132946014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1510 , TFLOPS: 70.60154325304931, Tokens per sec: 174958.40158308743, Loss: 3.499648332595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1511 , TFLOPS: 67.61913966733378, Tokens per sec: 167567.67695880777, Loss: 3.490086317062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1512 , TFLOPS: 69.64744041209873, Tokens per sec: 172594.02964010747, Loss: 3.5219407081604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1513 , TFLOPS: 70.58779833811411, Tokens per sec: 174924.34016975065, Loss: 3.4974284172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1514 , TFLOPS: 68.46824023121233, Tokens per sec: 169671.84169224757, Loss: 3.5063114166259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1515 , TFLOPS: 69.54665347662599, Tokens per sec: 172344.26851140853, Loss: 3.4973843097686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1516 , TFLOPS: 70.61666435329104, Tokens per sec: 174995.87333521314, Loss: 3.5114071369171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1517 , TFLOPS: 69.55444974214555, Tokens per sec: 172363.58851619554, Loss: 3.4903411865234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1518 , TFLOPS: 68.68440584903358, Tokens per sec: 170207.5239058172, Loss: 3.518014907836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1519 , TFLOPS: 69.55175317613917, Tokens per sec: 172356.90612858048, Loss: 3.5017199516296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1520 , TFLOPS: 70.58031565658092, Tokens per sec: 174905.797260626, Loss: 3.5163562297821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1521 , TFLOPS: 68.69640505162539, Tokens per sec: 170237.259251661, Loss: 3.5037455558776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1522 , TFLOPS: 70.5708288363767, Tokens per sec: 174882.2878751004, Loss: 3.4997761249542236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1523 , TFLOPS: 69.61429590297595, Tokens per sec: 172511.89389533212, Loss: 3.517835855484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1524 , TFLOPS: 69.51710079744572, Tokens per sec: 172271.03371690345, Loss: 3.5087523460388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1525 , TFLOPS: 68.78779421787911, Tokens per sec: 170463.73167298542, Loss: 3.5044655799865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1526 , TFLOPS: 70.53125006021685, Tokens per sec: 174784.20730781415, Loss: 3.490691900253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1527 , TFLOPS: 69.5283540858601, Tokens per sec: 172298.92060524633, Loss: 3.500645637512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1528 , TFLOPS: 68.69618905717387, Tokens per sec: 170236.72399361632, Loss: 3.4954397678375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1529 , TFLOPS: 70.61964702779478, Tokens per sec: 175003.26473120213, Loss: 3.4962308406829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1530 , TFLOPS: 69.65066909043105, Tokens per sec: 172602.0306606832, Loss: 3.508916139602661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1531 , TFLOPS: 67.48133373297449, Tokens per sec: 167226.17867288858, Loss: 3.5167014598846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1532 , TFLOPS: 70.61823227116575, Tokens per sec: 174999.7588084271, Loss: 3.4787001609802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1533 , TFLOPS: 70.58645038504814, Tokens per sec: 174920.99979356385, Loss: 3.478729724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1534 , TFLOPS: 68.54698466225538, Tokens per sec: 169866.9790667874, Loss: 3.4950246810913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1535 , TFLOPS: 69.7599891167644, Tokens per sec: 172872.9377285323, Loss: 3.49599289894104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1536 , TFLOPS: 70.5580609801821, Tokens per sec: 174850.6477209547, Loss: 3.5204644203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1537 , TFLOPS: 70.59314590815204, Tokens per sec: 174937.59203738807, Loss: 3.4921329021453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1538 , TFLOPS: 67.19217932732465, Tokens per sec: 166509.62220270094, Loss: 3.506270408630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1539 , TFLOPS: 70.61124026241211, Tokens per sec: 174982.4318405022, Loss: 3.478710651397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1540 , TFLOPS: 70.60172624125403, Tokens per sec: 174958.8550480165, Loss: 3.49938702583313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1541 , TFLOPS: 67.12701954163667, Tokens per sec: 166348.1490758232, Loss: 3.5140552520751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1542 , TFLOPS: 70.64729076355705, Tokens per sec: 175071.76895362991, Loss: 3.4995882511138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1543 , TFLOPS: 70.59497825012564, Tokens per sec: 174942.1327826478, Loss: 3.503622055053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1544 , TFLOPS: 69.46920370790077, Tokens per sec: 172152.3394527111, Loss: 3.4910247325897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1545 , TFLOPS: 65.1650874593001, Tokens per sec: 161486.26525113272, Loss: 3.518874406814575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1546 , TFLOPS: 70.59063002508351, Tokens per sec: 174931.35740199688, Loss: 3.513892889022827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1547 , TFLOPS: 70.6300714173386, Tokens per sec: 175029.09751683302, Loss: 3.486445188522339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1548 , TFLOPS: 68.74492652611484, Tokens per sec: 170357.5008105275, Loss: 3.4911704063415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1549 , TFLOPS: 69.57092969408409, Tokens per sec: 172404.4276525158, Loss: 3.5024733543395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1550 , TFLOPS: 70.61585750002972, Tokens per sec: 174993.87386394935, Loss: 3.4823806285858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1551 , TFLOPS: 68.49413353098497, Tokens per sec: 169736.0081414666, Loss: 3.4874908924102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1552 , TFLOPS: 69.05178332940075, Tokens per sec: 171117.92577213736, Loss: 3.495364189147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1553 , TFLOPS: 70.61175184217957, Tokens per sec: 174983.69959152114, Loss: 3.503549098968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1554 , TFLOPS: 70.56171243087111, Tokens per sec: 174859.69641800254, Loss: 3.4914114475250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1555 , TFLOPS: 68.74623248515094, Tokens per sec: 170360.7371208822, Loss: 3.5032801628112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1556 , TFLOPS: 69.60583796661602, Tokens per sec: 172490.9341973136, Loss: 3.498584747314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1557 , TFLOPS: 70.62483392381228, Tokens per sec: 175016.1184309172, Loss: 3.518860340118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1558 , TFLOPS: 67.25322377573491, Tokens per sec: 166660.8970704627, Loss: 3.5150222778320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1559 , TFLOPS: 70.58089322796347, Tokens per sec: 174907.22854613667, Loss: 3.478440284729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1560 , TFLOPS: 69.67270801136577, Tokens per sec: 172656.645534547, Loss: 3.5005366802215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1561 , TFLOPS: 68.74543536069434, Tokens per sec: 170358.76175866823, Loss: 3.493377208709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1562 , TFLOPS: 70.61028054064778, Tokens per sec: 174980.0535442481, Loss: 3.5122199058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1563 , TFLOPS: 70.58073246901522, Tokens per sec: 174906.83016775383, Loss: 3.509450912475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1564 , TFLOPS: 68.48603089493967, Tokens per sec: 169715.92891676727, Loss: 3.5151472091674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1565 , TFLOPS: 68.71281251607647, Tokens per sec: 170277.91875600215, Loss: 3.486250638961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1566 , TFLOPS: 70.58939504464016, Tokens per sec: 174928.2969844139, Loss: 3.4983468055725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1567 , TFLOPS: 69.62187071057747, Tokens per sec: 172530.66510300263, Loss: 3.4942824840545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1568 , TFLOPS: 68.67914209523454, Tokens per sec: 170194.4797440528, Loss: 3.462935447692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1569 , TFLOPS: 70.61461524165522, Tokens per sec: 174990.7954108513, Loss: 3.4972305297851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1570 , TFLOPS: 70.61192200940539, Tokens per sec: 174984.1212846522, Loss: 3.4890639781951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1571 , TFLOPS: 68.52812006893402, Tokens per sec: 169820.23052643053, Loss: 3.532770872116089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1572 , TFLOPS: 69.05168177678812, Tokens per sec: 171117.67411357633, Loss: 3.5156290531158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1573 , TFLOPS: 70.6043210744138, Tokens per sec: 174965.28533043092, Loss: 3.507398843765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1574 , TFLOPS: 70.58882312221385, Tokens per sec: 174926.8796976391, Loss: 3.5128321647644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1575 , TFLOPS: 67.46723137260744, Tokens per sec: 167191.2314703958, Loss: 3.5316202640533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1576 , TFLOPS: 70.5864266791071, Tokens per sec: 174920.94104763056, Loss: 3.5157458782196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1577 , TFLOPS: 70.5505943319501, Tokens per sec: 174832.14454411657, Loss: 3.5261192321777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1578 , TFLOPS: 67.42758097579794, Tokens per sec: 167092.97341925005, Loss: 3.487900733947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1579 , TFLOPS: 69.65411582004035, Tokens per sec: 172610.57203634366, Loss: 3.5011115074157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1580 , TFLOPS: 70.57972724606718, Tokens per sec: 174904.33911455417, Loss: 3.502978563308716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1581 , TFLOPS: 68.19005576734648, Tokens per sec: 168982.46994623946, Loss: 3.4982666969299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1582 , TFLOPS: 69.57140512833894, Tokens per sec: 172405.60583097272, Loss: 3.517927885055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1583 , TFLOPS: 70.59989950836072, Tokens per sec: 174954.32820267568, Loss: 3.505796194076538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1584 , TFLOPS: 69.4792142156738, Tokens per sec: 172177.14659372115, Loss: 3.500211238861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1585 , TFLOPS: 68.62058792574108, Tokens per sec: 170049.37606177374, Loss: 3.518918991088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1586 , TFLOPS: 69.65339265838222, Tokens per sec: 172608.779962092, Loss: 3.532457113265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1587 , TFLOPS: 70.59103343416574, Tokens per sec: 174932.3570941418, Loss: 3.499994993209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1588 , TFLOPS: 68.73416768441867, Tokens per sec: 170330.83921562013, Loss: 3.50911808013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1589 , TFLOPS: 69.5933318030972, Tokens per sec: 172459.9425464463, Loss: 3.507899284362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1590 , TFLOPS: 70.62656536672063, Tokens per sec: 175020.40913717833, Loss: 3.5075509548187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1591 , TFLOPS: 69.49864548543216, Tokens per sec: 172225.2994207134, Loss: 3.4891958236694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1592 , TFLOPS: 68.66118335308671, Tokens per sec: 170149.97600269172, Loss: 3.491361141204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1593 , TFLOPS: 69.65930235963317, Tokens per sec: 172623.4248527992, Loss: 3.5100560188293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1594 , TFLOPS: 70.59560638495451, Tokens per sec: 174943.6893699485, Loss: 3.5103604793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1595 , TFLOPS: 68.48039033905931, Tokens per sec: 169701.9509979372, Loss: 3.5080859661102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1596 , TFLOPS: 70.6215669690612, Tokens per sec: 175008.0225571593, Loss: 3.4849305152893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1597 , TFLOPS: 69.50813741353454, Tokens per sec: 172248.8214641733, Loss: 3.5095207691192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1598 , TFLOPS: 67.68396578660708, Tokens per sec: 167728.32322355354, Loss: 3.502112865447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1599 , TFLOPS: 70.6143173491132, Tokens per sec: 174990.05720031657, Loss: 3.4877758026123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1600 , TFLOPS: 70.5765027543598, Tokens per sec: 174896.34846889615, Loss: 3.5045058727264404 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/optimizer.pt +[rank0]:[2024-08-24 23:51:29,569] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006299630000285106, 'preprocessing_with_comm': 0.001268503001483623, 'state_converting': 0.9842877690025489, : 0.9932663389990921}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600, took 4.40s + eval ppl=24.49645233154297, eval loss=3.198528289794922 +------------------------------------------------------------------ +iteration: 1601 , TFLOPS: 62.71429331600363, Tokens per sec: 155412.92738678196, Loss: 3.4931535720825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1602 , TFLOPS: 70.49696198590713, Tokens per sec: 174699.23768253147, Loss: 3.51155161857605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1603 , TFLOPS: 70.51945773114261, Tokens per sec: 174754.9846173359, Loss: 3.508125066757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1604 , TFLOPS: 68.55563241760821, Tokens per sec: 169888.40915718058, Loss: 3.5012593269348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1605 , TFLOPS: 67.91437435129065, Tokens per sec: 168299.30102843812, Loss: 3.504345655441284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1606 , TFLOPS: 70.57675184704742, Tokens per sec: 174896.9657480159, Loss: 3.5332722663879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1607 , TFLOPS: 70.51054094890128, Tokens per sec: 174732.88784867723, Loss: 3.498941659927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1608 , TFLOPS: 66.79643775840945, Tokens per sec: 165528.93100039643, Loss: 3.4917752742767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1609 , TFLOPS: 70.52814120857926, Tokens per sec: 174776.50323098793, Loss: 3.50467848777771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1610 , TFLOPS: 70.52482163132835, Tokens per sec: 174768.27695287857, Loss: 3.4998250007629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1611 , TFLOPS: 67.81792305523842, Tokens per sec: 168060.28409183337, Loss: 3.50917387008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1612 , TFLOPS: 68.8858001613401, Tokens per sec: 170706.6011971277, Loss: 3.505025863647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1613 , TFLOPS: 70.57961394221326, Tokens per sec: 174904.05833512268, Loss: 3.5075337886810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1614 , TFLOPS: 70.47627170716339, Tokens per sec: 174647.96489258212, Loss: 3.510324239730835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1615 , TFLOPS: 66.57591785719598, Tokens per sec: 164982.45839291695, Loss: 3.4859371185302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1616 , TFLOPS: 70.57407409891972, Tokens per sec: 174890.3299931772, Loss: 3.4841203689575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1617 , TFLOPS: 70.58923785163235, Tokens per sec: 174927.90744282433, Loss: 3.4940261840820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1618 , TFLOPS: 67.00213086689722, Tokens per sec: 166038.6611821956, Loss: 3.5021286010742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1619 , TFLOPS: 69.62514782677073, Tokens per sec: 172538.78615793365, Loss: 3.5047988891601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1620 , TFLOPS: 70.57721453972752, Tokens per sec: 174898.11235144702, Loss: 3.499095916748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1621 , TFLOPS: 67.06566292776782, Tokens per sec: 166196.10062766718, Loss: 3.513601303100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1622 , TFLOPS: 70.58086063384945, Tokens per sec: 174907.14777433156, Loss: 3.5064642429351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1623 , TFLOPS: 69.50174733797505, Tokens per sec: 172232.98615301194, Loss: 3.5128796100616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1624 , TFLOPS: 69.59216706453729, Tokens per sec: 172457.05619598832, Loss: 3.516293525695801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1625 , TFLOPS: 67.83114326500775, Tokens per sec: 168093.04522796808, Loss: 3.4871768951416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1626 , TFLOPS: 69.59402899080101, Tokens per sec: 172461.6702543779, Loss: 3.4983997344970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1627 , TFLOPS: 70.60391051772514, Tokens per sec: 174964.26792572986, Loss: 3.4932851791381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1628 , TFLOPS: 67.04880246081487, Tokens per sec: 166154.31853322402, Loss: 3.517825126647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1629 , TFLOPS: 70.5998677439895, Tokens per sec: 174954.2494870647, Loss: 3.51031231880188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1630 , TFLOPS: 69.5090781041207, Tokens per sec: 172251.1525990714, Loss: 3.5116615295410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1631 , TFLOPS: 68.7749547014597, Tokens per sec: 170431.91393690865, Loss: 3.517331123352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1632 , TFLOPS: 68.5886070929639, Tokens per sec: 169970.12403517234, Loss: 3.499040126800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1633 , TFLOPS: 70.6192895795985, Tokens per sec: 175002.37893519556, Loss: 3.497168779373169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1634 , TFLOPS: 69.50104069371082, Tokens per sec: 172231.23501068188, Loss: 3.5029656887054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1635 , TFLOPS: 67.82873853093686, Tokens per sec: 168087.0860320371, Loss: 3.508636713027954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1636 , TFLOPS: 70.59936315241185, Tokens per sec: 174952.9990535499, Loss: 3.5051069259643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1637 , TFLOPS: 70.55616685955161, Tokens per sec: 174845.95388137773, Loss: 3.5306787490844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1638 , TFLOPS: 65.9599929252387, Tokens per sec: 163456.1285617944, Loss: 3.5066051483154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1639 , TFLOPS: 70.61294264381466, Tokens per sec: 174986.65052915053, Loss: 3.5273983478546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1640 , TFLOPS: 70.6529187056761, Tokens per sec: 175085.7156141699, Loss: 3.501380443572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1641 , TFLOPS: 67.01325820133778, Tokens per sec: 166066.23594271636, Loss: 3.524415969848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1642 , TFLOPS: 69.97856868778811, Tokens per sec: 173414.60198405656, Loss: 3.502129316329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1643 , TFLOPS: 70.63328145975355, Tokens per sec: 175037.05235555262, Loss: 3.504680871963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1644 , TFLOPS: 69.65857860451176, Tokens per sec: 172621.63130788013, Loss: 3.4848103523254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1645 , TFLOPS: 67.71655017889778, Tokens per sec: 167809.070937117, Loss: 3.51863956451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1646 , TFLOPS: 70.59262564667925, Tokens per sec: 174936.30277214697, Loss: 3.5212225914001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1647 , TFLOPS: 70.5814005101559, Tokens per sec: 174908.48564729127, Loss: 3.5113162994384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1648 , TFLOPS: 66.15640016205629, Tokens per sec: 163942.84733067165, Loss: 3.5276169776916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1649 , TFLOPS: 70.55618220491675, Tokens per sec: 174845.99190888298, Loss: 3.5233421325683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1650 , TFLOPS: 70.59440590776092, Tokens per sec: 174940.7144552193, Loss: 3.5057878494262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1651 , TFLOPS: 69.58195036755964, Tokens per sec: 172431.73809541564, Loss: 3.4955735206604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1652 , TFLOPS: 67.26010131877007, Tokens per sec: 166677.94037972772, Loss: 3.490511894226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1653 , TFLOPS: 70.57508480555256, Tokens per sec: 174892.8346355519, Loss: 3.5020103454589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1654 , TFLOPS: 70.62129158860242, Tokens per sec: 175007.34013404677, Loss: 3.502553939819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1655 , TFLOPS: 67.71006487632478, Tokens per sec: 167792.99964292606, Loss: 3.490680694580078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1656 , TFLOPS: 69.57077468228611, Tokens per sec: 172404.04351620446, Loss: 3.497342348098755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1657 , TFLOPS: 70.61911930150494, Tokens per sec: 175001.95696732204, Loss: 3.511390447616577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1658 , TFLOPS: 67.8471069435555, Tokens per sec: 168132.60498195377, Loss: 3.5054829120635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1659 , TFLOPS: 69.96090100893284, Tokens per sec: 173370.81952959768, Loss: 3.5064852237701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1660 , TFLOPS: 69.60732611186978, Tokens per sec: 172494.62198518516, Loss: 3.4982738494873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1661 , TFLOPS: 68.70891123997919, Tokens per sec: 170268.25096407207, Loss: 3.4965009689331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1662 , TFLOPS: 68.92687568117981, Tokens per sec: 170808.39086013255, Loss: 3.486574411392212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1663 , TFLOPS: 69.51528974480338, Tokens per sec: 172266.54572895184, Loss: 3.507493019104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1664 , TFLOPS: 70.60511396402758, Tokens per sec: 174967.25019823786, Loss: 3.4955830574035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1665 , TFLOPS: 66.93099852071342, Tokens per sec: 165862.38739247734, Loss: 3.4989707469940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1666 , TFLOPS: 70.54297324499213, Tokens per sec: 174813.25865110272, Loss: 3.479987621307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1667 , TFLOPS: 69.66337574226756, Tokens per sec: 172633.5191436874, Loss: 3.521359920501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1668 , TFLOPS: 66.91576474624922, Tokens per sec: 165824.63642121624, Loss: 3.5009047985076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1669 , TFLOPS: 70.62803520821528, Tokens per sec: 175024.05156631846, Loss: 3.490086793899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1670 , TFLOPS: 70.58355056552935, Tokens per sec: 174913.8137213548, Loss: 3.5000016689300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1671 , TFLOPS: 68.60739126173891, Tokens per sec: 170016.67327464384, Loss: 3.4824378490448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1672 , TFLOPS: 67.88579569857949, Tokens per sec: 168228.4799788214, Loss: 3.5052785873413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1673 , TFLOPS: 70.58686843687451, Tokens per sec: 174922.0357720418, Loss: 3.5237069129943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1674 , TFLOPS: 70.59067294371913, Tokens per sec: 174931.4637591038, Loss: 3.4998626708984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1675 , TFLOPS: 66.8583819562341, Tokens per sec: 165682.4355463226, Loss: 3.4884450435638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1676 , TFLOPS: 70.60225154218703, Tokens per sec: 174960.1568015952, Loss: 3.494040012359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1677 , TFLOPS: 70.59091497387091, Tokens per sec: 174932.06353648173, Loss: 3.506256580352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1678 , TFLOPS: 67.25760140881407, Tokens per sec: 166671.74532746893, Loss: 3.5080254077911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1679 , TFLOPS: 69.07669828702133, Tokens per sec: 171179.6677817303, Loss: 3.4917149543762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1680 , TFLOPS: 70.61995769145165, Tokens per sec: 175004.03458996626, Loss: 3.475231885910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1681 , TFLOPS: 69.78474976411655, Tokens per sec: 172934.29734028142, Loss: 3.499432325363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1682 , TFLOPS: 67.03308332346533, Tokens per sec: 166115.36477926595, Loss: 3.4887349605560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1683 , TFLOPS: 70.57411676223475, Tokens per sec: 174890.43571757156, Loss: 3.4934680461883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1684 , TFLOPS: 70.58743171645622, Tokens per sec: 174923.43164089567, Loss: 3.506232261657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1685 , TFLOPS: 66.14896342719473, Tokens per sec: 163924.41828246074, Loss: 3.5039262771606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1686 , TFLOPS: 70.50597111514548, Tokens per sec: 174721.5632972216, Loss: 3.490025758743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1687 , TFLOPS: 70.6185893413089, Tokens per sec: 175000.64366757634, Loss: 3.493844747543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1688 , TFLOPS: 67.98770217668543, Tokens per sec: 168481.01545749322, Loss: 3.498408555984497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1689 , TFLOPS: 68.78675792800678, Tokens per sec: 170461.16363252501, Loss: 3.483057737350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1690 , TFLOPS: 70.56783898756258, Tokens per sec: 174874.87870037995, Loss: 3.500394821166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1691 , TFLOPS: 69.6237032522439, Tokens per sec: 172535.20634312317, Loss: 3.502253532409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1692 , TFLOPS: 67.70534160859965, Tokens per sec: 167781.29486519232, Loss: 3.508535146713257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1693 , TFLOPS: 69.66652717051737, Tokens per sec: 172641.32873004887, Loss: 3.495333671569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1694 , TFLOPS: 70.56346235463488, Tokens per sec: 174864.03292185918, Loss: 3.5266735553741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1695 , TFLOPS: 67.75231262751558, Tokens per sec: 167897.69422435085, Loss: 3.524440288543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1696 , TFLOPS: 70.5849442650261, Tokens per sec: 174917.2674622364, Loss: 3.473414897918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1697 , TFLOPS: 69.55181564575695, Tokens per sec: 172357.06093517487, Loss: 3.4961423873901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1698 , TFLOPS: 68.67808799806348, Tokens per sec: 170191.8675751445, Loss: 3.5025875568389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1699 , TFLOPS: 68.83754009124667, Tokens per sec: 170587.00742715056, Loss: 3.4901974201202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1700 , TFLOPS: 69.55332109992207, Tokens per sec: 172360.79161643557, Loss: 3.501532793045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1701 , TFLOPS: 70.4980205850486, Tokens per sec: 174701.86100781834, Loss: 3.5192718505859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1702 , TFLOPS: 67.17666589165202, Tokens per sec: 166471.17820015817, Loss: 3.4938693046569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1703 , TFLOPS: 70.59306258749294, Tokens per sec: 174937.38555961632, Loss: 3.503356695175171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1704 , TFLOPS: 69.57589873934117, Tokens per sec: 172416.7414940484, Loss: 3.511751890182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1705 , TFLOPS: 66.82545845124152, Tokens per sec: 165600.8474741275, Loss: 3.483985424041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1706 , TFLOPS: 70.5758564067893, Tokens per sec: 174894.7467484154, Loss: 3.4955005645751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1707 , TFLOPS: 70.5666614166201, Tokens per sec: 174871.96054986434, Loss: 3.5153818130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1708 , TFLOPS: 67.84340892452063, Tokens per sec: 168123.44088341494, Loss: 3.4820194244384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1709 , TFLOPS: 68.96507472508702, Tokens per sec: 170903.05229890667, Loss: 3.497553825378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1710 , TFLOPS: 70.58792423230575, Tokens per sec: 174924.6521494259, Loss: 3.5015711784362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1711 , TFLOPS: 70.59583918614432, Tokens per sec: 174944.26627694204, Loss: 3.4728736877441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1712 , TFLOPS: 66.96980956887592, Tokens per sec: 165958.56544521387, Loss: 3.5069377422332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1713 , TFLOPS: 70.57778497568407, Tokens per sec: 174899.52595458605, Loss: 3.5006604194641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1714 , TFLOPS: 70.52518113945696, Tokens per sec: 174769.16785362436, Loss: 3.495415449142456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1715 , TFLOPS: 67.25904955600065, Tokens per sec: 166675.33399572386, Loss: 3.532008409500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1716 , TFLOPS: 70.60663220676003, Tokens per sec: 174971.01257097712, Loss: 3.4994122982025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1717 , TFLOPS: 70.64699158660568, Tokens per sec: 175071.02756018733, Loss: 3.4897985458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1718 , TFLOPS: 69.61967000087841, Tokens per sec: 172525.21150194018, Loss: 3.492447853088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1719 , TFLOPS: 67.29536730576972, Tokens per sec: 166765.333380382, Loss: 3.5210609436035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1720 , TFLOPS: 70.59129967484792, Tokens per sec: 174933.0168678811, Loss: 3.495192766189575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1721 , TFLOPS: 70.60366031058085, Tokens per sec: 174963.64788486372, Loss: 3.5098869800567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1722 , TFLOPS: 66.00506595234843, Tokens per sec: 163567.82448817618, Loss: 3.4928741455078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1723 , TFLOPS: 70.60049296257426, Tokens per sec: 174955.79884759118, Loss: 3.4901680946350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1724 , TFLOPS: 70.59117992417357, Tokens per sec: 174932.72011251844, Loss: 3.5010604858398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1725 , TFLOPS: 67.98582763160756, Tokens per sec: 168476.37012829207, Loss: 3.4807047843933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1726 , TFLOPS: 68.83828901551955, Tokens per sec: 170588.86334399958, Loss: 3.5026276111602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1727 , TFLOPS: 70.61252749454556, Tokens per sec: 174985.62174352922, Loss: 3.4979324340820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1728 , TFLOPS: 68.14839949392763, Tokens per sec: 168879.24111189024, Loss: 3.5210020542144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1729 , TFLOPS: 68.66333927050982, Tokens per sec: 170155.3186035601, Loss: 3.4785289764404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1730 , TFLOPS: 69.53847513049094, Tokens per sec: 172324.00166876666, Loss: 3.495501756668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1731 , TFLOPS: 70.59236905316773, Tokens per sec: 174935.6669051593, Loss: 3.4858760833740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1732 , TFLOPS: 66.72357424923135, Tokens per sec: 165348.36719807494, Loss: 3.4912614822387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1733 , TFLOPS: 70.59268886115127, Tokens per sec: 174936.4594245723, Loss: 3.5182783603668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1734 , TFLOPS: 69.64782945722368, Tokens per sec: 172594.99373678394, Loss: 3.5051562786102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1735 , TFLOPS: 67.28898810504614, Tokens per sec: 166749.52501826207, Loss: 3.473102331161499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1736 , TFLOPS: 70.60137250504252, Tokens per sec: 174957.97845071717, Loss: 3.494523286819458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1737 , TFLOPS: 69.10079637833294, Tokens per sec: 171239.3855066244, Loss: 3.488179922103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1738 , TFLOPS: 69.55452086089915, Tokens per sec: 172363.76475630133, Loss: 3.4709465503692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1739 , TFLOPS: 67.96813661849542, Tokens per sec: 168432.5298489742, Loss: 3.509674072265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1740 , TFLOPS: 70.64704938369027, Tokens per sec: 175071.17078772985, Loss: 3.5085132122039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1741 , TFLOPS: 68.85870168717561, Tokens per sec: 170639.44819300473, Loss: 3.510956287384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1742 , TFLOPS: 66.97816467073326, Tokens per sec: 165979.27030800685, Loss: 3.512178897857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1743 , TFLOPS: 70.56075314273522, Tokens per sec: 174857.31919632753, Loss: 3.503809690475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1744 , TFLOPS: 70.61249667947652, Tokens per sec: 174985.54538039357, Loss: 3.4853711128234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1745 , TFLOPS: 67.71754172381313, Tokens per sec: 167811.52809464498, Loss: 3.5031731128692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1746 , TFLOPS: 69.08509140371582, Tokens per sec: 171200.4668494774, Loss: 3.479556083679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1747 , TFLOPS: 70.58208945347286, Tokens per sec: 174910.19292472402, Loss: 3.5188369750976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1748 , TFLOPS: 69.71184119728424, Tokens per sec: 172753.62188012927, Loss: 3.484917163848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1749 , TFLOPS: 67.08542522795805, Tokens per sec: 166245.07378453566, Loss: 3.495950222015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1750 , TFLOPS: 70.60308367532258, Tokens per sec: 174962.2189191721, Loss: 3.5074284076690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1751 , TFLOPS: 70.58806745309313, Tokens per sec: 174925.00706631408, Loss: 3.501800775527954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1752 , TFLOPS: 66.75177482905006, Tokens per sec: 165418.25134141606, Loss: 3.4890689849853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1753 , TFLOPS: 70.56606402577987, Tokens per sec: 174870.48014955423, Loss: 3.4954023361206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1754 , TFLOPS: 70.55331156409414, Tokens per sec: 174838.87814469737, Loss: 3.5176150798797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1755 , TFLOPS: 67.70834133165334, Tokens per sec: 167788.72850936613, Loss: 3.5077600479125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1756 , TFLOPS: 69.64197292900508, Tokens per sec: 172580.48061470842, Loss: 3.497013807296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1757 , TFLOPS: 70.62162545931571, Tokens per sec: 175008.1675024532, Loss: 3.491171360015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1758 , TFLOPS: 69.53917905623491, Tokens per sec: 172325.74607430407, Loss: 3.5127763748168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1759 , TFLOPS: 67.09427199584034, Tokens per sec: 166266.99704989992, Loss: 3.5149102210998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1760 , TFLOPS: 70.52476125330874, Tokens per sec: 174768.12732949478, Loss: 3.497983932495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1761 , TFLOPS: 70.55295517258703, Tokens per sec: 174837.99496728298, Loss: 3.5025906562805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1762 , TFLOPS: 66.59057982335005, Tokens per sec: 165018.79236019566, Loss: 3.494464874267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1763 , TFLOPS: 69.62105212515954, Tokens per sec: 172528.63655816237, Loss: 3.518216133117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1764 , TFLOPS: 70.6246321903433, Tokens per sec: 175015.61851315835, Loss: 3.5016562938690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1765 , TFLOPS: 68.71336320034774, Tokens per sec: 170279.28341228925, Loss: 3.499584197998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1766 , TFLOPS: 68.8725782815278, Tokens per sec: 170673.8359224425, Loss: 3.5074098110198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1767 , TFLOPS: 69.60833907974857, Tokens per sec: 172497.13223117645, Loss: 3.4929299354553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1768 , TFLOPS: 70.53867798207837, Tokens per sec: 174802.61451644014, Loss: 3.495887041091919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1769 , TFLOPS: 67.16102156534106, Tokens per sec: 166432.40983619477, Loss: 3.5237860679626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1770 , TFLOPS: 70.51689291323794, Tokens per sec: 174748.62871603013, Loss: 3.5120317935943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1771 , TFLOPS: 69.56419177862232, Tokens per sec: 172387.7303557589, Loss: 3.502493143081665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1772 , TFLOPS: 67.01871307827913, Tokens per sec: 166079.75372868133, Loss: 3.491464614868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1773 , TFLOPS: 70.5819930888508, Tokens per sec: 174909.95412257512, Loss: 3.488550901412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1774 , TFLOPS: 69.70159826300109, Tokens per sec: 172728.23876062327, Loss: 3.502516746520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1775 , TFLOPS: 68.54014029486106, Tokens per sec: 169850.01797041492, Loss: 3.4965975284576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1776 , TFLOPS: 69.83117638608125, Tokens per sec: 173049.34762382455, Loss: 3.503873586654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1777 , TFLOPS: 70.5624682223386, Tokens per sec: 174861.5693525163, Loss: 3.4810593128204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1778 , TFLOPS: 68.58055143629294, Tokens per sec: 169950.16123053784, Loss: 3.487938404083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1779 , TFLOPS: 68.02237186179377, Tokens per sec: 168566.93075637278, Loss: 3.4982447624206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1780 , TFLOPS: 70.58357898400288, Tokens per sec: 174913.88414546277, Loss: 3.508045196533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1781 , TFLOPS: 70.64487787778684, Tokens per sec: 175065.78955689, Loss: 3.50848650932312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1782 , TFLOPS: 66.0125972425774, Tokens per sec: 163586.48785500365, Loss: 3.487827777862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1783 , TFLOPS: 70.58047954690588, Tokens per sec: 174906.20339890523, Loss: 3.501646041870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1784 , TFLOPS: 70.5817129168885, Tokens per sec: 174909.2598255891, Loss: 3.5089759826660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1785 , TFLOPS: 69.49796983071917, Tokens per sec: 172223.62507390507, Loss: 3.5022666454315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1786 , TFLOPS: 67.84670407310682, Tokens per sec: 168131.6066246021, Loss: 3.493736743927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1787 , TFLOPS: 70.56947904801433, Tokens per sec: 174878.94295084695, Loss: 3.533102035522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1788 , TFLOPS: 70.59357563582824, Tokens per sec: 174938.65694990812, Loss: 3.487175464630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1789 , TFLOPS: 66.91323811159909, Tokens per sec: 165818.37514222143, Loss: 3.5046286582946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1790 , TFLOPS: 70.52604036453076, Tokens per sec: 174771.29710800867, Loss: 3.502823829650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1791 , TFLOPS: 70.56721414027538, Tokens per sec: 174873.33025996966, Loss: 3.4985690116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1792 , TFLOPS: 67.84817097887945, Tokens per sec: 168135.2417786997, Loss: 3.515198230743408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1793 , TFLOPS: 68.85923376436708, Tokens per sec: 170640.76673889832, Loss: 3.527204751968384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1794 , TFLOPS: 70.55255825339881, Tokens per sec: 174837.0113578103, Loss: 3.496169328689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1795 , TFLOPS: 69.55814163960308, Tokens per sec: 172372.73744479136, Loss: 3.496206045150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1796 , TFLOPS: 67.93449409771918, Tokens per sec: 168349.1599764611, Loss: 3.4888598918914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1797 , TFLOPS: 70.57761943257992, Tokens per sec: 174899.11572053787, Loss: 3.4888505935668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1798 , TFLOPS: 70.58491174248749, Tokens per sec: 174917.1868678031, Loss: 3.499453544616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1799 , TFLOPS: 67.3120945698758, Tokens per sec: 166806.78538349713, Loss: 3.511195182800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1800 , TFLOPS: 69.67002786545366, Tokens per sec: 172650.00383773434, Loss: 3.502493381500244 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/optimizer.pt +[rank0]:[2024-08-25 00:17:05,160] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006417646000045352, 'preprocessing_with_comm': 0.0012551610016089398, 'state_converting': 0.9705206520011416, : 0.9795591490001243}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800, took 4.30s + eval ppl=23.057601928710938, eval loss=3.137995481491089 +------------------------------------------------------------------ +iteration: 1801 , TFLOPS: 66.29904855195751, Tokens per sec: 164296.34575486425, Loss: 3.48416805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1802 , TFLOPS: 67.70691507475277, Tokens per sec: 167785.19408765135, Loss: 3.514606475830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1803 , TFLOPS: 70.5987294246066, Tokens per sec: 174951.4286062376, Loss: 3.47674822807312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1804 , TFLOPS: 69.18419166052506, Tokens per sec: 171446.0482026452, Loss: 3.5146255493164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1805 , TFLOPS: 69.57057523217193, Tokens per sec: 172403.54925685027, Loss: 3.5015807151794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1806 , TFLOPS: 68.89552823622357, Tokens per sec: 170730.70843832483, Loss: 3.488892078399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1807 , TFLOPS: 69.53927147096442, Tokens per sec: 172325.9750881841, Loss: 3.5111031532287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1808 , TFLOPS: 70.54312225197636, Tokens per sec: 174813.62790682408, Loss: 3.4969773292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1809 , TFLOPS: 67.99388932634685, Tokens per sec: 168496.347895925, Loss: 3.5140576362609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1810 , TFLOPS: 70.4782412032323, Tokens per sec: 174652.8455207988, Loss: 3.5081467628479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1811 , TFLOPS: 69.58220303351582, Tokens per sec: 172432.3642294885, Loss: 3.5040407180786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1812 , TFLOPS: 68.6055525943248, Tokens per sec: 170012.11685424575, Loss: 3.5014216899871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1813 , TFLOPS: 69.85131953914029, Tokens per sec: 173099.26457605636, Loss: 3.4971256256103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1814 , TFLOPS: 70.54338021710363, Tokens per sec: 174814.26717282672, Loss: 3.4930527210235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1815 , TFLOPS: 67.69703347393637, Tokens per sec: 167760.70639228012, Loss: 3.4923794269561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1816 , TFLOPS: 69.5864886203846, Tokens per sec: 172442.9843858457, Loss: 3.5025675296783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1817 , TFLOPS: 70.63902413263257, Tokens per sec: 175051.2833315545, Loss: 3.4766199588775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1818 , TFLOPS: 69.63546310688503, Tokens per sec: 172564.3485583788, Loss: 3.481935977935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1819 , TFLOPS: 67.65694859193985, Tokens per sec: 167661.37163897845, Loss: 3.5088212490081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1820 , TFLOPS: 70.48348976177745, Tokens per sec: 174665.85202705013, Loss: 3.508641481399536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1821 , TFLOPS: 70.56518510890663, Tokens per sec: 174868.30209672163, Loss: 3.469926595687866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1822 , TFLOPS: 66.7666427161497, Tokens per sec: 165455.09560348192, Loss: 3.48889422416687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1823 , TFLOPS: 70.59083641154251, Tokens per sec: 174931.86885037762, Loss: 3.527259588241577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1824 , TFLOPS: 70.59667650567826, Tokens per sec: 174946.34124698548, Loss: 3.4727447032928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1825 , TFLOPS: 69.47349274396298, Tokens per sec: 172162.96815654956, Loss: 3.509643793106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1826 , TFLOPS: 67.74198558161854, Tokens per sec: 167872.10266699997, Loss: 3.51408052444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1827 , TFLOPS: 70.63213496707571, Tokens per sec: 175034.21122040035, Loss: 3.5078585147857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1828 , TFLOPS: 70.55950976613462, Tokens per sec: 174854.23797214203, Loss: 3.4958252906799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1829 , TFLOPS: 67.65146840841979, Tokens per sec: 167647.79114052368, Loss: 3.4956307411193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1830 , TFLOPS: 69.522227540794, Tokens per sec: 172283.7383516779, Loss: 3.5161938667297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1831 , TFLOPS: 70.61660411934756, Tokens per sec: 174995.72406886588, Loss: 3.4812099933624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1832 , TFLOPS: 68.79161610146602, Tokens per sec: 170473.20272152903, Loss: 3.499668598175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1833 , TFLOPS: 69.54875292054886, Tokens per sec: 172349.47116472226, Loss: 3.4973835945129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1834 , TFLOPS: 69.65930936187209, Tokens per sec: 172623.44220511863, Loss: 3.500296115875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1835 , TFLOPS: 68.6379943698891, Tokens per sec: 170092.51114785066, Loss: 3.493316650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1836 , TFLOPS: 69.59602712967606, Tokens per sec: 172466.6218626235, Loss: 3.5166265964508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1837 , TFLOPS: 69.54061352148415, Tokens per sec: 172329.30083721102, Loss: 3.4924564361572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1838 , TFLOPS: 69.61139281381276, Tokens per sec: 172504.69972058406, Loss: 3.4981110095977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1839 , TFLOPS: 68.71995509006564, Tokens per sec: 170295.6188411668, Loss: 3.5118720531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1840 , TFLOPS: 70.57776532122116, Tokens per sec: 174899.47724866186, Loss: 3.468903064727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1841 , TFLOPS: 69.56522182736528, Tokens per sec: 172390.28293001314, Loss: 3.519862651824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1842 , TFLOPS: 68.62309848281315, Tokens per sec: 170055.59749875902, Loss: 3.4910805225372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1843 , TFLOPS: 69.86197769775457, Tokens per sec: 173125.6766671956, Loss: 3.4841747283935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1844 , TFLOPS: 69.7028232144093, Tokens per sec: 172731.2743251524, Loss: 3.498650550842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1845 , TFLOPS: 69.67976009515871, Tokens per sec: 172674.12137503712, Loss: 3.5131659507751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1846 , TFLOPS: 68.78590525022153, Tokens per sec: 170459.05060304297, Loss: 3.4908688068389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1847 , TFLOPS: 70.59264169498374, Tokens per sec: 174936.34254161338, Loss: 3.51381778717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1848 , TFLOPS: 69.54648180912359, Tokens per sec: 172343.84310042684, Loss: 3.5011203289031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1849 , TFLOPS: 67.7224705436102, Tokens per sec: 167823.74225305644, Loss: 3.48606538772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1850 , TFLOPS: 70.57812965969123, Tokens per sec: 174900.3801195259, Loss: 3.501889228820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1851 , TFLOPS: 70.579592019174, Tokens per sec: 174904.0040074164, Loss: 3.489323139190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1852 , TFLOPS: 67.78079373999381, Tokens per sec: 167968.27355852548, Loss: 3.496333122253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1853 , TFLOPS: 69.60434207934658, Tokens per sec: 172487.22722387436, Loss: 3.4962263107299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1854 , TFLOPS: 70.62759904038145, Tokens per sec: 175022.9706943784, Loss: 3.51179838180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1855 , TFLOPS: 69.56323870083936, Tokens per sec: 172385.36852402252, Loss: 3.5300323963165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1856 , TFLOPS: 68.06738923518398, Tokens per sec: 168678.48876670675, Loss: 3.496626853942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1857 , TFLOPS: 70.60712333328645, Tokens per sec: 174972.22963661165, Loss: 3.477393865585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1858 , TFLOPS: 70.58702748271702, Tokens per sec: 174922.4299051599, Loss: 3.5059738159179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1859 , TFLOPS: 67.76672792984543, Tokens per sec: 167933.4169315009, Loss: 3.5132007598876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1860 , TFLOPS: 70.62602890704582, Tokens per sec: 175019.07973100824, Loss: 3.5019454956054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1861 , TFLOPS: 70.65721771832465, Tokens per sec: 175096.36904108836, Loss: 3.4858081340789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1862 , TFLOPS: 68.65745434381859, Tokens per sec: 170140.73510694102, Loss: 3.4813742637634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1863 , TFLOPS: 68.6104627995029, Tokens per sec: 170024.2848835808, Loss: 3.5022709369659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1864 , TFLOPS: 70.62290048421376, Tokens per sec: 175011.32715460614, Loss: 3.4783260822296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1865 , TFLOPS: 69.59140212261987, Tokens per sec: 172455.16058565164, Loss: 3.4934139251708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1866 , TFLOPS: 68.69524362476939, Tokens per sec: 170234.38110797005, Loss: 3.505018472671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1867 , TFLOPS: 69.54070822349817, Tokens per sec: 172329.53551923408, Loss: 3.5205721855163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1868 , TFLOPS: 70.58002096084424, Tokens per sec: 174905.06697212707, Loss: 3.4857282638549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1869 , TFLOPS: 67.60277906693024, Tokens per sec: 167527.13358873886, Loss: 3.515252113342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1870 , TFLOPS: 69.59931933459535, Tokens per sec: 172474.78030907977, Loss: 3.4970321655273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1871 , TFLOPS: 70.58223192605463, Tokens per sec: 174910.54598747633, Loss: 3.4950404167175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1872 , TFLOPS: 69.59647234106923, Tokens per sec: 172467.72514549986, Loss: 3.505459785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1873 , TFLOPS: 68.87513042848275, Tokens per sec: 170680.1604237391, Loss: 3.4958014488220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1874 , TFLOPS: 69.60922729733758, Tokens per sec: 172499.33333220723, Loss: 3.512510299682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1875 , TFLOPS: 70.58101334828456, Tokens per sec: 174907.52621752466, Loss: 3.502063512802124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1876 , TFLOPS: 67.90790258057963, Tokens per sec: 168283.26326769183, Loss: 3.4935967922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1877 , TFLOPS: 70.6155237497766, Tokens per sec: 174993.0467940565, Loss: 3.498932123184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1878 , TFLOPS: 69.56891162328117, Tokens per sec: 172399.42665075732, Loss: 3.4904751777648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1879 , TFLOPS: 68.68153973934322, Tokens per sec: 170200.42137027642, Loss: 3.504373073577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1880 , TFLOPS: 69.93358164618373, Tokens per sec: 173303.11913922915, Loss: 3.502673625946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1881 , TFLOPS: 69.44015665306667, Tokens per sec: 172080.35765103492, Loss: 3.4917118549346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1882 , TFLOPS: 68.65926392689141, Tokens per sec: 170145.21945313754, Loss: 3.504472255706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1883 , TFLOPS: 69.49525915158921, Tokens per sec: 172216.9077124151, Loss: 3.484097957611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1884 , TFLOPS: 70.56071250800218, Tokens per sec: 174857.21849898278, Loss: 3.5283002853393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1885 , TFLOPS: 68.72742077439584, Tokens per sec: 170314.11962935003, Loss: 3.525714874267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1886 , TFLOPS: 68.69754583745481, Tokens per sec: 170240.08624461418, Loss: 3.5136494636535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1887 , TFLOPS: 70.6164013308758, Tokens per sec: 174995.22153669386, Loss: 3.5057570934295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1888 , TFLOPS: 70.6456620162455, Tokens per sec: 175067.73273837054, Loss: 3.4896490573883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1889 , TFLOPS: 66.85060222659139, Tokens per sec: 165663.15651926002, Loss: 3.489528179168701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1890 , TFLOPS: 70.61599097277293, Tokens per sec: 174994.2046241103, Loss: 3.5090456008911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1891 , TFLOPS: 70.58759909368865, Tokens per sec: 174923.84642011573, Loss: 3.4757213592529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1892 , TFLOPS: 69.60504807131969, Tokens per sec: 172488.97674975512, Loss: 3.4891741275787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1893 , TFLOPS: 67.84498057271402, Tokens per sec: 168127.33560076947, Loss: 3.5012388229370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1894 , TFLOPS: 70.59335277054463, Tokens per sec: 174938.10466518466, Loss: 3.5131640434265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1895 , TFLOPS: 70.606537300565, Tokens per sec: 174970.77738297105, Loss: 3.4822423458099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1896 , TFLOPS: 66.601444157165, Tokens per sec: 165045.71537619428, Loss: 3.5032284259796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1897 , TFLOPS: 70.61602348630038, Tokens per sec: 174994.28519621294, Loss: 3.486133575439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1898 , TFLOPS: 70.60875779985341, Tokens per sec: 174976.2800248159, Loss: 3.49513578414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1899 , TFLOPS: 68.68850694699469, Tokens per sec: 170217.68687833805, Loss: 3.5229923725128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1900 , TFLOPS: 68.47301069801814, Tokens per sec: 169683.6634345026, Loss: 3.4737021923065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1901 , TFLOPS: 70.5825880111441, Tokens per sec: 174911.42840555403, Loss: 3.499666213989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1902 , TFLOPS: 68.77173844029467, Tokens per sec: 170423.9436874413, Loss: 3.5136263370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1903 , TFLOPS: 69.55724256200585, Tokens per sec: 172370.50943146387, Loss: 3.4987335205078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1904 , TFLOPS: 69.68819948973416, Tokens per sec: 172695.03512447112, Loss: 3.5110299587249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1905 , TFLOPS: 69.56962770214044, Tokens per sec: 172401.20117305318, Loss: 3.516716480255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1906 , TFLOPS: 68.48672411068831, Tokens per sec: 169717.64678175643, Loss: 3.4829583168029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1907 , TFLOPS: 70.58143750839726, Tokens per sec: 174908.57733300902, Loss: 3.5232748985290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1908 , TFLOPS: 69.59118090247091, Tokens per sec: 172454.61237775313, Loss: 3.5005202293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1909 , TFLOPS: 67.78337690223415, Tokens per sec: 167974.67491911585, Loss: 3.510575532913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1910 , TFLOPS: 70.54229705834537, Tokens per sec: 174811.58298610392, Loss: 3.5022084712982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1911 , TFLOPS: 69.54437323030633, Tokens per sec: 172338.61780983853, Loss: 3.503960132598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1912 , TFLOPS: 69.51585046424705, Tokens per sec: 172267.93525350158, Loss: 3.5166447162628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1913 , TFLOPS: 68.64265029156341, Tokens per sec: 170104.0490346517, Loss: 3.494044065475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1914 , TFLOPS: 70.5903946605702, Tokens per sec: 174930.77414280534, Loss: 3.4915895462036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1915 , TFLOPS: 69.62353905987077, Tokens per sec: 172534.79945633525, Loss: 3.5115208625793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1916 , TFLOPS: 67.80488406083367, Tokens per sec: 168027.97202733456, Loss: 3.5011088848114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1917 , TFLOPS: 70.6006468823388, Tokens per sec: 174956.18027772335, Loss: 3.5065717697143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1918 , TFLOPS: 69.68927641974773, Tokens per sec: 172697.7038756784, Loss: 3.5304431915283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1919 , TFLOPS: 68.62504123853638, Tokens per sec: 170060.4118614534, Loss: 3.4844167232513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1920 , TFLOPS: 69.55350361076769, Tokens per sec: 172361.24389841623, Loss: 3.510540723800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1921 , TFLOPS: 70.56594622301803, Tokens per sec: 174870.18822133334, Loss: 3.504514455795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1922 , TFLOPS: 68.60995501851413, Tokens per sec: 170023.02654635394, Loss: 3.5152032375335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1923 , TFLOPS: 68.99978906018391, Tokens per sec: 170989.07824537647, Loss: 3.4865198135375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1924 , TFLOPS: 70.60862721715611, Tokens per sec: 174975.95642650715, Loss: 3.530109405517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1925 , TFLOPS: 70.55105420921078, Tokens per sec: 174833.28417062823, Loss: 3.479249954223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1926 , TFLOPS: 66.98417681422227, Tokens per sec: 165994.16906186487, Loss: 3.5200417041778564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1927 , TFLOPS: 70.61308051315153, Tokens per sec: 174986.99218455484, Loss: 3.5107369422912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1928 , TFLOPS: 70.5958592931287, Tokens per sec: 174944.31610426426, Loss: 3.4928314685821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1929 , TFLOPS: 68.73963191208043, Tokens per sec: 170344.3801737003, Loss: 3.5050179958343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1930 , TFLOPS: 68.96984711332985, Tokens per sec: 170914.87880269435, Loss: 3.5153777599334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1931 , TFLOPS: 70.5651117766891, Tokens per sec: 174868.12037140838, Loss: 3.4958484172821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1932 , TFLOPS: 69.4979606440021, Tokens per sec: 172223.60230820824, Loss: 3.4702775478363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1933 , TFLOPS: 68.59082049878599, Tokens per sec: 169975.60909862624, Loss: 3.4865028858184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1934 , TFLOPS: 70.64423811992398, Tokens per sec: 175064.20416643072, Loss: 3.512249231338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1935 , TFLOPS: 70.6057238991858, Tokens per sec: 174968.76168474986, Loss: 3.4994733333587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1936 , TFLOPS: 67.92134262623627, Tokens per sec: 168316.56918137224, Loss: 3.4880542755126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1937 , TFLOPS: 69.65766474864147, Tokens per sec: 172619.366672364, Loss: 3.5164780616760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1938 , TFLOPS: 70.55265133157576, Tokens per sec: 174837.2420157862, Loss: 3.4882538318634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1939 , TFLOPS: 69.53608523214375, Tokens per sec: 172318.07923739328, Loss: 3.524033546447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1940 , TFLOPS: 68.88206305233999, Tokens per sec: 170697.34022934883, Loss: 3.49171781539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1941 , TFLOPS: 69.5289444361107, Tokens per sec: 172300.3835581998, Loss: 3.4655869007110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1942 , TFLOPS: 70.62367701276402, Tokens per sec: 175013.25147789536, Loss: 3.4703598022460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1943 , TFLOPS: 68.1291179429002, Tokens per sec: 168831.45930440607, Loss: 3.5030879974365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1944 , TFLOPS: 69.69651920315886, Tokens per sec: 172715.6522908296, Loss: 3.5007336139678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1945 , TFLOPS: 70.606167367477, Tokens per sec: 174969.8606480275, Loss: 3.4995291233062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1946 , TFLOPS: 68.57047788079126, Tokens per sec: 169925.19785031094, Loss: 3.524052143096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1947 , TFLOPS: 69.81686519711832, Tokens per sec: 173013.8829210668, Loss: 3.5068647861480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1948 , TFLOPS: 69.56864031391098, Tokens per sec: 172398.7543162508, Loss: 3.5072927474975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1949 , TFLOPS: 68.7212092739267, Tokens per sec: 170298.72684693427, Loss: 3.501382350921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1950 , TFLOPS: 69.66699457081448, Tokens per sec: 172642.48699947348, Loss: 3.49006986618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1951 , TFLOPS: 70.57723238748889, Tokens per sec: 174898.15658016579, Loss: 3.4962165355682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1952 , TFLOPS: 68.70460708210413, Tokens per sec: 170257.5847867155, Loss: 3.5211706161499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1953 , TFLOPS: 68.59040735264024, Tokens per sec: 169974.58527696528, Loss: 3.49929141998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1954 , TFLOPS: 70.58958631809841, Tokens per sec: 174928.770981114, Loss: 3.4964680671691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1955 , TFLOPS: 69.58003564750032, Tokens per sec: 172426.9932081848, Loss: 3.4874916076660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1956 , TFLOPS: 67.90601329810572, Tokens per sec: 168278.58141759987, Loss: 3.5140552520751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1957 , TFLOPS: 70.58983270781148, Tokens per sec: 174929.38156196504, Loss: 3.4926435947418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1958 , TFLOPS: 70.57830691900853, Tokens per sec: 174900.81938764086, Loss: 3.5223214626312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1959 , TFLOPS: 68.20092456534704, Tokens per sec: 169009.4040249818, Loss: 3.5276267528533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1960 , TFLOPS: 68.62723009597936, Tokens per sec: 170065.83609131965, Loss: 3.4906258583068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1961 , TFLOPS: 70.53205334624313, Tokens per sec: 174786.19793907658, Loss: 3.4891133308410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1962 , TFLOPS: 70.62017197368934, Tokens per sec: 175004.5656049564, Loss: 3.491039752960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1963 , TFLOPS: 67.64650608785747, Tokens per sec: 167635.49396352554, Loss: 3.510465621948242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1964 , TFLOPS: 70.56445608225832, Tokens per sec: 174866.4954883782, Loss: 3.5111844539642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1965 , TFLOPS: 70.60876535210065, Tokens per sec: 174976.29874011653, Loss: 3.5206425189971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1966 , TFLOPS: 68.80077179062525, Tokens per sec: 170495.89152784803, Loss: 3.502944231033325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1967 , TFLOPS: 68.56074414036843, Tokens per sec: 169901.07656928312, Loss: 3.485586166381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1968 , TFLOPS: 70.60317475904621, Tokens per sec: 174962.44463467298, Loss: 3.510878562927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1969 , TFLOPS: 69.55077230716465, Tokens per sec: 172354.47542720995, Loss: 3.497300148010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1970 , TFLOPS: 67.58186418891765, Tokens per sec: 167475.30421705966, Loss: 3.501157283782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1971 , TFLOPS: 70.59599067581078, Tokens per sec: 174944.64168502347, Loss: 3.4868273735046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1972 , TFLOPS: 70.58437325223152, Tokens per sec: 174915.85242962916, Loss: 3.5081472396850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1973 , TFLOPS: 67.97113672340403, Tokens per sec: 168439.96443942658, Loss: 3.5308244228363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1974 , TFLOPS: 69.55084768522636, Tokens per sec: 172354.66222235048, Loss: 3.499361515045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1975 , TFLOPS: 70.59118905391584, Tokens per sec: 174932.74273702552, Loss: 3.503772735595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1976 , TFLOPS: 67.68626686968032, Tokens per sec: 167734.02556089425, Loss: 3.5154221057891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1977 , TFLOPS: 70.52618704526084, Tokens per sec: 174771.66059901603, Loss: 3.5081920623779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1978 , TFLOPS: 69.64516818298941, Tokens per sec: 172588.39880606785, Loss: 3.4968628883361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1979 , TFLOPS: 69.632982546209, Tokens per sec: 172558.2014557673, Loss: 3.4780001640319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1980 , TFLOPS: 68.69107034404779, Tokens per sec: 170224.03925861712, Loss: 3.4997849464416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1981 , TFLOPS: 70.56993805208401, Tokens per sec: 174880.0804134951, Loss: 3.4974327087402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1982 , TFLOPS: 69.52086095850542, Tokens per sec: 172280.35181022531, Loss: 3.4952526092529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1983 , TFLOPS: 67.7563022924259, Tokens per sec: 167907.5810534965, Loss: 3.506411075592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1984 , TFLOPS: 70.58793983615243, Tokens per sec: 174924.69081747686, Loss: 3.521876335144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1985 , TFLOPS: 69.66740715626521, Tokens per sec: 172643.50943167045, Loss: 3.5237069129943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1986 , TFLOPS: 68.73977146140079, Tokens per sec: 170344.7259922888, Loss: 3.524625539779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1987 , TFLOPS: 69.53693126579648, Tokens per sec: 172320.17580198304, Loss: 3.501781940460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1988 , TFLOPS: 70.58933463133073, Tokens per sec: 174928.14727357807, Loss: 3.484053134918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1989 , TFLOPS: 68.51327281269181, Tokens per sec: 169783.43738990204, Loss: 3.493074893951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1990 , TFLOPS: 68.74534681128978, Tokens per sec: 170358.54232348924, Loss: 3.4754862785339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1991 , TFLOPS: 70.60870086584598, Tokens per sec: 174976.13893607372, Loss: 3.486015796661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1992 , TFLOPS: 69.76009770862603, Tokens per sec: 172873.2068311278, Loss: 3.493335723876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1993 , TFLOPS: 67.76713098733892, Tokens per sec: 167934.4157523702, Loss: 3.5041587352752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1994 , TFLOPS: 70.5583522296174, Tokens per sec: 174851.36946913914, Loss: 3.472834825515747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1995 , TFLOPS: 70.59144229754253, Tokens per sec: 174933.3703026295, Loss: 3.492654323577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1996 , TFLOPS: 67.69508444498945, Tokens per sec: 167755.87648384637, Loss: 3.4825096130371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1997 , TFLOPS: 69.87053340885916, Tokens per sec: 173146.87866180215, Loss: 3.506840944290161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1998 , TFLOPS: 70.62612912281871, Tokens per sec: 175019.32807673316, Loss: 3.4879555702209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1999 , TFLOPS: 69.52990114912132, Tokens per sec: 172302.7543984307, Loss: 3.5150833129882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2000 , TFLOPS: 67.80017079885653, Tokens per sec: 168016.29204494652, Loss: 3.518500328063965 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/model.pt +[rank0]:[2024-08-25 00:42:36,154] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006335953999951016, 'preprocessing_with_comm': 0.0012989419992663898, 'state_converting': 0.9989286260024528, : 1.0079901170029189}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000, took 4.26s + eval ppl=25.606250762939453, eval loss=3.2428364753723145 +------------------------------------------------------------------ +iteration: 2001 , TFLOPS: 65.68001781369865, Tokens per sec: 162762.31939359359, Loss: 3.508151054382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2002 , TFLOPS: 70.57336406080915, Tokens per sec: 174888.57044052205, Loss: 3.503185749053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2003 , TFLOPS: 66.94613230869055, Tokens per sec: 165899.89058620902, Loss: 3.519651174545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2004 , TFLOPS: 69.92446165623579, Tokens per sec: 173280.51879948835, Loss: 3.4938178062438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2005 , TFLOPS: 70.5066767086655, Tokens per sec: 174723.31183569174, Loss: 3.4966282844543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2006 , TFLOPS: 68.04580980039086, Tokens per sec: 168625.0125501184, Loss: 3.4913735389709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2007 , TFLOPS: 68.5371640042703, Tokens per sec: 169842.64239446557, Loss: 3.5140581130981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2008 , TFLOPS: 70.60587747454213, Tokens per sec: 174969.14226140053, Loss: 3.4982731342315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2009 , TFLOPS: 68.7409761375729, Tokens per sec: 170347.71131254933, Loss: 3.4896628856658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2010 , TFLOPS: 68.91232289558121, Tokens per sec: 170772.32745429754, Loss: 3.4997761249542236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2011 , TFLOPS: 69.54889541808551, Tokens per sec: 172349.82428931547, Loss: 3.4863247871398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2012 , TFLOPS: 70.6132342429324, Tokens per sec: 174987.37314388657, Loss: 3.4872307777404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2013 , TFLOPS: 68.30124177304043, Tokens per sec: 169258.00111649872, Loss: 3.4994559288024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2014 , TFLOPS: 69.51081464587497, Tokens per sec: 172255.45594083436, Loss: 3.5210509300231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2015 , TFLOPS: 69.58127483878695, Tokens per sec: 172430.06406070126, Loss: 3.5135722160339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2016 , TFLOPS: 67.49692148855223, Tokens per sec: 167264.80684834326, Loss: 3.5022459030151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2017 , TFLOPS: 69.63277526610315, Tokens per sec: 172557.68779283122, Loss: 3.4943506717681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2018 , TFLOPS: 68.87927023188023, Tokens per sec: 170690.41931259498, Loss: 3.525588274002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2019 , TFLOPS: 68.7967927841294, Tokens per sec: 170486.0311114275, Loss: 3.5133213996887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2020 , TFLOPS: 68.73923755176962, Tokens per sec: 170343.40290540774, Loss: 3.522758960723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2021 , TFLOPS: 70.61557978115137, Tokens per sec: 174993.18564597552, Loss: 3.483039617538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2022 , TFLOPS: 69.73105185576036, Tokens per sec: 172801.22800806182, Loss: 3.495917558670044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2023 , TFLOPS: 67.24365656982611, Tokens per sec: 166637.18848030598, Loss: 3.494751453399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2024 , TFLOPS: 70.58619154961514, Tokens per sec: 174920.3583708478, Loss: 3.4745099544525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2025 , TFLOPS: 70.62067964173059, Tokens per sec: 175005.82366228686, Loss: 3.5167319774627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2026 , TFLOPS: 67.8384031865642, Tokens per sec: 168111.0361133313, Loss: 3.5141561031341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2027 , TFLOPS: 68.94703805780996, Tokens per sec: 170858.3554504912, Loss: 3.507160186767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2028 , TFLOPS: 70.58622861875017, Tokens per sec: 174920.4502322479, Loss: 3.4952216148376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2029 , TFLOPS: 69.79974769624936, Tokens per sec: 172971.4638682651, Loss: 3.500053882598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2030 , TFLOPS: 67.8118015956572, Tokens per sec: 168045.11444065665, Loss: 3.4988088607788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2031 , TFLOPS: 70.59799071438694, Tokens per sec: 174949.5980009376, Loss: 3.5015854835510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2032 , TFLOPS: 70.55901885042869, Tokens per sec: 174853.02142894384, Loss: 3.511272668838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2033 , TFLOPS: 67.036120636172, Tokens per sec: 166122.8915747407, Loss: 3.515193462371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2034 , TFLOPS: 69.58075315338996, Tokens per sec: 172428.77126681912, Loss: 3.4940335750579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2035 , TFLOPS: 70.61839253218416, Tokens per sec: 175000.15595288493, Loss: 3.4889934062957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2036 , TFLOPS: 67.9501208107341, Tokens per sec: 168387.8846927366, Loss: 3.487262725830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2037 , TFLOPS: 68.62604872398647, Tokens per sec: 170062.90852138243, Loss: 3.5058600902557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2038 , TFLOPS: 70.58279474580038, Tokens per sec: 174911.940716806, Loss: 3.5003154277801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2039 , TFLOPS: 69.73245421260604, Tokens per sec: 172804.7032028078, Loss: 3.5055387020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2040 , TFLOPS: 68.02175367359551, Tokens per sec: 168565.39881791844, Loss: 3.5008602142333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2041 , TFLOPS: 70.47890891940287, Tokens per sec: 174654.50019502448, Loss: 3.502204179763794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2042 , TFLOPS: 70.59252563889552, Tokens per sec: 174936.05494184213, Loss: 3.502987861633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2043 , TFLOPS: 67.5553058164276, Tokens per sec: 167409.48964438305, Loss: 3.486180543899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2044 , TFLOPS: 69.02913466518174, Tokens per sec: 171061.79988724634, Loss: 3.4886200428009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2045 , TFLOPS: 70.62199324645688, Tokens per sec: 175009.07891950413, Loss: 3.499866247177124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2046 , TFLOPS: 68.86341714974213, Tokens per sec: 170651.13362869568, Loss: 3.5270180702209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2047 , TFLOPS: 68.64250785235615, Tokens per sec: 170103.69605460513, Loss: 3.508179187774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2048 , TFLOPS: 68.87624067132126, Tokens per sec: 170682.91172779622, Loss: 3.5043678283691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2049 , TFLOPS: 69.76022254276222, Tokens per sec: 172873.51618386892, Loss: 3.5088157653808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2050 , TFLOPS: 67.42310855707964, Tokens per sec: 167081.8902729885, Loss: 3.5076799392700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2051 , TFLOPS: 70.61807970646684, Tokens per sec: 174999.3807362968, Loss: 3.5148303508758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2052 , TFLOPS: 69.64199992560724, Tokens per sec: 172580.54751526247, Loss: 3.49995493888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2053 , TFLOPS: 68.12024859426495, Tokens per sec: 168809.4800814485, Loss: 3.4920597076416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2054 , TFLOPS: 69.45032894435825, Tokens per sec: 172105.56570942586, Loss: 3.4939398765563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2055 , TFLOPS: 66.5588994547053, Tokens per sec: 164940.28491681325, Loss: 3.4882051944732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2056 , TFLOPS: 67.93990793952354, Tokens per sec: 168362.57607283618, Loss: 3.4971368312835693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2057 , TFLOPS: 69.68418917646916, Tokens per sec: 172685.09712642798, Loss: 3.4671168327331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2058 , TFLOPS: 70.59050982491253, Tokens per sec: 174931.05953273203, Loss: 3.519103765487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2059 , TFLOPS: 68.73984328902158, Tokens per sec: 170344.9039890456, Loss: 3.49133563041687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2060 , TFLOPS: 68.88541356887231, Tokens per sec: 170705.64317840582, Loss: 3.52244234085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2061 , TFLOPS: 69.82316358507141, Tokens per sec: 173029.49102023928, Loss: 3.4845032691955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2062 , TFLOPS: 70.56675091088626, Tokens per sec: 174872.18232651445, Loss: 3.489119529724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2063 , TFLOPS: 67.05287179038558, Tokens per sec: 166164.4027801739, Loss: 3.485222339630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2064 , TFLOPS: 69.61080425685897, Tokens per sec: 172503.24121161766, Loss: 3.488525867462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2065 , TFLOPS: 70.56855682504768, Tokens per sec: 174876.65758074436, Loss: 3.505795478820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2066 , TFLOPS: 68.92653657265276, Tokens per sec: 170807.55051184664, Loss: 3.525404453277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2067 , TFLOPS: 67.7452464485732, Tokens per sec: 167880.1834545278, Loss: 3.5150115489959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2068 , TFLOPS: 70.50054675998629, Tokens per sec: 174708.12114759508, Loss: 3.5247836112976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2069 , TFLOPS: 69.75652250839688, Tokens per sec: 172864.34709111915, Loss: 3.513695001602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2070 , TFLOPS: 66.97106640504299, Tokens per sec: 165961.68002368783, Loss: 3.4834790229797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2071 , TFLOPS: 70.53702797235486, Tokens per sec: 174798.5256105817, Loss: 3.516735315322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2072 , TFLOPS: 70.50802017058321, Tokens per sec: 174726.64108231838, Loss: 3.486725091934204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2073 , TFLOPS: 68.15155664407835, Tokens per sec: 168887.06487775253, Loss: 3.5059359073638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2074 , TFLOPS: 68.59022585355773, Tokens per sec: 169974.13550224504, Loss: 3.480985641479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2075 , TFLOPS: 70.56010382986778, Tokens per sec: 174855.7101275143, Loss: 3.502467155456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2076 , TFLOPS: 68.71045732792305, Tokens per sec: 170272.0823403126, Loss: 3.5252814292907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2077 , TFLOPS: 68.18830907028955, Tokens per sec: 168978.14143851784, Loss: 3.499241590499878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2078 , TFLOPS: 70.56037399382018, Tokens per sec: 174856.37962354935, Loss: 3.4932634830474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2079 , TFLOPS: 70.58564327158706, Tokens per sec: 174918.9996774964, Loss: 3.4925808906555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2080 , TFLOPS: 68.39358595268163, Tokens per sec: 169486.8401662008, Loss: 3.5092148780822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2081 , TFLOPS: 68.56468347464514, Tokens per sec: 169910.83867357433, Loss: 3.4726169109344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2082 , TFLOPS: 70.5989376282614, Tokens per sec: 174951.9445578297, Loss: 3.4731345176696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2083 , TFLOPS: 67.90310293695326, Tokens per sec: 168271.3692220613, Loss: 3.516653299331665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2084 , TFLOPS: 69.69174673675089, Tokens per sec: 172703.82559334007, Loss: 3.4938011169433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2085 , TFLOPS: 69.59036690779158, Tokens per sec: 172452.59520926882, Loss: 3.5033161640167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2086 , TFLOPS: 68.908532127105, Tokens per sec: 170762.93351242587, Loss: 3.497213363647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2087 , TFLOPS: 68.64693859007895, Tokens per sec: 170114.67591076827, Loss: 3.520094394683838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2088 , TFLOPS: 70.54444002272632, Tokens per sec: 174816.89348790696, Loss: 3.4979405403137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2089 , TFLOPS: 69.65297274308736, Tokens per sec: 172607.73936573387, Loss: 3.51550555229187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2090 , TFLOPS: 68.09743190874822, Tokens per sec: 168752.93782127264, Loss: 3.5160322189331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2091 , TFLOPS: 69.90407525908122, Tokens per sec: 173229.99906159227, Loss: 3.5098936557769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2092 , TFLOPS: 69.58839775945194, Tokens per sec: 172447.7154427636, Loss: 3.4730117321014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2093 , TFLOPS: 69.00488875490412, Tokens per sec: 171001.71585067088, Loss: 3.530334949493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2094 , TFLOPS: 69.0248449826408, Tokens per sec: 171051.16958136237, Loss: 3.4941177368164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2095 , TFLOPS: 70.5893204291381, Tokens per sec: 174928.11207898022, Loss: 3.516300916671753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2096 , TFLOPS: 68.8107524167193, Tokens per sec: 170520.6246187658, Loss: 3.496638774871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2097 , TFLOPS: 68.119849283633, Tokens per sec: 168808.49054571483, Loss: 3.513784170150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2098 , TFLOPS: 70.53727339388313, Tokens per sec: 174799.13379216418, Loss: 3.5162100791931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2099 , TFLOPS: 70.58960643090273, Tokens per sec: 174928.8208228587, Loss: 3.4939517974853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2100 , TFLOPS: 67.90980415443923, Tokens per sec: 168287.97557719157, Loss: 3.4945545196533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2101 , TFLOPS: 69.56409182094217, Tokens per sec: 172387.48264961623, Loss: 3.50358247756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2102 , TFLOPS: 70.51074497036943, Tokens per sec: 174733.39343635013, Loss: 3.517477512359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2103 , TFLOPS: 67.80350479228412, Tokens per sec: 168024.5540479298, Loss: 3.507309675216675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2104 , TFLOPS: 68.76017406943322, Tokens per sec: 170395.285902526, Loss: 3.5009334087371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2105 , TFLOPS: 70.56913179592001, Tokens per sec: 174878.08242190428, Loss: 3.4909918308258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2106 , TFLOPS: 69.6935312249882, Tokens per sec: 172708.24775176973, Loss: 3.5089755058288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2107 , TFLOPS: 67.1632862052146, Tokens per sec: 166438.02186327212, Loss: 3.5025787353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2108 , TFLOPS: 70.59197252804405, Tokens per sec: 174934.68427222202, Loss: 3.489821672439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2109 , TFLOPS: 70.60852714518691, Tokens per sec: 174975.70843714365, Loss: 3.4833321571350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2110 , TFLOPS: 67.86835857550071, Tokens per sec: 168185.26886697448, Loss: 3.515575885772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2111 , TFLOPS: 68.61545192708378, Tokens per sec: 170036.648491324, Loss: 3.485682487487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2112 , TFLOPS: 70.52956442183309, Tokens per sec: 174780.0301102108, Loss: 3.5312647819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2113 , TFLOPS: 68.96902398658678, Tokens per sec: 170912.8390039501, Loss: 3.5220236778259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2114 , TFLOPS: 67.85359096738375, Tokens per sec: 168148.67310727434, Loss: 3.509242057800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2115 , TFLOPS: 70.59301718097906, Tokens per sec: 174937.27303727294, Loss: 3.4970524311065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2116 , TFLOPS: 69.67032091544917, Tokens per sec: 172650.73004790535, Loss: 3.519486904144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2117 , TFLOPS: 67.8191548415056, Tokens per sec: 168063.3365938959, Loss: 3.5029423236846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2118 , TFLOPS: 69.56998892846936, Tokens per sec: 172402.09633168863, Loss: 3.4998373985290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2119 , TFLOPS: 70.60704848260656, Tokens per sec: 174972.0441483814, Loss: 3.5036535263061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2120 , TFLOPS: 68.18503293290668, Tokens per sec: 168970.02280918678, Loss: 3.489314556121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2121 , TFLOPS: 69.51325215641948, Tokens per sec: 172261.49636047764, Loss: 3.4932053089141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2122 , TFLOPS: 68.92793929578411, Tokens per sec: 170811.0266142888, Loss: 3.511246919631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2123 , TFLOPS: 68.83041688408335, Tokens per sec: 170569.3553351138, Loss: 3.481382131576538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2124 , TFLOPS: 68.96260990462058, Tokens per sec: 170896.94420226847, Loss: 3.5074923038482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2125 , TFLOPS: 70.55632905805317, Tokens per sec: 174846.35582713224, Loss: 3.495396614074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2126 , TFLOPS: 69.55055633386232, Tokens per sec: 172353.94022157538, Loss: 3.488327741622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2127 , TFLOPS: 68.38332092890559, Tokens per sec: 169461.40230649803, Loss: 3.501645565032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2128 , TFLOPS: 69.85747921854211, Tokens per sec: 173114.52894015875, Loss: 3.525787830352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2129 , TFLOPS: 69.68141845386101, Tokens per sec: 172678.2309705847, Loss: 3.4964439868927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2130 , TFLOPS: 67.93814856881686, Tokens per sec: 168358.21615841435, Loss: 3.4937362670898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2131 , TFLOPS: 69.54766683547406, Tokens per sec: 172346.77972626474, Loss: 3.5121257305145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2132 , TFLOPS: 70.5601534163399, Tokens per sec: 174855.83300825462, Loss: 3.505722999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2133 , TFLOPS: 67.99615619121411, Tokens per sec: 168501.96543678184, Loss: 3.490003824234009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2134 , TFLOPS: 68.70179232178714, Tokens per sec: 170250.6095005786, Loss: 3.512967109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2135 , TFLOPS: 70.58495321434573, Tokens per sec: 174917.2896396363, Loss: 3.4905335903167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2136 , TFLOPS: 70.5705223956864, Tokens per sec: 174881.5284813126, Loss: 3.5017166137695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2137 , TFLOPS: 66.39573520800057, Tokens per sec: 164535.94593945338, Loss: 3.485964775085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2138 , TFLOPS: 70.53271725256609, Tokens per sec: 174787.84317208015, Loss: 3.5210824012756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2139 , TFLOPS: 70.5437152126861, Tokens per sec: 174815.09732878284, Loss: 3.500452756881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2140 , TFLOPS: 68.06839823112179, Tokens per sec: 168680.98916979073, Loss: 3.494389057159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2141 , TFLOPS: 68.62156773501056, Tokens per sec: 170051.8041370784, Loss: 3.52852201461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2142 , TFLOPS: 70.62072115563002, Tokens per sec: 175005.9265383028, Loss: 3.510794162750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2143 , TFLOPS: 68.8099664766876, Tokens per sec: 170518.67697278882, Loss: 3.50941801071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2144 , TFLOPS: 68.62649627151956, Tokens per sec: 170064.01759347087, Loss: 3.481527090072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2145 , TFLOPS: 70.61251715378107, Tokens per sec: 174985.5961179757, Loss: 3.5060958862304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2146 , TFLOPS: 70.62632894285672, Tokens per sec: 175019.82325279928, Loss: 3.5094759464263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2147 , TFLOPS: 68.0249013549215, Tokens per sec: 168573.19911898996, Loss: 3.49605393409729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2148 , TFLOPS: 68.61417330514382, Tokens per sec: 170033.47992530497, Loss: 3.4795074462890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2149 , TFLOPS: 70.55422600785744, Tokens per sec: 174841.1442370771, Loss: 3.4619827270507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2150 , TFLOPS: 68.04851039975647, Tokens per sec: 168631.704932842, Loss: 3.5127408504486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2151 , TFLOPS: 68.64617469619802, Tokens per sec: 170112.78289758146, Loss: 3.5266811847686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2152 , TFLOPS: 70.59912135615949, Tokens per sec: 174952.39985580047, Loss: 3.4806556701660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2153 , TFLOPS: 68.85508675149715, Tokens per sec: 170630.48998417548, Loss: 3.482170581817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2154 , TFLOPS: 68.64614360642113, Tokens per sec: 170112.70585368955, Loss: 3.5235953330993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2155 , TFLOPS: 69.59717757872819, Tokens per sec: 172469.47280210708, Loss: 3.4866318702697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2156 , TFLOPS: 70.5952653896431, Tokens per sec: 174942.8443460031, Loss: 3.496006965637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2157 , TFLOPS: 67.08053028984497, Tokens per sec: 166232.9435886695, Loss: 3.4868927001953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2158 , TFLOPS: 70.56904840022727, Tokens per sec: 174877.87575819087, Loss: 3.509768009185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2159 , TFLOPS: 69.71077489643953, Tokens per sec: 172750.97946917338, Loss: 3.469712734222412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2160 , TFLOPS: 68.84819335416516, Tokens per sec: 170613.4073862164, Loss: 3.4930765628814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2161 , TFLOPS: 69.0016015179402, Tokens per sec: 170993.56971535523, Loss: 3.4660425186157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2162 , TFLOPS: 70.57673460165216, Tokens per sec: 174896.92301202676, Loss: 3.4814162254333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2163 , TFLOPS: 68.77168287589011, Tokens per sec: 170423.80599272586, Loss: 3.5000925064086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2164 , TFLOPS: 68.09795570216495, Tokens per sec: 168754.23583905993, Loss: 3.5007143020629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2165 , TFLOPS: 70.61668003577412, Tokens per sec: 174995.91219813377, Loss: 3.495231866836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2166 , TFLOPS: 69.60058593945469, Tokens per sec: 172477.91909544927, Loss: 3.495171308517456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2167 , TFLOPS: 67.30261164128126, Tokens per sec: 166783.28564180908, Loss: 3.49381160736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2168 , TFLOPS: 69.62102655472249, Tokens per sec: 172528.5731918025, Loss: 3.4936816692352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2169 , TFLOPS: 70.58535614450341, Tokens per sec: 174918.28814495346, Loss: 3.506333827972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2170 , TFLOPS: 67.17881030482164, Tokens per sec: 166476.49229221907, Loss: 3.4973063468933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2171 , TFLOPS: 69.64947059156697, Tokens per sec: 172599.06064847246, Loss: 3.5118672847747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2172 , TFLOPS: 70.63913491293167, Tokens per sec: 175051.55785733907, Loss: 3.5176501274108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2173 , TFLOPS: 69.78522735257195, Tokens per sec: 172935.48085708462, Loss: 3.5172958374023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2174 , TFLOPS: 67.18859934597725, Tokens per sec: 166500.75061455462, Loss: 3.482133626937866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2175 , TFLOPS: 70.61097204436274, Tokens per sec: 174981.76716662923, Loss: 3.4839999675750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2176 , TFLOPS: 70.54919255950611, Tokens per sec: 174828.6707975824, Loss: 3.5081920623779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2177 , TFLOPS: 67.9984991182357, Tokens per sec: 168507.77146803608, Loss: 3.494541645050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2178 , TFLOPS: 68.94771693314692, Tokens per sec: 170860.03777836045, Loss: 3.5061240196228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2179 , TFLOPS: 70.57058408432046, Tokens per sec: 174881.68135254335, Loss: 3.4913885593414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2180 , TFLOPS: 68.83192989357995, Tokens per sec: 170573.10473931817, Loss: 3.4744949340820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2181 , TFLOPS: 67.75637084814169, Tokens per sec: 167907.7509421123, Loss: 3.492356538772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2182 , TFLOPS: 70.59385495126806, Tokens per sec: 174939.3491243371, Loss: 3.518399953842163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2183 , TFLOPS: 69.7968008890268, Tokens per sec: 172964.16135535, Loss: 3.4999539852142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2184 , TFLOPS: 67.77841562054421, Tokens per sec: 167962.3803165579, Loss: 3.510303497314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2185 , TFLOPS: 69.58940201750644, Tokens per sec: 172450.2041048508, Loss: 3.478513240814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2186 , TFLOPS: 70.61076476788884, Tokens per sec: 174981.25351269348, Loss: 3.516915798187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2187 , TFLOPS: 67.94385925130389, Tokens per sec: 168372.36785870083, Loss: 3.474424362182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2188 , TFLOPS: 68.73029422116477, Tokens per sec: 170321.24034697976, Loss: 3.4852561950683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2189 , TFLOPS: 70.61438054590532, Tokens per sec: 174990.21380892917, Loss: 3.512028217315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2190 , TFLOPS: 67.96108658433145, Tokens per sec: 168415.05908769052, Loss: 3.508748769760132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2191 , TFLOPS: 69.51259139536918, Tokens per sec: 172259.85892180618, Loss: 3.4895169734954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2192 , TFLOPS: 69.5164611942858, Tokens per sec: 172269.44870981504, Loss: 3.499662399291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2193 , TFLOPS: 69.74403624155438, Tokens per sec: 172833.40474640758, Loss: 3.481529474258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2194 , TFLOPS: 68.79880516154279, Tokens per sec: 170491.0180043396, Loss: 3.504800319671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2195 , TFLOPS: 69.86108904716681, Tokens per sec: 173123.4744931462, Loss: 3.511368989944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2196 , TFLOPS: 69.69641548842118, Tokens per sec: 172715.3952742845, Loss: 3.505728244781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2197 , TFLOPS: 67.33273426791807, Tokens per sec: 166857.93282889636, Loss: 3.5153591632843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2198 , TFLOPS: 69.57222778523506, Tokens per sec: 172407.64446538445, Loss: 3.4945333003997803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2199 , TFLOPS: 70.61125613085083, Tokens per sec: 174982.47116424143, Loss: 3.498734474182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2200 , TFLOPS: 67.87648520717158, Tokens per sec: 168205.40755547732, Loss: 3.489387273788452 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/optimizer.pt +[rank0]:[2024-08-25 01:08:11,939] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006419497000024421, 'preprocessing_with_comm': 0.0012335489991528448, 'state_converting': 0.9953707550012041, : 1.0044395469994924}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000200 + eval ppl=24.72534942626953, eval loss=3.207828998565674 +------------------------------------------------------------------ +iteration: 2201 , TFLOPS: 63.02711001880871, Tokens per sec: 156188.12163593748, Loss: 3.5096054077148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2202 , TFLOPS: 70.55208368048505, Tokens per sec: 174835.83531385148, Loss: 3.5043609142303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2203 , TFLOPS: 69.21442735130938, Tokens per sec: 171520.975575145, Loss: 3.4954469203948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2204 , TFLOPS: 68.26634689232064, Tokens per sec: 169171.52775808665, Loss: 3.520108461380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2205 , TFLOPS: 69.19198386700278, Tokens per sec: 171465.35814867937, Loss: 3.5062737464904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2206 , TFLOPS: 70.58015513556354, Tokens per sec: 174905.39947186204, Loss: 3.510150671005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2207 , TFLOPS: 68.54236247324351, Tokens per sec: 169855.52477323983, Loss: 3.5148699283599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2208 , TFLOPS: 68.64821997341346, Tokens per sec: 170117.85131982726, Loss: 3.506749391555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2209 , TFLOPS: 70.56128402204521, Tokens per sec: 174858.6347737389, Loss: 3.4735183715820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2210 , TFLOPS: 69.67708461521511, Tokens per sec: 172667.49124100842, Loss: 3.506951332092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2211 , TFLOPS: 66.86392065644782, Tokens per sec: 165696.16105559043, Loss: 3.466322898864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2212 , TFLOPS: 70.51849032089896, Tokens per sec: 174752.58726818312, Loss: 3.5030245780944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2213 , TFLOPS: 70.59545810691655, Tokens per sec: 174943.32192063628, Loss: 3.4990170001983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2214 , TFLOPS: 66.89989752737823, Tokens per sec: 165785.3157049355, Loss: 3.4835281372070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2215 , TFLOPS: 69.87080707803693, Tokens per sec: 173147.55684417186, Loss: 3.503925085067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2216 , TFLOPS: 70.576350294649, Tokens per sec: 174895.97065693798, Loss: 3.5088891983032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2217 , TFLOPS: 68.8208658519374, Tokens per sec: 170545.68682532312, Loss: 3.506103754043579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2218 , TFLOPS: 67.17706815743033, Tokens per sec: 166472.17505907005, Loss: 3.5011065006256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2219 , TFLOPS: 70.59104321331871, Tokens per sec: 174932.38132796012, Loss: 3.5213623046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2220 , TFLOPS: 69.61241761585089, Tokens per sec: 172507.2392929258, Loss: 3.463611125946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2221 , TFLOPS: 67.39001392848473, Tokens per sec: 166999.87813765605, Loss: 3.539149761199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2222 , TFLOPS: 69.49406308521492, Tokens per sec: 172213.94372818203, Loss: 3.485774278640747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2223 , TFLOPS: 69.75700210944709, Tokens per sec: 172865.53559535395, Loss: 3.4759202003479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2224 , TFLOPS: 68.8141209447553, Tokens per sec: 170528.97220231302, Loss: 3.4892525672912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2225 , TFLOPS: 68.84729846188588, Tokens per sec: 170611.18974456782, Loss: 3.5179049968719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2226 , TFLOPS: 69.53001125385484, Tokens per sec: 172303.02725008942, Loss: 3.5185420513153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2227 , TFLOPS: 68.85398181105184, Tokens per sec: 170627.75182003272, Loss: 3.505290985107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2228 , TFLOPS: 68.57671768671703, Tokens per sec: 169940.6607767664, Loss: 3.495079517364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2229 , TFLOPS: 69.43992086493044, Tokens per sec: 172079.77334205914, Loss: 3.4987521171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2230 , TFLOPS: 69.66550339648275, Tokens per sec: 172638.79170521305, Loss: 3.50276517868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2231 , TFLOPS: 67.11729391285519, Tokens per sec: 166324.04789633624, Loss: 3.498987913131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2232 , TFLOPS: 70.51492490981015, Tokens per sec: 174743.75178674256, Loss: 3.4906091690063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2233 , TFLOPS: 69.55650643291011, Tokens per sec: 172368.68522247332, Loss: 3.473752975463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2234 , TFLOPS: 68.06468439827664, Tokens per sec: 168671.78588288798, Loss: 3.505171298980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2235 , TFLOPS: 69.0043931063992, Tokens per sec: 171000.48757907553, Loss: 3.505690097808838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2236 , TFLOPS: 70.50258259090654, Tokens per sec: 174713.16616088076, Loss: 3.4923341274261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2237 , TFLOPS: 68.59537521124649, Tokens per sec: 169986.89617784636, Loss: 3.4656219482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2238 , TFLOPS: 67.36966931340045, Tokens per sec: 166949.4619403328, Loss: 3.4928903579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2239 , TFLOPS: 70.58709236235246, Tokens per sec: 174922.59068404362, Loss: 3.482276439666748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2240 , TFLOPS: 69.74493524822107, Tokens per sec: 172835.63258396141, Loss: 3.484708547592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2241 , TFLOPS: 66.32118869645993, Tokens per sec: 164351.21147187948, Loss: 3.5132932662963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2242 , TFLOPS: 70.58896902892924, Tokens per sec: 174927.24127055274, Loss: 3.4985077381134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2243 , TFLOPS: 70.57572735696948, Tokens per sec: 174894.42694874588, Loss: 3.4915919303894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2244 , TFLOPS: 68.02235189187124, Tokens per sec: 168566.88126870486, Loss: 3.5128941535949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2245 , TFLOPS: 68.39684817869153, Tokens per sec: 169494.92432161717, Loss: 3.4953322410583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2246 , TFLOPS: 70.61922637401933, Tokens per sec: 175002.22230480774, Loss: 3.5158698558807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2247 , TFLOPS: 69.00221505358891, Tokens per sec: 170995.09012427926, Loss: 3.516010046005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2248 , TFLOPS: 67.04978139951875, Tokens per sec: 166156.74445117125, Loss: 3.5005645751953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2249 , TFLOPS: 70.55603839886922, Tokens per sec: 174845.63554165565, Loss: 3.5046448707580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2250 , TFLOPS: 69.62275943827704, Tokens per sec: 172532.8674681436, Loss: 3.4897820949554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2251 , TFLOPS: 67.05495870947868, Tokens per sec: 166169.57439557972, Loss: 3.5206854343414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2252 , TFLOPS: 70.06446472083192, Tokens per sec: 173627.4618161683, Loss: 3.5042176246643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2253 , TFLOPS: 70.52374797624694, Tokens per sec: 174765.61631731395, Loss: 3.518115997314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2254 , TFLOPS: 68.92137017919954, Tokens per sec: 170794.7476197443, Loss: 3.492670774459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2255 , TFLOPS: 66.7296883434679, Tokens per sec: 165363.5185971162, Loss: 3.490321159362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2256 , TFLOPS: 70.59964212766239, Tokens per sec: 174953.6903849524, Loss: 3.485318899154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2257 , TFLOPS: 69.7225878292233, Tokens per sec: 172780.25321791353, Loss: 3.4948790073394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2258 , TFLOPS: 67.24800185701127, Tokens per sec: 166647.9565806237, Loss: 3.5235772132873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2259 , TFLOPS: 69.58328731672037, Tokens per sec: 172435.05120271305, Loss: 3.5001461505889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2260 , TFLOPS: 70.50332756791755, Tokens per sec: 174715.01229598004, Loss: 3.4946863651275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2261 , TFLOPS: 66.68783174501911, Tokens per sec: 165259.79333527514, Loss: 3.5009212493896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2262 , TFLOPS: 70.56831150559167, Tokens per sec: 174876.0496521082, Loss: 3.5150766372680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2263 , TFLOPS: 69.52335149495183, Tokens per sec: 172286.52363389477, Loss: 3.5160868167877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2264 , TFLOPS: 68.78959231987959, Tokens per sec: 170468.1875678199, Loss: 3.481111526489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2265 , TFLOPS: 68.0439135975437, Tokens per sec: 168620.3135505792, Loss: 3.483621120452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2266 , TFLOPS: 69.71927593745085, Tokens per sec: 172772.04598526636, Loss: 3.493499994277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2267 , TFLOPS: 69.79890961970857, Tokens per sec: 172969.38702227527, Loss: 3.4927282333374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2268 , TFLOPS: 68.00908004526102, Tokens per sec: 168533.99217079012, Loss: 3.5098836421966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2269 , TFLOPS: 70.59267052555957, Tokens per sec: 174936.41398695624, Loss: 3.5221309661865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2270 , TFLOPS: 68.60473465518773, Tokens per sec: 170010.08991096064, Loss: 3.4980454444885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2271 , TFLOPS: 68.78648415010386, Tokens per sec: 170460.48518072246, Loss: 3.503143548965454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2272 , TFLOPS: 69.10526977360777, Tokens per sec: 171250.4710729006, Loss: 3.498560905456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2273 , TFLOPS: 70.56109109633212, Tokens per sec: 174858.1566825693, Loss: 3.520188093185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2274 , TFLOPS: 68.06769459907369, Tokens per sec: 168679.24549206407, Loss: 3.504045248031616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2275 , TFLOPS: 67.95279023641888, Tokens per sec: 168394.499823645, Loss: 3.5069117546081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2276 , TFLOPS: 70.52872724661647, Tokens per sec: 174777.95549780157, Loss: 3.5317463874816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2277 , TFLOPS: 68.68854586391178, Tokens per sec: 170217.78331874564, Loss: 3.4995880126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2278 , TFLOPS: 67.13026945240583, Tokens per sec: 166356.2027127164, Loss: 3.5066099166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2279 , TFLOPS: 70.65018037064718, Tokens per sec: 175078.92971831578, Loss: 3.5122084617614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2280 , TFLOPS: 70.61297614341233, Tokens per sec: 174986.73354484382, Loss: 3.500450849533081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2281 , TFLOPS: 66.46097477666707, Tokens per sec: 164697.61677734074, Loss: 3.5011496543884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2282 , TFLOPS: 69.97669411076572, Tokens per sec: 173409.95657569336, Loss: 3.5068511962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2283 , TFLOPS: 70.51953434059352, Tokens per sec: 174755.1744639945, Loss: 3.4934799671173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2284 , TFLOPS: 68.83091775011717, Tokens per sec: 170570.59653632058, Loss: 3.4925320148468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2285 , TFLOPS: 67.79026267235017, Tokens per sec: 167991.73861599353, Loss: 3.4946465492248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2286 , TFLOPS: 70.56220614898318, Tokens per sec: 174860.91990587182, Loss: 3.4892706871032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2287 , TFLOPS: 69.78748400310711, Tokens per sec: 172941.07308570133, Loss: 3.4764535427093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2288 , TFLOPS: 66.90728440475966, Tokens per sec: 165803.6212008184, Loss: 3.524506092071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2289 , TFLOPS: 70.57414570937732, Tokens per sec: 174890.50745177965, Loss: 3.506722927093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2290 , TFLOPS: 70.59804809240704, Tokens per sec: 174949.74018999215, Loss: 3.466336250305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2291 , TFLOPS: 68.21503802639336, Tokens per sec: 169044.37873618066, Loss: 3.5063400268554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2292 , TFLOPS: 67.89329475317167, Tokens per sec: 168247.0634621899, Loss: 3.469334363937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2293 , TFLOPS: 70.55779935328357, Tokens per sec: 174849.99938067954, Loss: 3.5053606033325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2294 , TFLOPS: 68.96150486106961, Tokens per sec: 170894.20578261855, Loss: 3.4938342571258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2295 , TFLOPS: 68.04806048468954, Tokens per sec: 168630.58999374392, Loss: 3.5171687602996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2296 , TFLOPS: 69.53944661936751, Tokens per sec: 172326.40912522117, Loss: 3.5050060749053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2297 , TFLOPS: 69.62756849239835, Tokens per sec: 172544.78483401722, Loss: 3.5232229232788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2298 , TFLOPS: 67.21495782845572, Tokens per sec: 166566.0698377628, Loss: 3.501851797103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2299 , TFLOPS: 70.54300672525139, Tokens per sec: 174813.3416188733, Loss: 3.4837236404418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2300 , TFLOPS: 69.62019902641279, Tokens per sec: 172526.52248549132, Loss: 3.515000343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2301 , TFLOPS: 68.10005771872831, Tokens per sec: 168759.44486766015, Loss: 3.5079972743988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2302 , TFLOPS: 69.07502189469515, Tokens per sec: 171175.5134968762, Loss: 3.5071747303009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2303 , TFLOPS: 69.53267351480125, Tokens per sec: 172309.62462598688, Loss: 3.4616897106170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2304 , TFLOPS: 69.61906011090996, Tokens per sec: 172523.7001274134, Loss: 3.5011913776397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2305 , TFLOPS: 67.50283889185623, Tokens per sec: 167279.47082558632, Loss: 3.4956634044647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2306 , TFLOPS: 70.5560386339399, Tokens per sec: 174845.63612418668, Loss: 3.507560968399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2307 , TFLOPS: 68.97136850519668, Tokens per sec: 170918.6489793355, Loss: 3.498748540878296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2308 , TFLOPS: 67.88029555205729, Tokens per sec: 168214.85000984825, Loss: 3.4892451763153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2309 , TFLOPS: 70.59033975145702, Tokens per sec: 174930.63807197416, Loss: 3.509725332260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2310 , TFLOPS: 70.60251179470076, Tokens per sec: 174960.8017359934, Loss: 3.505460262298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2311 , TFLOPS: 66.92970232958936, Tokens per sec: 165859.17528808466, Loss: 3.491725444793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2312 , TFLOPS: 68.75075138474563, Tokens per sec: 170371.9354518757, Loss: 3.515371561050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2313 , TFLOPS: 70.59105498783364, Tokens per sec: 174932.4105065052, Loss: 3.5119543075561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2314 , TFLOPS: 68.91474023948342, Tokens per sec: 170778.3178987796, Loss: 3.521747589111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2315 , TFLOPS: 67.18399609412691, Tokens per sec: 166489.34324937942, Loss: 3.471477746963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2316 , TFLOPS: 70.62144758668786, Tokens per sec: 175007.72671448713, Loss: 3.4820775985717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2317 , TFLOPS: 69.74555190291042, Tokens per sec: 172837.16072221173, Loss: 3.508929967880249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2318 , TFLOPS: 67.4805543531213, Tokens per sec: 167224.24728375662, Loss: 3.511367082595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2319 , TFLOPS: 70.15211244914278, Tokens per sec: 173844.66254211692, Loss: 3.469355344772339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2320 , TFLOPS: 70.60208544179088, Tokens per sec: 174959.74518651603, Loss: 3.4791948795318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2321 , TFLOPS: 68.88680210617744, Tokens per sec: 170709.08412680795, Loss: 3.499119281768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2322 , TFLOPS: 67.54926296572205, Tokens per sec: 167394.51479465974, Loss: 3.5010833740234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2323 , TFLOPS: 70.5783279474369, Tokens per sec: 174900.87149840288, Loss: 3.5034384727478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2324 , TFLOPS: 69.70085824944042, Tokens per sec: 172726.4049255007, Loss: 3.4824564456939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2325 , TFLOPS: 66.19161119029481, Tokens per sec: 164030.10413746166, Loss: 3.4721951484680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2326 , TFLOPS: 70.62336781425579, Tokens per sec: 175012.48524993093, Loss: 3.5042569637298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2327 , TFLOPS: 70.57626404063815, Tokens per sec: 174895.75690999746, Loss: 3.512028455734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2328 , TFLOPS: 67.38106515467564, Tokens per sec: 166977.7020903683, Loss: 3.506542444229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2329 , TFLOPS: 69.5702951793639, Tokens per sec: 172402.8552551416, Loss: 3.5388853549957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2330 , TFLOPS: 70.52307855534568, Tokens per sec: 174763.9574185778, Loss: 3.4893088340759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2331 , TFLOPS: 68.66115913289562, Tokens per sec: 170149.91598239014, Loss: 3.4982073307037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2332 , TFLOPS: 68.2192242931544, Tokens per sec: 169054.7527663699, Loss: 3.5009169578552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2333 , TFLOPS: 69.61610505127011, Tokens per sec: 172516.3771640992, Loss: 3.499323844909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2334 , TFLOPS: 68.88540090449112, Tokens per sec: 170705.61179467413, Loss: 3.5090174674987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2335 , TFLOPS: 68.02740186787948, Tokens per sec: 168579.39566555404, Loss: 3.5338733196258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2336 , TFLOPS: 70.61490414206747, Tokens per sec: 174991.51133789786, Loss: 3.497465133666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2337 , TFLOPS: 68.16715316644975, Tokens per sec: 168925.71477828763, Loss: 3.495509386062622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2338 , TFLOPS: 68.86541604905595, Tokens per sec: 170656.08712139222, Loss: 3.493738889694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2339 , TFLOPS: 69.06300686598546, Tokens per sec: 171145.7389321689, Loss: 3.4865939617156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2340 , TFLOPS: 69.61818594163934, Tokens per sec: 172521.5338396631, Loss: 3.491008758544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2341 , TFLOPS: 68.8849000339593, Tokens per sec: 170704.37058232084, Loss: 3.502034902572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2342 , TFLOPS: 68.07117699472887, Tokens per sec: 168687.87525211516, Loss: 3.4991583824157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2343 , TFLOPS: 70.62171367060834, Tokens per sec: 175008.3860997536, Loss: 3.520339012145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2344 , TFLOPS: 68.60938268700394, Tokens per sec: 170021.6082458242, Loss: 3.5237677097320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2345 , TFLOPS: 67.20618690427963, Tokens per sec: 166544.33452145738, Loss: 3.496351480484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2346 , TFLOPS: 70.60145053534053, Tokens per sec: 174958.1718183914, Loss: 3.4892613887786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2347 , TFLOPS: 70.63024307683827, Tokens per sec: 175029.52290798305, Loss: 3.5010671615600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2348 , TFLOPS: 67.1951382357491, Tokens per sec: 166516.95470372742, Loss: 3.5296194553375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2349 , TFLOPS: 69.74784893554319, Tokens per sec: 172842.8530221148, Loss: 3.4547929763793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2350 , TFLOPS: 70.52174697545809, Tokens per sec: 174760.65761693075, Loss: 3.4823639392852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2351 , TFLOPS: 68.64690261310626, Tokens per sec: 170114.5867558668, Loss: 3.498457431793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2352 , TFLOPS: 68.02073799379157, Tokens per sec: 168562.88185147793, Loss: 3.5004663467407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2353 , TFLOPS: 70.59099958832175, Tokens per sec: 174932.27322041197, Loss: 3.487842082977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2354 , TFLOPS: 69.71432319367361, Tokens per sec: 172759.77254059663, Loss: 3.496147871017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2355 , TFLOPS: 66.47066760660901, Tokens per sec: 164721.6366776902, Loss: 3.492205858230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2356 , TFLOPS: 70.60640552078749, Tokens per sec: 174970.45081816564, Loss: 3.4998068809509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2357 , TFLOPS: 69.68406910843822, Tokens per sec: 172684.79958462072, Loss: 3.4802589416503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2358 , TFLOPS: 68.86936704686774, Tokens per sec: 170665.87812920817, Loss: 3.519645929336548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2359 , TFLOPS: 67.55140201500281, Tokens per sec: 167399.81559441323, Loss: 3.517042636871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2360 , TFLOPS: 70.59716575215552, Tokens per sec: 174947.55365365106, Loss: 3.4952378273010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2361 , TFLOPS: 68.20820046953753, Tokens per sec: 169027.43451707286, Loss: 3.5064938068389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2362 , TFLOPS: 67.15027891599487, Tokens per sec: 166405.78836771444, Loss: 3.501830577850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2363 , TFLOPS: 70.60745720193765, Tokens per sec: 174973.05699990797, Loss: 3.4994359016418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2364 , TFLOPS: 69.6890447983512, Tokens per sec: 172697.12989234255, Loss: 3.501408100128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2365 , TFLOPS: 67.22914575457997, Tokens per sec: 166601.22908162777, Loss: 3.4950599670410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2366 , TFLOPS: 69.66406640506212, Tokens per sec: 172635.23068217502, Loss: 3.4857828617095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2367 , TFLOPS: 70.57608290605683, Tokens per sec: 174895.3080385513, Loss: 3.4885334968566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2368 , TFLOPS: 68.89475384204952, Tokens per sec: 170728.789404255, Loss: 3.5200159549713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2369 , TFLOPS: 69.04429315047044, Tokens per sec: 171099.36428363674, Loss: 3.5091824531555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2370 , TFLOPS: 69.01840140387115, Tokens per sec: 171035.20168335238, Loss: 3.5060441493988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2371 , TFLOPS: 69.65069671686982, Tokens per sec: 172602.0991220418, Loss: 3.4914989471435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2372 , TFLOPS: 67.25069933066757, Tokens per sec: 166654.64121749526, Loss: 3.5077083110809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2373 , TFLOPS: 70.57028212327837, Tokens per sec: 174880.93305981733, Loss: 3.4943687915802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2374 , TFLOPS: 68.85247809344324, Tokens per sec: 170624.02544215048, Loss: 3.516629457473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2375 , TFLOPS: 67.35660871498499, Tokens per sec: 166917.0963387156, Loss: 3.4903337955474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2376 , TFLOPS: 70.58061902079999, Tokens per sec: 174906.54903057913, Loss: 3.4857959747314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2377 , TFLOPS: 69.47667948531681, Tokens per sec: 172170.8652526729, Loss: 3.5183732509613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2378 , TFLOPS: 67.74950497412783, Tokens per sec: 167890.7365499675, Loss: 3.485976457595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2379 , TFLOPS: 68.80283820934284, Tokens per sec: 170501.012341065, Loss: 3.5101752281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2380 , TFLOPS: 70.57213779326926, Tokens per sec: 174885.53161447367, Loss: 3.4959323406219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2381 , TFLOPS: 68.07682134643548, Tokens per sec: 168701.8625774208, Loss: 3.5138487815856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2382 , TFLOPS: 67.619687193798, Tokens per sec: 167569.03378970112, Loss: 3.4785332679748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2383 , TFLOPS: 70.61039822781655, Tokens per sec: 174980.34518601664, Loss: 3.4953410625457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2384 , TFLOPS: 69.64832485420538, Tokens per sec: 172596.22138507708, Loss: 3.5098934173583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2385 , TFLOPS: 66.86365231878158, Tokens per sec: 165695.49608529362, Loss: 3.4924001693725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2386 , TFLOPS: 69.97887744575361, Tokens per sec: 173415.36712030758, Loss: 3.500375747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2387 , TFLOPS: 70.65771012407, Tokens per sec: 175097.5892767683, Loss: 3.508458375930786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2388 , TFLOPS: 68.87465937365903, Tokens per sec: 170678.9930979949, Loss: 3.516521453857422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2389 , TFLOPS: 67.76582717213516, Tokens per sec: 167931.18475466818, Loss: 3.4986352920532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2390 , TFLOPS: 70.57435457709657, Tokens per sec: 174891.0250489966, Loss: 3.495192050933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2391 , TFLOPS: 69.6710519008535, Tokens per sec: 172652.54151026197, Loss: 3.5356829166412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2392 , TFLOPS: 65.89149649835254, Tokens per sec: 163286.3868704662, Loss: 3.5032598972320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2393 , TFLOPS: 70.61447443718905, Tokens per sec: 174990.4464818732, Loss: 3.496046781539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2394 , TFLOPS: 70.58916378237166, Tokens per sec: 174927.72389103693, Loss: 3.4918673038482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2395 , TFLOPS: 66.6725773168397, Tokens per sec: 165221.99118183262, Loss: 3.4934048652648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2396 , TFLOPS: 69.67380765119232, Tokens per sec: 172659.37056317253, Loss: 3.515530586242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2397 , TFLOPS: 70.59521803178761, Tokens per sec: 174942.7269880203, Loss: 3.5070791244506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2398 , TFLOPS: 68.88920809559679, Tokens per sec: 170715.04643363072, Loss: 3.493603229522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2399 , TFLOPS: 67.34661130866967, Tokens per sec: 166892.32166456102, Loss: 3.513775587081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2400 , TFLOPS: 70.54383460496759, Tokens per sec: 174815.39319600852, Loss: 3.4981911182403564 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/optimizer.pt +[rank0]:[2024-08-25 01:33:51,376] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006349597999360412, 'preprocessing_with_comm': 0.0012475950024963822, 'state_converting': 1.006559579000168, : 1.0155497539999487}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400, took 4.30s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000400 + eval ppl=23.30388069152832, eval loss=3.1486198902130127 +------------------------------------------------------------------ +iteration: 2401 , TFLOPS: 63.544169116592286, Tokens per sec: 157469.45103900685, Loss: 3.5090813636779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2402 , TFLOPS: 68.14906524179302, Tokens per sec: 168880.8909084389, Loss: 3.515517473220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2403 , TFLOPS: 69.19158767223698, Tokens per sec: 171464.37633440527, Loss: 3.520097494125366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2404 , TFLOPS: 70.57825905698832, Tokens per sec: 174900.70078028232, Loss: 3.520463466644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2405 , TFLOPS: 67.19671318531591, Tokens per sec: 166520.857602249, Loss: 3.5349535942077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2406 , TFLOPS: 70.6326539264565, Tokens per sec: 175035.49725891405, Loss: 3.4933252334594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2407 , TFLOPS: 69.35252901225321, Tokens per sec: 171863.20670411768, Loss: 3.5205202102661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2408 , TFLOPS: 68.0234627443215, Tokens per sec: 168569.6340834471, Loss: 3.5096096992492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2409 , TFLOPS: 68.30416260632101, Tokens per sec: 169265.23926312436, Loss: 3.475975275039673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2410 , TFLOPS: 69.340363522515, Tokens per sec: 171833.05928040706, Loss: 3.473109483718872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2411 , TFLOPS: 68.87818120925348, Tokens per sec: 170687.72059456017, Loss: 3.4964616298675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2412 , TFLOPS: 69.4651752208535, Tokens per sec: 172142.35641803345, Loss: 3.5192830562591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2413 , TFLOPS: 70.53605326689872, Tokens per sec: 174796.11018308875, Loss: 3.471395254135132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2414 , TFLOPS: 68.74335917834058, Tokens per sec: 170353.61675008526, Loss: 3.4943485260009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2415 , TFLOPS: 68.26811524819813, Tokens per sec: 169175.90993875155, Loss: 3.5118677616119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2416 , TFLOPS: 69.14156115397594, Tokens per sec: 171340.4051112785, Loss: 3.496591806411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2417 , TFLOPS: 70.59294491105443, Tokens per sec: 174937.09394443853, Loss: 3.500256061553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2418 , TFLOPS: 67.88469348982508, Tokens per sec: 168225.74858410968, Loss: 3.515697479248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2419 , TFLOPS: 68.69356459898141, Tokens per sec: 170230.2202971075, Loss: 3.514150619506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2420 , TFLOPS: 70.56588482464184, Tokens per sec: 174870.03606939357, Loss: 3.497997760772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2421 , TFLOPS: 67.9946396363874, Tokens per sec: 168498.20724685898, Loss: 3.5078024864196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2422 , TFLOPS: 67.26415513841117, Tokens per sec: 166687.98619136348, Loss: 3.496715784072876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2423 , TFLOPS: 70.64146328335782, Tokens per sec: 175057.32781574628, Loss: 3.4990062713623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2424 , TFLOPS: 70.50766297824161, Tokens per sec: 174725.75592034793, Loss: 3.5120558738708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2425 , TFLOPS: 66.21431734177001, Tokens per sec: 164086.3724821063, Loss: 3.490800142288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2426 , TFLOPS: 69.93523769750328, Tokens per sec: 173307.22301682812, Loss: 3.4965429306030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2427 , TFLOPS: 70.60246963367528, Tokens per sec: 174960.69725632784, Loss: 3.4957435131073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2428 , TFLOPS: 68.06816589954458, Tokens per sec: 168680.413426549, Loss: 3.510032892227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2429 , TFLOPS: 67.86188436790648, Tokens per sec: 168169.22506736318, Loss: 3.467562437057495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2430 , TFLOPS: 70.57387464971599, Tokens per sec: 174889.8357360792, Loss: 3.476834297180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2431 , TFLOPS: 69.69789768242065, Tokens per sec: 172719.06831429232, Loss: 3.521951675415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2432 , TFLOPS: 67.80521156245078, Tokens per sec: 168028.783612419, Loss: 3.4892029762268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2433 , TFLOPS: 70.58283007051651, Tokens per sec: 174912.0282553439, Loss: 3.5005249977111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2434 , TFLOPS: 70.57947869692788, Tokens per sec: 174903.723182407, Loss: 3.483032464981079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2435 , TFLOPS: 67.4764258195952, Tokens per sec: 167214.01632288343, Loss: 3.489553213119507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2436 , TFLOPS: 67.76342397052575, Tokens per sec: 167925.22935634555, Loss: 3.5182549953460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2437 , TFLOPS: 70.62095479137201, Tokens per sec: 175006.50551340857, Loss: 3.4939708709716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2438 , TFLOPS: 68.81488020014787, Tokens per sec: 170530.85372081466, Loss: 3.491610527038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2439 , TFLOPS: 68.85876435994103, Tokens per sec: 170639.60350302144, Loss: 3.490828037261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2440 , TFLOPS: 69.65716888880924, Tokens per sec: 172618.1378770763, Loss: 3.5063271522521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2441 , TFLOPS: 68.96613262835872, Tokens per sec: 170905.6738997516, Loss: 3.495300769805908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2442 , TFLOPS: 67.14155006610932, Tokens per sec: 166384.15731613582, Loss: 3.5026021003723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2443 , TFLOPS: 70.5261956884557, Tokens per sec: 174771.68201780503, Loss: 3.478567123413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2444 , TFLOPS: 69.5241129147783, Tokens per sec: 172288.41051610207, Loss: 3.4943106174468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2445 , TFLOPS: 68.14770684715604, Tokens per sec: 168877.52465688888, Loss: 3.491105079650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2446 , TFLOPS: 69.13998425477851, Tokens per sec: 171336.49738135748, Loss: 3.511427640914917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2447 , TFLOPS: 69.65834759039932, Tokens per sec: 172621.0588294612, Loss: 3.489039897918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2448 , TFLOPS: 67.99436883544037, Tokens per sec: 168497.53617228102, Loss: 3.5010271072387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2449 , TFLOPS: 67.81201008423818, Tokens per sec: 168045.63109832723, Loss: 3.494995594024658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2450 , TFLOPS: 70.5906726331225, Tokens per sec: 174931.4629894111, Loss: 3.4966821670532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2451 , TFLOPS: 69.61673971532525, Tokens per sec: 172517.94993154184, Loss: 3.497826337814331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2452 , TFLOPS: 68.2280695997446, Tokens per sec: 169076.67241049133, Loss: 3.493114471435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2453 , TFLOPS: 70.58970149863504, Tokens per sec: 174929.05641117194, Loss: 3.4796552658081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2454 , TFLOPS: 70.58197548337317, Tokens per sec: 174909.91049426218, Loss: 3.5053675174713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2455 , TFLOPS: 66.2565317221817, Tokens per sec: 164190.98436706437, Loss: 3.4797329902648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2456 , TFLOPS: 68.79277477367243, Tokens per sec: 170476.07403889482, Loss: 3.4913172721862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2457 , TFLOPS: 70.61159385268371, Tokens per sec: 174983.3080761464, Loss: 3.486605644226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2458 , TFLOPS: 68.89834891492316, Tokens per sec: 170737.69839086584, Loss: 3.5231173038482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2459 , TFLOPS: 68.04395341402349, Tokens per sec: 168620.41222020227, Loss: 3.509972095489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2460 , TFLOPS: 70.61841388592315, Tokens per sec: 175000.20886980245, Loss: 3.4912590980529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2461 , TFLOPS: 68.866192874266, Tokens per sec: 170658.01217983713, Loss: 3.502535820007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2462 , TFLOPS: 66.9961881215861, Tokens per sec: 166023.93440466718, Loss: 3.5097639560699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2463 , TFLOPS: 69.91039235583771, Tokens per sec: 173245.6535232405, Loss: 3.5049691200256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2464 , TFLOPS: 70.59016261052925, Tokens per sec: 174930.19909724142, Loss: 3.5084564685821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2465 , TFLOPS: 68.86812461344401, Tokens per sec: 170662.7992423188, Loss: 3.494647741317749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2466 , TFLOPS: 67.56012727467493, Tokens per sec: 167421.4377490474, Loss: 3.483726739883423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2467 , TFLOPS: 70.62554878530909, Tokens per sec: 175017.88993645486, Loss: 3.489863395690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2468 , TFLOPS: 68.90903074036709, Tokens per sec: 170764.1691310161, Loss: 3.495065212249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2469 , TFLOPS: 66.40135191762509, Tokens per sec: 164549.86476463458, Loss: 3.514803647994995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2470 , TFLOPS: 70.57724664122492, Tokens per sec: 174898.19190249383, Loss: 3.505387783050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2471 , TFLOPS: 70.5662133749112, Tokens per sec: 174870.85025315383, Loss: 3.532269239425659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2472 , TFLOPS: 67.7012189675649, Tokens per sec: 167771.0785065618, Loss: 3.494940757751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2473 , TFLOPS: 69.57209064089227, Tokens per sec: 172407.30460659534, Loss: 3.4913084506988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2474 , TFLOPS: 70.53336875835019, Tokens per sec: 174789.45767518244, Loss: 3.470362663269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2475 , TFLOPS: 67.35477735186991, Tokens per sec: 166912.55801917514, Loss: 3.5047385692596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2476 , TFLOPS: 68.78517024476689, Tokens per sec: 170457.22917855886, Loss: 3.519106864929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2477 , TFLOPS: 69.57668090521787, Tokens per sec: 172418.67978725355, Loss: 3.5140464305877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2478 , TFLOPS: 68.74622583485882, Tokens per sec: 170360.72064072575, Loss: 3.484344482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2479 , TFLOPS: 68.72841058708237, Tokens per sec: 170316.57249422415, Loss: 3.5142579078674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2480 , TFLOPS: 70.54640526533517, Tokens per sec: 174821.76357558102, Loss: 3.495936155319214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2481 , TFLOPS: 68.76463573904417, Tokens per sec: 170406.34241131536, Loss: 3.473573684692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2482 , TFLOPS: 67.74409967615907, Tokens per sec: 167877.34162615816, Loss: 3.4926252365112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2483 , TFLOPS: 68.95700736996501, Tokens per sec: 170883.0605042218, Loss: 3.527353286743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2484 , TFLOPS: 69.65819246374132, Tokens per sec: 172620.67440851405, Loss: 3.5055091381073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2485 , TFLOPS: 68.68076161284526, Tokens per sec: 170198.49308709678, Loss: 3.484456777572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2486 , TFLOPS: 68.01891129726569, Tokens per sec: 168558.35509625953, Loss: 3.5076234340667725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2487 , TFLOPS: 70.5674080306144, Tokens per sec: 174873.81074158856, Loss: 3.502023696899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2488 , TFLOPS: 67.8552411139194, Tokens per sec: 168152.76235216836, Loss: 3.4785232543945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2489 , TFLOPS: 67.21594066081616, Tokens per sec: 166568.50540461988, Loss: 3.5009875297546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2490 , TFLOPS: 70.56243750739728, Tokens per sec: 174861.49323750817, Loss: 3.4984889030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2491 , TFLOPS: 70.63456266343667, Tokens per sec: 175040.22731941563, Loss: 3.5151236057281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2492 , TFLOPS: 67.00607054127737, Tokens per sec: 166048.42412930055, Loss: 3.483840227127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2493 , TFLOPS: 70.5930958893796, Tokens per sec: 174937.46808535993, Loss: 3.4982290267944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2494 , TFLOPS: 70.57511770472361, Tokens per sec: 174892.91616332185, Loss: 3.464338779449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2495 , TFLOPS: 67.15405652595256, Tokens per sec: 166415.14970132764, Loss: 3.4899580478668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2496 , TFLOPS: 68.13167295178964, Tokens per sec: 168837.7908978916, Loss: 3.514317274093628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2497 , TFLOPS: 70.64001450420533, Tokens per sec: 175053.73758141018, Loss: 3.523897647857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2498 , TFLOPS: 69.70119710076291, Tokens per sec: 172727.2446364054, Loss: 3.4898157119750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2499 , TFLOPS: 67.34232646291508, Tokens per sec: 166881.70334476646, Loss: 3.4848718643188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2500 , TFLOPS: 70.57931197916244, Tokens per sec: 174903.31003741876, Loss: 3.4968149662017822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2501 , TFLOPS: 70.60908024238276, Tokens per sec: 174977.0790729236, Loss: 3.5077173709869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2502 , TFLOPS: 67.53757284340693, Tokens per sec: 167365.5453837896, Loss: 3.505824565887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2503 , TFLOPS: 67.83738097974778, Tokens per sec: 168108.50297223486, Loss: 3.4847583770751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2504 , TFLOPS: 70.61893798454089, Tokens per sec: 175001.50764391143, Loss: 3.4848084449768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2505 , TFLOPS: 68.81894963555528, Tokens per sec: 170540.93823003955, Loss: 3.49485445022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2506 , TFLOPS: 67.33246049294124, Tokens per sec: 166857.25438434494, Loss: 3.5226826667785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2507 , TFLOPS: 70.56815846984719, Tokens per sec: 174875.6704126751, Loss: 3.5002670288085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2508 , TFLOPS: 68.83003857276239, Tokens per sec: 170568.41783798605, Loss: 3.4947681427001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2509 , TFLOPS: 67.21438175671383, Tokens per sec: 166564.64226852686, Loss: 3.5146965980529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2510 , TFLOPS: 69.4077367772274, Tokens per sec: 172000.01762736109, Loss: 3.509115695953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2511 , TFLOPS: 70.61768094270423, Tokens per sec: 174998.39255576557, Loss: 3.493311643600464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2512 , TFLOPS: 68.81267839050055, Tokens per sec: 170525.39739395925, Loss: 3.51462459564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2513 , TFLOPS: 69.83571783176542, Tokens per sec: 173060.60182651147, Loss: 3.495076894760132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2514 , TFLOPS: 69.70854957306756, Tokens per sec: 172745.464872718, Loss: 3.517089605331421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2515 , TFLOPS: 68.23446645256206, Tokens per sec: 169092.5245164445, Loss: 3.4942870140075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2516 , TFLOPS: 67.94853823555034, Tokens per sec: 168383.9628970962, Loss: 3.4954187870025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2517 , TFLOPS: 70.5834064314725, Tokens per sec: 174913.45654128425, Loss: 3.4897947311401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2518 , TFLOPS: 69.57653959215499, Tokens per sec: 172418.32959791677, Loss: 3.519617795944214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2519 , TFLOPS: 67.25843664532673, Tokens per sec: 166673.81513555621, Loss: 3.4713521003723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2520 , TFLOPS: 70.5719958788452, Tokens per sec: 174885.17993489766, Loss: 3.4988853931427 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2521 , TFLOPS: 69.68960287544907, Tokens per sec: 172698.51286886836, Loss: 3.5070981979370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2522 , TFLOPS: 67.41972952431806, Tokens per sec: 167073.51665757413, Loss: 3.497379779815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2523 , TFLOPS: 68.87042028685725, Tokens per sec: 170668.48817392616, Loss: 3.494346857070923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2524 , TFLOPS: 70.6349802487544, Tokens per sec: 175041.26214183378, Loss: 3.4770140647888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2525 , TFLOPS: 67.83793002961079, Tokens per sec: 168109.86357827822, Loss: 3.514552116394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2526 , TFLOPS: 68.08559184849918, Tokens per sec: 168723.59684768523, Loss: 3.5102803707122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2527 , TFLOPS: 70.57094219665602, Tokens per sec: 174882.5687943602, Loss: 3.4933087825775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2528 , TFLOPS: 69.80270079864549, Tokens per sec: 172978.78198131383, Loss: 3.491333246231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2529 , TFLOPS: 67.28447703426774, Tokens per sec: 166738.3460879377, Loss: 3.499185085296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2530 , TFLOPS: 69.89733600325306, Tokens per sec: 173213.29844325976, Loss: 3.494746685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2531 , TFLOPS: 70.55768644894289, Tokens per sec: 174849.7195912859, Loss: 3.4965267181396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2532 , TFLOPS: 68.51086790337644, Tokens per sec: 169777.47775969593, Loss: 3.5046818256378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2533 , TFLOPS: 68.60623290498654, Tokens per sec: 170013.80273900795, Loss: 3.480987071990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2534 , TFLOPS: 70.51232318584454, Tokens per sec: 174737.30442815225, Loss: 3.5116796493530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2535 , TFLOPS: 68.93811678733898, Tokens per sec: 170836.24755950502, Loss: 3.4912190437316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2536 , TFLOPS: 66.48157880542809, Tokens per sec: 164748.6758303317, Loss: 3.5158097743988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2537 , TFLOPS: 70.5668419008895, Tokens per sec: 174872.40780976592, Loss: 3.4949960708618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2538 , TFLOPS: 70.5864756438378, Tokens per sec: 174921.0623876271, Loss: 3.5204355716705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2539 , TFLOPS: 67.19796372921225, Tokens per sec: 166523.9565877829, Loss: 3.5349838733673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2540 , TFLOPS: 69.67940220322632, Tokens per sec: 172673.23447940353, Loss: 3.504000186920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2541 , TFLOPS: 70.60040006873602, Tokens per sec: 174955.56864642695, Loss: 3.5206122398376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2542 , TFLOPS: 67.30705886358436, Tokens per sec: 166794.30634857816, Loss: 3.4885406494140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2543 , TFLOPS: 67.81522326715515, Tokens per sec: 168053.59371955678, Loss: 3.503272533416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2544 , TFLOPS: 70.59614881196032, Tokens per sec: 174945.0335638222, Loss: 3.4883956909179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2545 , TFLOPS: 68.78527344571485, Tokens per sec: 170457.48492187643, Loss: 3.4994585514068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2546 , TFLOPS: 68.20337927428658, Tokens per sec: 169015.48706414248, Loss: 3.515627145767212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2547 , TFLOPS: 69.69090273863324, Tokens per sec: 172701.73407303047, Loss: 3.475642204284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2548 , TFLOPS: 70.56846769299517, Tokens per sec: 174876.43670169954, Loss: 3.4720194339752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2549 , TFLOPS: 67.58871841251604, Tokens per sec: 167492.28973819152, Loss: 3.489461898803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2550 , TFLOPS: 68.98959799996635, Tokens per sec: 170963.82367551935, Loss: 3.5138003826141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2551 , TFLOPS: 69.54836641586746, Tokens per sec: 172348.51336354474, Loss: 3.5066821575164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2552 , TFLOPS: 68.74392408044126, Tokens per sec: 170355.01663971983, Loss: 3.499504804611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2553 , TFLOPS: 69.60225055119537, Tokens per sec: 172482.0441867148, Loss: 3.503031015396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2554 , TFLOPS: 70.54334672712051, Tokens per sec: 174814.18418095927, Loss: 3.499730348587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2555 , TFLOPS: 67.81494700512911, Tokens per sec: 168052.90911182348, Loss: 3.510505437850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2556 , TFLOPS: 67.11630514011847, Tokens per sec: 166321.59760857234, Loss: 3.4871809482574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2557 , TFLOPS: 70.57749153352125, Tokens per sec: 174898.79877258127, Loss: 3.4948670864105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2558 , TFLOPS: 69.13052751126564, Tokens per sec: 171313.06252918628, Loss: 3.486825942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2559 , TFLOPS: 67.20465179262796, Tokens per sec: 166540.530345678, Loss: 3.5075559616088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2560 , TFLOPS: 70.47071375260441, Tokens per sec: 174634.19166892572, Loss: 3.51837158203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2561 , TFLOPS: 70.58091308175072, Tokens per sec: 174907.27774600856, Loss: 3.4933624267578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2562 , TFLOPS: 66.95152171237373, Tokens per sec: 165913.2461222278, Loss: 3.512586832046509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2563 , TFLOPS: 68.15422072359436, Tokens per sec: 168893.66676026577, Loss: 3.504627227783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2564 , TFLOPS: 70.60273289124545, Tokens per sec: 174961.3496375869, Loss: 3.491424798965454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2565 , TFLOPS: 69.68547423562156, Tokens per sec: 172688.28164456855, Loss: 3.512248992919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2566 , TFLOPS: 64.45612993249034, Tokens per sec: 159729.39040157938, Loss: 3.493489980697632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2567 , TFLOPS: 70.57606179425578, Tokens per sec: 174895.25572118262, Loss: 3.4800095558166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2568 , TFLOPS: 70.5414798166033, Tokens per sec: 174809.55777104048, Loss: 3.5051159858703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2569 , TFLOPS: 66.39958612914887, Tokens per sec: 164545.48894627285, Loss: 3.489243507385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2570 , TFLOPS: 68.68870948116495, Tokens per sec: 170218.1887803229, Loss: 3.482253313064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2571 , TFLOPS: 70.58938659295903, Tokens per sec: 174928.27604021705, Loss: 3.499366044998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2572 , TFLOPS: 68.7142361978994, Tokens per sec: 170281.44679639087, Loss: 3.4842681884765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2573 , TFLOPS: 68.7213022657647, Tokens per sec: 170298.95729095276, Loss: 3.501303195953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2574 , TFLOPS: 70.55987462143867, Tokens per sec: 174855.14212377754, Loss: 3.5238635540008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2575 , TFLOPS: 68.93267715234228, Tokens per sec: 170822.76754475792, Loss: 3.510761260986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2576 , TFLOPS: 67.13280872454452, Tokens per sec: 166362.49530879996, Loss: 3.5069403648376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2577 , TFLOPS: 69.55992163190156, Tokens per sec: 172377.14846178936, Loss: 3.4889578819274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2578 , TFLOPS: 70.5616266734429, Tokens per sec: 174859.48390164852, Loss: 3.486701488494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2579 , TFLOPS: 67.97922527444925, Tokens per sec: 168460.0087599441, Loss: 3.4980735778808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2580 , TFLOPS: 68.97960385868382, Tokens per sec: 170939.0570924757, Loss: 3.5045578479766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2581 , TFLOPS: 70.59573128090054, Tokens per sec: 174943.99887586117, Loss: 3.5236868858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2582 , TFLOPS: 68.1922431642782, Tokens per sec: 168987.89055679794, Loss: 3.5042176246643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2583 , TFLOPS: 67.91558324592482, Tokens per sec: 168302.29680251234, Loss: 3.508955478668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2584 , TFLOPS: 69.39656241952618, Tokens per sec: 171972.32633802033, Loss: 3.480799913406372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2585 , TFLOPS: 70.5779588931712, Tokens per sec: 174899.95694127775, Loss: 3.4764719009399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2586 , TFLOPS: 66.7520036727753, Tokens per sec: 165418.8184413764, Loss: 3.461707353591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2587 , TFLOPS: 70.57085761146044, Tokens per sec: 174882.35918292773, Loss: 3.511683940887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2588 , TFLOPS: 69.71314223896819, Tokens per sec: 172756.84600474368, Loss: 3.4892287254333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2589 , TFLOPS: 67.23027680649757, Tokens per sec: 166604.03195287526, Loss: 3.4936845302581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2590 , TFLOPS: 68.75007435603855, Tokens per sec: 170370.25770015872, Loss: 3.500661849975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2591 , TFLOPS: 70.49469963738777, Tokens per sec: 174693.63133368225, Loss: 3.4948976039886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2592 , TFLOPS: 67.69856425798153, Tokens per sec: 167764.4998437738, Loss: 3.5203020572662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2593 , TFLOPS: 69.51994875893025, Tokens per sec: 172278.09127919096, Loss: 3.5157558917999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2594 , TFLOPS: 70.6275472768116, Tokens per sec: 175022.84241855008, Loss: 3.510977029800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2595 , TFLOPS: 68.00181227586081, Tokens per sec: 168515.98183760553, Loss: 3.504652976989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2596 , TFLOPS: 67.85228031870854, Tokens per sec: 168145.42517547728, Loss: 3.4761688709259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2597 , TFLOPS: 69.87967693032107, Tokens per sec: 173169.53731522683, Loss: 3.4875569343566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2598 , TFLOPS: 70.58683309870094, Tokens per sec: 174921.94820015487, Loss: 3.5101261138916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2599 , TFLOPS: 68.57070869836885, Tokens per sec: 169925.76984169488, Loss: 3.498858690261841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2600 , TFLOPS: 69.0868758186121, Tokens per sec: 171204.8888261599, Loss: 3.505913019180298 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/model.pt +[rank0]:[2024-08-25 01:59:32,373] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063717859993630555, 'preprocessing_with_comm': 0.0012455930009309668, 'state_converting': 1.0131874200014863, : 1.0221902989978844}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600, took 4.45s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000600 + eval ppl=24.304561614990234, eval loss=3.190664052963257 +------------------------------------------------------------------ +iteration: 2601 , TFLOPS: 65.14964621211884, Tokens per sec: 161448.00013962373, Loss: 3.4928524494171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2602 , TFLOPS: 66.71073104423944, Tokens per sec: 165316.54032130918, Loss: 3.51473331451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2603 , TFLOPS: 68.07789631954415, Tokens per sec: 168704.52647920218, Loss: 3.4877209663391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2604 , TFLOPS: 70.50413500637043, Tokens per sec: 174717.01321741295, Loss: 3.4994091987609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2605 , TFLOPS: 68.78049737351323, Tokens per sec: 170445.64928883302, Loss: 3.4913582801818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2606 , TFLOPS: 67.40084050518821, Tokens per sec: 167026.70758737312, Loss: 3.4950685501098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2607 , TFLOPS: 69.62218641827258, Tokens per sec: 172531.44746144916, Loss: 3.5108425617218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2608 , TFLOPS: 70.5120815440611, Tokens per sec: 174736.70561319383, Loss: 3.498570442199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2609 , TFLOPS: 67.49824905413031, Tokens per sec: 167268.09670208915, Loss: 3.4928226470947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2610 , TFLOPS: 68.09807938831192, Tokens per sec: 168754.54234695729, Loss: 3.5016286373138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2611 , TFLOPS: 70.53455089420922, Tokens per sec: 174792.38713806414, Loss: 3.490541696548462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2612 , TFLOPS: 68.64372556000954, Tokens per sec: 170106.71366831183, Loss: 3.5125434398651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2613 , TFLOPS: 68.36430080039023, Tokens per sec: 169414.26833279684, Loss: 3.492046594619751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2614 , TFLOPS: 69.49106647291048, Tokens per sec: 172206.51779278752, Loss: 3.4797921180725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2615 , TFLOPS: 68.94363292863218, Tokens per sec: 170849.9171652827, Loss: 3.518392324447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2616 , TFLOPS: 68.03026952655665, Tokens per sec: 168586.5020396548, Loss: 3.496814012527466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2617 , TFLOPS: 68.69642656025061, Tokens per sec: 170237.31255240363, Loss: 3.4912004470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2618 , TFLOPS: 70.52674358825392, Tokens per sec: 174773.03977386118, Loss: 3.5167694091796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2619 , TFLOPS: 67.04919841608572, Tokens per sec: 166155.29975400324, Loss: 3.4882705211639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2620 , TFLOPS: 69.6652029252433, Tokens per sec: 172638.04710438207, Loss: 3.483534812927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2621 , TFLOPS: 69.58092270394376, Tokens per sec: 172429.19143176897, Loss: 3.4839251041412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2622 , TFLOPS: 68.0271633916226, Tokens per sec: 168578.80469511918, Loss: 3.512770652770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2623 , TFLOPS: 68.34829311018257, Tokens per sec: 169374.59951306958, Loss: 3.4798147678375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2624 , TFLOPS: 70.52970772535778, Tokens per sec: 174780.38523213117, Loss: 3.517317533493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2625 , TFLOPS: 68.68366026042116, Tokens per sec: 170205.67625510224, Loss: 3.514169931411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2626 , TFLOPS: 68.21505472578738, Tokens per sec: 169044.42011911864, Loss: 3.5133538246154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2627 , TFLOPS: 69.90320900798766, Tokens per sec: 173227.85239595696, Loss: 3.482297420501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2628 , TFLOPS: 70.55399396055914, Tokens per sec: 174840.5691983098, Loss: 3.4675796031951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2629 , TFLOPS: 67.09598634870804, Tokens per sec: 166271.2454051576, Loss: 3.499934196472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2630 , TFLOPS: 68.90353145140915, Tokens per sec: 170750.54128718167, Loss: 3.499558210372925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2631 , TFLOPS: 70.58257748155513, Tokens per sec: 174911.40231207266, Loss: 3.475400924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2632 , TFLOPS: 67.81008205667159, Tokens per sec: 168040.85323362745, Loss: 3.4811477661132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2633 , TFLOPS: 68.6136468312401, Tokens per sec: 170032.17526497567, Loss: 3.4818837642669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2634 , TFLOPS: 70.56731263728429, Tokens per sec: 174873.5743464081, Loss: 3.4778244495391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2635 , TFLOPS: 68.93428315897273, Tokens per sec: 170826.74740610446, Loss: 3.496945858001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2636 , TFLOPS: 67.81386175785347, Tokens per sec: 168050.2197495246, Loss: 3.5086236000061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2637 , TFLOPS: 69.73588555447634, Tokens per sec: 172813.20644595593, Loss: 3.511082887649536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2638 , TFLOPS: 70.57914217609316, Tokens per sec: 174902.88924670772, Loss: 3.5036075115203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2639 , TFLOPS: 67.42105685407411, Tokens per sec: 167076.80592692707, Loss: 3.515984296798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2640 , TFLOPS: 68.97630245029896, Tokens per sec: 170930.87583881788, Loss: 3.4759297370910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2641 , TFLOPS: 70.57225397048742, Tokens per sec: 174885.81951441822, Loss: 3.518174648284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2642 , TFLOPS: 68.16981479648365, Tokens per sec: 168932.3105907144, Loss: 3.4831316471099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2643 , TFLOPS: 67.90989738492846, Tokens per sec: 168288.20661261407, Loss: 3.50361704826355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2644 , TFLOPS: 69.981090387881, Tokens per sec: 173420.85103467494, Loss: 3.4975907802581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2645 , TFLOPS: 70.53452742987345, Tokens per sec: 174792.3289908553, Loss: 3.4886474609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2646 , TFLOPS: 67.43696304178205, Tokens per sec: 167116.22321227856, Loss: 3.4961400032043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2647 , TFLOPS: 68.70161304987047, Tokens per sec: 170250.16524502065, Loss: 3.487700939178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2648 , TFLOPS: 70.57429744457461, Tokens per sec: 174890.88346831355, Loss: 3.490243673324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2649 , TFLOPS: 67.27980976447412, Tokens per sec: 166726.7800792469, Loss: 3.4951822757720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2650 , TFLOPS: 68.87998692541736, Tokens per sec: 170692.19535813032, Loss: 3.5017547607421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2651 , TFLOPS: 69.58227611117697, Tokens per sec: 172432.545323983, Loss: 3.4970312118530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2652 , TFLOPS: 68.7198798788629, Tokens per sec: 170295.43245952122, Loss: 3.491161584854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2653 , TFLOPS: 68.41937376112539, Tokens per sec: 169550.74519628604, Loss: 3.519885778427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2654 , TFLOPS: 69.62652799840411, Tokens per sec: 172542.20637527967, Loss: 3.490204334259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2655 , TFLOPS: 68.95678907794218, Tokens per sec: 170882.5195525424, Loss: 3.530341625213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2656 , TFLOPS: 68.1132870578903, Tokens per sec: 168792.2286274356, Loss: 3.5113446712493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2657 , TFLOPS: 69.69297616751307, Tokens per sec: 172706.87225820174, Loss: 3.4949419498443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2658 , TFLOPS: 69.52182510841996, Tokens per sec: 172282.74107992343, Loss: 3.506455898284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2659 , TFLOPS: 68.75949821248086, Tokens per sec: 170393.61105454626, Loss: 3.48588228225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2660 , TFLOPS: 68.26956030826375, Tokens per sec: 169179.49095678068, Loss: 3.515272617340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2661 , TFLOPS: 70.59308085740489, Tokens per sec: 174937.4308344707, Loss: 3.4939091205596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2662 , TFLOPS: 68.11216573392772, Tokens per sec: 168789.4498631323, Loss: 3.494382858276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2663 , TFLOPS: 68.35497087683338, Tokens per sec: 169391.14775444704, Loss: 3.5007848739624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2664 , TFLOPS: 69.70783119133588, Tokens per sec: 172743.68464365063, Loss: 3.4922983646392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2665 , TFLOPS: 70.50318857772189, Tokens per sec: 174714.66786296415, Loss: 3.500577926635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2666 , TFLOPS: 66.20764049426727, Tokens per sec: 164069.82651847904, Loss: 3.4841058254241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2667 , TFLOPS: 69.63564781675433, Tokens per sec: 172564.80628978243, Loss: 3.496198892593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2668 , TFLOPS: 70.60707931176174, Tokens per sec: 174972.12054642403, Loss: 3.5103657245635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2669 , TFLOPS: 67.30331655330413, Tokens per sec: 166785.0324914542, Loss: 3.49983811378479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2670 , TFLOPS: 68.26029017775649, Tokens per sec: 169156.5185521945, Loss: 3.4916021823883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2671 , TFLOPS: 70.5760623776113, Tokens per sec: 174895.25716680184, Loss: 3.491481065750122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2672 , TFLOPS: 68.70536811326153, Tokens per sec: 170259.47070575805, Loss: 3.5050764083862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2673 , TFLOPS: 69.51265608545238, Tokens per sec: 172260.01923095848, Loss: 3.497468948364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2674 , TFLOPS: 69.87755793059928, Tokens per sec: 173164.28620048915, Loss: 3.505715847015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2675 , TFLOPS: 69.75710409286853, Tokens per sec: 172865.78832150673, Loss: 3.4966585636138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2676 , TFLOPS: 67.40129349498292, Tokens per sec: 167027.83014598538, Loss: 3.4896678924560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2677 , TFLOPS: 69.03048376585171, Tokens per sec: 171065.14310732225, Loss: 3.517605781555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2678 , TFLOPS: 70.59456821623628, Tokens per sec: 174941.11667350098, Loss: 3.495293378829956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2679 , TFLOPS: 68.467548195344, Tokens per sec: 169670.12675113173, Loss: 3.505525588989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2680 , TFLOPS: 67.06028171959329, Tokens per sec: 166182.7654010217, Loss: 3.484370231628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2681 , TFLOPS: 70.58251470861524, Tokens per sec: 174911.24675381256, Loss: 3.4873886108398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2682 , TFLOPS: 68.92804431655694, Tokens per sec: 170811.28686733285, Loss: 3.480480909347534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2683 , TFLOPS: 68.13914886247183, Tokens per sec: 168856.3170281016, Loss: 3.518754482269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2684 , TFLOPS: 68.83703291742358, Tokens per sec: 170585.75059454716, Loss: 3.48895525932312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2685 , TFLOPS: 70.61294246486307, Tokens per sec: 174986.65008568874, Loss: 3.512550115585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2686 , TFLOPS: 67.16820937694935, Tokens per sec: 166450.22202516295, Loss: 3.479999303817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2687 , TFLOPS: 69.7050203833196, Tokens per sec: 172736.71915175027, Loss: 3.5155935287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2688 , TFLOPS: 69.5257689949848, Tokens per sec: 172292.5144652861, Loss: 3.4892444610595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2689 , TFLOPS: 67.47818988429492, Tokens per sec: 167218.38786953705, Loss: 3.487212896347046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2690 , TFLOPS: 68.1102052669766, Tokens per sec: 168784.59161006418, Loss: 3.5179073810577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2691 , TFLOPS: 69.55011095142132, Tokens per sec: 172352.83651482363, Loss: 3.505326271057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2692 , TFLOPS: 69.82818532414558, Tokens per sec: 173041.9354428553, Loss: 3.5002212524414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2693 , TFLOPS: 68.94751261726734, Tokens per sec: 170859.53146110362, Loss: 3.532744884490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2694 , TFLOPS: 69.66015262243823, Tokens per sec: 172625.5318976943, Loss: 3.513479709625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2695 , TFLOPS: 69.58816664352442, Tokens per sec: 172447.14271203568, Loss: 3.5131795406341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2696 , TFLOPS: 67.51577447650487, Tokens per sec: 167311.52662931796, Loss: 3.482192039489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2697 , TFLOPS: 68.80186220246866, Tokens per sec: 170498.59368851472, Loss: 3.5255908966064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2698 , TFLOPS: 70.57165684247394, Tokens per sec: 174884.33976542176, Loss: 3.4965555667877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2699 , TFLOPS: 67.1215573084516, Tokens per sec: 166334.61306027594, Loss: 3.4935355186462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2700 , TFLOPS: 68.0452546780449, Tokens per sec: 168623.63689579334, Loss: 3.4936258792877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2701 , TFLOPS: 70.57564149024292, Tokens per sec: 174894.21416153846, Loss: 3.491675853729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2702 , TFLOPS: 68.8889183260906, Tokens per sec: 170714.32835287362, Loss: 3.5058977603912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2703 , TFLOPS: 67.20059204021052, Tokens per sec: 166530.46983196898, Loss: 3.4913511276245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2704 , TFLOPS: 69.71003913958383, Tokens per sec: 172749.15618263462, Loss: 3.473881959915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2705 , TFLOPS: 70.52412389455701, Tokens per sec: 174766.54788429712, Loss: 3.4783711433410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2706 , TFLOPS: 66.99832186696493, Tokens per sec: 166029.2220607674, Loss: 3.49568510055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2707 , TFLOPS: 69.14631272878364, Tokens per sec: 171352.18003708113, Loss: 3.501133918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2708 , TFLOPS: 70.57414786367815, Tokens per sec: 174890.5127903744, Loss: 3.5239627361297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2709 , TFLOPS: 67.78717191408383, Tokens per sec: 167984.07937653438, Loss: 3.5022335052490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2710 , TFLOPS: 67.8955140620626, Tokens per sec: 168252.56315409867, Loss: 3.5065722465515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2711 , TFLOPS: 69.99843019920334, Tokens per sec: 173463.8209972696, Loss: 3.492612838745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2712 , TFLOPS: 70.58323480196512, Tokens per sec: 174913.03122445842, Loss: 3.5223026275634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2713 , TFLOPS: 67.24359414647832, Tokens per sec: 166637.03378837364, Loss: 3.4877283573150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2714 , TFLOPS: 69.51413303855249, Tokens per sec: 172263.6792834404, Loss: 3.495392322540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2715 , TFLOPS: 70.5140252525002, Tokens per sec: 174741.5223368231, Loss: 3.515320062637329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2716 , TFLOPS: 67.24973239507148, Tokens per sec: 166652.24504457138, Loss: 3.4990909099578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2717 , TFLOPS: 67.79780080899364, Tokens per sec: 168010.41894899038, Loss: 3.5241470336914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2718 , TFLOPS: 70.56026848206365, Tokens per sec: 174856.11815379356, Loss: 3.5052151679992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2719 , TFLOPS: 68.79965716680222, Tokens per sec: 170493.1293672286, Loss: 3.5050582885742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2720 , TFLOPS: 68.11135057935421, Tokens per sec: 168787.42982030247, Loss: 3.5245275497436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2721 , TFLOPS: 69.58662889460489, Tokens per sec: 172443.33200081604, Loss: 3.505258560180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2722 , TFLOPS: 68.85850369292228, Tokens per sec: 170638.95754143412, Loss: 3.5174169540405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2723 , TFLOPS: 67.9679028069501, Tokens per sec: 168431.9504382084, Loss: 3.4926798343658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2724 , TFLOPS: 69.6775368798091, Tokens per sec: 172668.61200249335, Loss: 3.493298053741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2725 , TFLOPS: 69.57055878581963, Tokens per sec: 172403.50850097756, Loss: 3.465085983276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2726 , TFLOPS: 68.83632567370982, Tokens per sec: 170583.99796671513, Loss: 3.50990629196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2727 , TFLOPS: 68.41138478471079, Tokens per sec: 169530.94763267224, Loss: 3.489888906478882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2728 , TFLOPS: 69.66433244595926, Tokens per sec: 172635.88996082498, Loss: 3.4902446269989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2729 , TFLOPS: 68.7568665218271, Tokens per sec: 170387.08943520157, Loss: 3.4804975986480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2730 , TFLOPS: 68.23098620925873, Tokens per sec: 169083.9000901584, Loss: 3.4916200637817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2731 , TFLOPS: 69.75797569727868, Tokens per sec: 172867.94825325024, Loss: 3.5005128383636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2732 , TFLOPS: 69.50259430632235, Tokens per sec: 172235.0850338778, Loss: 3.4975125789642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2733 , TFLOPS: 67.917545271427, Tokens per sec: 168307.1589178417, Loss: 3.50014591217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2734 , TFLOPS: 69.5964321964352, Tokens per sec: 172467.6256626744, Loss: 3.514112949371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2735 , TFLOPS: 70.60434426514001, Tokens per sec: 174965.34279960516, Loss: 3.490220069885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2736 , TFLOPS: 65.90804868270214, Tokens per sec: 163327.40500665814, Loss: 3.4923484325408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2737 , TFLOPS: 68.82525272129794, Tokens per sec: 170556.55797085207, Loss: 3.496880054473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2738 , TFLOPS: 70.54650933499647, Tokens per sec: 174822.02147166594, Loss: 3.471040725708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2739 , TFLOPS: 68.80219278940999, Tokens per sec: 170499.41291937255, Loss: 3.495267868041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2740 , TFLOPS: 67.20159791828385, Tokens per sec: 166532.9625086413, Loss: 3.497457504272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2741 , TFLOPS: 70.54206986418205, Tokens per sec: 174811.01997393972, Loss: 3.4860715866088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2742 , TFLOPS: 69.64127593458292, Tokens per sec: 172578.75338575005, Loss: 3.5055675506591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2743 , TFLOPS: 67.45011113441488, Tokens per sec: 167148.8056342047, Loss: 3.4823412895202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2744 , TFLOPS: 69.06470022566103, Tokens per sec: 171149.9352639839, Loss: 3.466297149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2745 , TFLOPS: 70.55688607686886, Tokens per sec: 174847.7361811182, Loss: 3.4893672466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2746 , TFLOPS: 68.75413413902461, Tokens per sec: 170380.3182896202, Loss: 3.4889590740203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2747 , TFLOPS: 67.19214728081589, Tokens per sec: 166509.54278792202, Loss: 3.4844484329223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2748 , TFLOPS: 70.60076003716732, Tokens per sec: 174956.46068785348, Loss: 3.494168996810913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2749 , TFLOPS: 68.76061608097169, Tokens per sec: 170396.3812558099, Loss: 3.477576494216919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2750 , TFLOPS: 66.95800857424412, Tokens per sec: 165929.32128052943, Loss: 3.471165418624878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2751 , TFLOPS: 69.63667177599999, Tokens per sec: 172567.34377359156, Loss: 3.490030288696289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2752 , TFLOPS: 70.55732243435871, Tokens per sec: 174848.81752304678, Loss: 3.4976065158843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2753 , TFLOPS: 67.9436698487624, Tokens per sec: 168371.8984983386, Loss: 3.516669988632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2754 , TFLOPS: 68.79648140670747, Tokens per sec: 170485.25948387495, Loss: 3.493638515472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2755 , TFLOPS: 70.53994464707392, Tokens per sec: 174805.7534518338, Loss: 3.5262088775634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2756 , TFLOPS: 68.13523197562306, Tokens per sec: 168846.61055100962, Loss: 3.5125317573547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2757 , TFLOPS: 67.67973661491486, Tokens per sec: 167717.8428702185, Loss: 3.5157594680786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2758 , TFLOPS: 69.5756947887931, Tokens per sec: 172416.23608212324, Loss: 3.4854068756103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2759 , TFLOPS: 69.73914298807742, Tokens per sec: 172821.27872525534, Loss: 3.4909446239471436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2760 , TFLOPS: 68.24747878551601, Tokens per sec: 169124.77051093124, Loss: 3.4914753437042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2761 , TFLOPS: 69.93372383131288, Tokens per sec: 173303.47148964214, Loss: 3.514190912246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2762 , TFLOPS: 69.28440815122764, Tokens per sec: 171694.39570636398, Loss: 3.51229190826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2763 , TFLOPS: 67.44865262572887, Tokens per sec: 167145.19128901276, Loss: 3.4963226318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2764 , TFLOPS: 68.78023614606211, Tokens per sec: 170445.00193843257, Loss: 3.5073652267456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2765 , TFLOPS: 69.52264105171588, Tokens per sec: 172284.7630772943, Loss: 3.485924243927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2766 , TFLOPS: 68.8306024127051, Tokens per sec: 170569.8150954762, Loss: 3.4917373657226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2767 , TFLOPS: 68.09944111341352, Tokens per sec: 168757.91685176542, Loss: 3.5024781227111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2768 , TFLOPS: 70.59199122096402, Tokens per sec: 174934.73059533685, Loss: 3.4930121898651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2769 , TFLOPS: 67.96883212928608, Tokens per sec: 168434.25340133032, Loss: 3.5221805572509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2770 , TFLOPS: 67.96996138296524, Tokens per sec: 168437.0518163448, Loss: 3.50917387008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2771 , TFLOPS: 69.70484796980085, Tokens per sec: 172736.29189205787, Loss: 3.4856209754943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2772 , TFLOPS: 70.56432230577893, Tokens per sec: 174866.16397552562, Loss: 3.4820609092712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2773 , TFLOPS: 67.7439724356808, Tokens per sec: 167877.0263102363, Loss: 3.4917092323303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2774 , TFLOPS: 69.05058119858455, Tokens per sec: 171114.94675954917, Loss: 3.4937126636505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2775 , TFLOPS: 70.57262603182586, Tokens per sec: 174886.74152340082, Loss: 3.4953625202178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2776 , TFLOPS: 68.03649051624625, Tokens per sec: 168601.9183374042, Loss: 3.4789767265319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2777 , TFLOPS: 68.80053960043769, Tokens per sec: 170495.31613498536, Loss: 3.485274314880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2778 , TFLOPS: 69.83339204460144, Tokens per sec: 173054.83826971252, Loss: 3.5136687755584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2779 , TFLOPS: 70.60487426578737, Tokens per sec: 174966.65619959368, Loss: 3.4778783321380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2780 , TFLOPS: 66.48998461936274, Tokens per sec: 164769.5063632386, Loss: 3.4908769130706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2781 , TFLOPS: 69.6724948375651, Tokens per sec: 172656.1172663859, Loss: 3.52377986907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2782 , TFLOPS: 70.57534576097062, Tokens per sec: 174893.48131182446, Loss: 3.515072822570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2783 , TFLOPS: 67.21413032735828, Tokens per sec: 166564.01919888656, Loss: 3.489525079727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2784 , TFLOPS: 67.91251440568537, Tokens per sec: 168294.69187833834, Loss: 3.5192270278930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2785 , TFLOPS: 70.62862552690883, Tokens per sec: 175025.51444107006, Loss: 3.4811997413635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2786 , TFLOPS: 68.90003511521127, Tokens per sec: 170741.87698093007, Loss: 3.5023350715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2787 , TFLOPS: 67.12788185500035, Tokens per sec: 166350.28598333005, Loss: 3.4888358116149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2788 , TFLOPS: 70.60491826572263, Tokens per sec: 174966.76523628025, Loss: 3.4887733459472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2789 , TFLOPS: 69.67118992157049, Tokens per sec: 172652.88354080293, Loss: 3.4892311096191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2790 , TFLOPS: 67.41602941788855, Tokens per sec: 167064.34738624142, Loss: 3.498767614364624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2791 , TFLOPS: 68.7419013545281, Tokens per sec: 170350.00410208537, Loss: 3.4964210987091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2792 , TFLOPS: 70.58283848297135, Tokens per sec: 174912.04910233372, Loss: 3.5003316402435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2793 , TFLOPS: 68.82753445116799, Tokens per sec: 170562.21234882335, Loss: 3.4905576705932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2794 , TFLOPS: 68.69601665116625, Tokens per sec: 170236.29675253737, Loss: 3.502009868621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2795 , TFLOPS: 69.48699095355737, Tokens per sec: 172196.418206875, Loss: 3.4947264194488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2796 , TFLOPS: 68.8423085312768, Tokens per sec: 170598.82414683225, Loss: 3.4885170459747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2797 , TFLOPS: 68.15992055454576, Tokens per sec: 168907.79156925154, Loss: 3.4770114421844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2798 , TFLOPS: 69.527971661026, Tokens per sec: 172297.97291437988, Loss: 3.49735951423645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2799 , TFLOPS: 69.44740963501167, Tokens per sec: 172098.33133927657, Loss: 3.5179526805877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2800 , TFLOPS: 67.28542125667138, Tokens per sec: 166740.6859750686, Loss: 3.489549398422241 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/optimizer.pt +[rank0]:[2024-08-25 02:25:15,123] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006389483001839835, 'preprocessing_with_comm': 0.001262481997400755, 'state_converting': 0.9992713719984749, : 1.0082938089981326}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0000800 + eval ppl=26.203641891479492, eval loss=3.2658984661102295 +------------------------------------------------------------------ +iteration: 2801 , TFLOPS: 64.10296308037043, Tokens per sec: 158854.20403748582, Loss: 3.5108299255371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2802 , TFLOPS: 69.16592944226844, Tokens per sec: 171400.79241405585, Loss: 3.4810361862182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2803 , TFLOPS: 67.18493014076161, Tokens per sec: 166491.65791983393, Loss: 3.502918004989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2804 , TFLOPS: 68.82790236448702, Tokens per sec: 170563.12407855692, Loss: 3.4897360801696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2805 , TFLOPS: 70.56060809554212, Tokens per sec: 174856.95975340475, Loss: 3.5107104778289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2806 , TFLOPS: 67.8105477467561, Tokens per sec: 168042.00726495683, Loss: 3.459271192550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2807 , TFLOPS: 68.05667519535379, Tokens per sec: 168651.93819575864, Loss: 3.503671646118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2808 , TFLOPS: 70.55870182055915, Tokens per sec: 174852.2357940034, Loss: 3.492417812347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2809 , TFLOPS: 68.74064377351156, Tokens per sec: 170346.8876777924, Loss: 3.5010125637054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2810 , TFLOPS: 66.8558596277054, Tokens per sec: 165676.1849383711, Loss: 3.5006957054138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2811 , TFLOPS: 69.60408822771093, Tokens per sec: 172486.59815155715, Loss: 3.5044455528259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2812 , TFLOPS: 70.60412159547398, Tokens per sec: 174964.79099964362, Loss: 3.518535852432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2813 , TFLOPS: 67.39257731575512, Tokens per sec: 167006.23049369245, Loss: 3.515263795852661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2814 , TFLOPS: 68.66841511749747, Tokens per sec: 170167.8971115462, Loss: 3.5163729190826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2815 , TFLOPS: 70.5805170052359, Tokens per sec: 174906.29622477337, Loss: 3.4927613735198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2816 , TFLOPS: 68.09883533687851, Tokens per sec: 168756.41567077985, Loss: 3.498518228530884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2817 , TFLOPS: 68.14385573097233, Tokens per sec: 168867.9811667502, Loss: 3.501214027404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2818 , TFLOPS: 70.54034422513733, Tokens per sec: 174806.74365029208, Loss: 3.5048418045043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2819 , TFLOPS: 69.69955588604557, Tokens per sec: 172723.17752554113, Loss: 3.514192819595337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2820 , TFLOPS: 68.14618164755224, Tokens per sec: 168873.74504425607, Loss: 3.488508701324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2821 , TFLOPS: 69.03471374821764, Tokens per sec: 171075.625469596, Loss: 3.499134063720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2822 , TFLOPS: 70.60597395146031, Tokens per sec: 174969.3813418316, Loss: 3.5091211795806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2823 , TFLOPS: 67.54076499051855, Tokens per sec: 167373.45587597624, Loss: 3.49645733833313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2824 , TFLOPS: 67.80012034448968, Tokens per sec: 168016.1670134675, Loss: 3.4981844425201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2825 , TFLOPS: 70.5625005267193, Tokens per sec: 174861.6494063303, Loss: 3.4779293537139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2826 , TFLOPS: 68.71798930557269, Tokens per sec: 170290.74741064422, Loss: 3.5167808532714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2827 , TFLOPS: 67.98035909820882, Tokens per sec: 168462.8185001213, Loss: 3.502328872680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2828 , TFLOPS: 69.55221504467639, Tokens per sec: 172358.0506896947, Loss: 3.482501983642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2829 , TFLOPS: 68.85367660471597, Tokens per sec: 170626.9954851112, Loss: 3.494675397872925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2830 , TFLOPS: 67.98202809117356, Tokens per sec: 168466.95444854267, Loss: 3.496731758117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2831 , TFLOPS: 69.74301852110153, Tokens per sec: 172830.88272301428, Loss: 3.502626419067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2832 , TFLOPS: 69.50204324401857, Tokens per sec: 172233.7194407845, Loss: 3.4914042949676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2833 , TFLOPS: 68.80409610884901, Tokens per sec: 170504.12955461038, Loss: 3.4927291870117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2834 , TFLOPS: 68.86889026585933, Tokens per sec: 170664.69661334757, Loss: 3.4911699295043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2835 , TFLOPS: 70.60178765495486, Tokens per sec: 174959.00723793247, Loss: 3.5168981552124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2836 , TFLOPS: 67.20734125942442, Tokens per sec: 166547.1951406689, Loss: 3.493539571762085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2837 , TFLOPS: 68.51862267135282, Tokens per sec: 169796.6949289996, Loss: 3.496251106262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2838 , TFLOPS: 69.93438684928059, Tokens per sec: 173305.11452120362, Loss: 3.51352858543396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2839 , TFLOPS: 69.53575308952682, Tokens per sec: 172317.2561514, Loss: 3.5346572399139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2840 , TFLOPS: 67.40566412285521, Tokens per sec: 167038.66104331732, Loss: 3.5057644844055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2841 , TFLOPS: 69.67340217839477, Tokens per sec: 172658.3657569137, Loss: 3.4956743717193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2842 , TFLOPS: 70.53031517448836, Tokens per sec: 174781.8905579928, Loss: 3.5297698974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2843 , TFLOPS: 67.12536732989112, Tokens per sec: 166344.05471311155, Loss: 3.511044979095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2844 , TFLOPS: 68.81441616753602, Tokens per sec: 170529.70379688512, Loss: 3.504779815673828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2845 , TFLOPS: 70.55522625461501, Tokens per sec: 174843.62295872867, Loss: 3.460930585861206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2846 , TFLOPS: 68.86657464673188, Tokens per sec: 170658.95825406324, Loss: 3.503664255142212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2847 , TFLOPS: 67.08640580589825, Tokens per sec: 166247.50376469106, Loss: 3.50185489654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2848 , TFLOPS: 70.61603525969295, Tokens per sec: 174994.31437197668, Loss: 3.4689671993255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2849 , TFLOPS: 69.62555846999608, Tokens per sec: 172539.80377708215, Loss: 3.4991307258605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2850 , TFLOPS: 67.61785718871978, Tokens per sec: 167564.49883552556, Loss: 3.492096185684204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2851 , TFLOPS: 69.61093790201812, Tokens per sec: 172503.5723990443, Loss: 3.49031400680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2852 , TFLOPS: 70.58988904671126, Tokens per sec: 174929.52117596494, Loss: 3.489178419113159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2853 , TFLOPS: 68.15463677370094, Tokens per sec: 168894.6977782616, Loss: 3.5032143592834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2854 , TFLOPS: 67.75837316690124, Tokens per sec: 167912.71290857208, Loss: 3.470646619796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2855 , TFLOPS: 70.6086417284182, Tokens per sec: 174975.99238701316, Loss: 3.5009210109710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2856 , TFLOPS: 68.33556001267617, Tokens per sec: 169343.04549478227, Loss: 3.5227959156036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2857 , TFLOPS: 68.34198875373843, Tokens per sec: 169358.97662331886, Loss: 3.4909274578094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2858 , TFLOPS: 68.61849242814968, Tokens per sec: 170044.18318790416, Loss: 3.5226664543151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2859 , TFLOPS: 70.62889950416466, Tokens per sec: 175026.1933868911, Loss: 3.4982378482818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2860 , TFLOPS: 67.22260910441655, Tokens per sec: 166585.03054245136, Loss: 3.488701105117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2861 , TFLOPS: 68.8219176268516, Tokens per sec: 170548.29323942307, Loss: 3.491121530532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2862 , TFLOPS: 70.61366356538439, Tokens per sec: 174988.43705221638, Loss: 3.4948389530181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2863 , TFLOPS: 67.46136924180983, Tokens per sec: 167176.7044645123, Loss: 3.4804561138153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2864 , TFLOPS: 68.75413143365263, Tokens per sec: 170380.3115854104, Loss: 3.4956188201904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2865 , TFLOPS: 69.52123139177176, Tokens per sec: 172281.26978466593, Loss: 3.490046739578247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2866 , TFLOPS: 68.69973100535752, Tokens per sec: 170245.5013313932, Loss: 3.4984893798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2867 , TFLOPS: 68.09088925622065, Tokens per sec: 168736.72440757666, Loss: 3.4853954315185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2868 , TFLOPS: 70.61805451928838, Tokens per sec: 174999.31831969396, Loss: 3.5117955207824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2869 , TFLOPS: 69.66671787559085, Tokens per sec: 172641.80131822886, Loss: 3.477348804473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2870 , TFLOPS: 67.23415675380491, Tokens per sec: 166613.64688971432, Loss: 3.5031917095184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2871 , TFLOPS: 69.11027551711187, Tokens per sec: 171262.87585673193, Loss: 3.4917261600494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2872 , TFLOPS: 70.58133169843211, Tokens per sec: 174908.31512425948, Loss: 3.4957799911499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2873 , TFLOPS: 67.70994829357348, Tokens per sec: 167792.7107380257, Loss: 3.5055747032165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2874 , TFLOPS: 68.6669896377966, Tokens per sec: 170164.3646158179, Loss: 3.5063083171844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2875 , TFLOPS: 70.56563831048591, Tokens per sec: 174869.4251801594, Loss: 3.5123491287231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2876 , TFLOPS: 68.73376563242236, Tokens per sec: 170329.84288648362, Loss: 3.4699034690856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2877 , TFLOPS: 67.90590082122374, Tokens per sec: 168278.30268749595, Loss: 3.5025033950805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2878 , TFLOPS: 69.70525027540337, Tokens per sec: 172737.28884965854, Loss: 3.4886302947998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2879 , TFLOPS: 70.58627811020627, Tokens per sec: 174920.57287752806, Loss: 3.4916439056396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2880 , TFLOPS: 66.1965285172081, Tokens per sec: 164042.28981524106, Loss: 3.5052053928375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2881 , TFLOPS: 69.6408522904944, Tokens per sec: 172577.70354903038, Loss: 3.505627393722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2882 , TFLOPS: 70.62747426719794, Tokens per sec: 175022.66149268474, Loss: 3.47442364692688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2883 , TFLOPS: 68.20677885744425, Tokens per sec: 169023.91160570222, Loss: 3.5204057693481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2884 , TFLOPS: 67.99091718785226, Tokens per sec: 168488.98260932733, Loss: 3.5292770862579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2885 , TFLOPS: 70.58805203736249, Tokens per sec: 174924.9688644354, Loss: 3.509960412979126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2886 , TFLOPS: 69.7285680803508, Tokens per sec: 172795.0729389865, Loss: 3.486584186553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2887 , TFLOPS: 68.38231807581974, Tokens per sec: 169458.9171260779, Loss: 3.506227731704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2888 , TFLOPS: 69.48758805277714, Tokens per sec: 172197.8978845175, Loss: 3.4852373600006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2889 , TFLOPS: 70.59637210914006, Tokens per sec: 174945.5869188318, Loss: 3.4881439208984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2890 , TFLOPS: 67.32592830086186, Tokens per sec: 166841.0668927919, Loss: 3.4673423767089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2891 , TFLOPS: 67.90814757285611, Tokens per sec: 168283.8703855411, Loss: 3.504798412322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2892 , TFLOPS: 70.53036577146648, Tokens per sec: 174782.0159428783, Loss: 3.477051258087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2893 , TFLOPS: 68.73360677072333, Tokens per sec: 170329.4492096933, Loss: 3.492234468460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2894 , TFLOPS: 68.61175225550716, Tokens per sec: 170027.48029760466, Loss: 3.5017940998077393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2895 , TFLOPS: 69.38542041241354, Tokens per sec: 171944.7152170018, Loss: 3.515540361404419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2896 , TFLOPS: 68.92404485093033, Tokens per sec: 170801.3757509326, Loss: 3.4695918560028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2897 , TFLOPS: 68.1136093345217, Tokens per sec: 168793.0272644299, Loss: 3.494640350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2898 , TFLOPS: 68.60434905076468, Tokens per sec: 170009.1343407225, Loss: 3.4767942428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2899 , TFLOPS: 70.57355501221264, Tokens per sec: 174889.04363913488, Loss: 3.5154612064361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2900 , TFLOPS: 67.84273288705964, Tokens per sec: 168121.7655881149, Loss: 3.5056347846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2901 , TFLOPS: 69.0108640154798, Tokens per sec: 171016.5232046067, Loss: 3.472752809524536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2902 , TFLOPS: 69.53748298195309, Tokens per sec: 172321.54301539567, Loss: 3.468987464904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2903 , TFLOPS: 68.28571859145964, Tokens per sec: 169219.53296252192, Loss: 3.492764949798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2904 , TFLOPS: 68.90678162062521, Tokens per sec: 170758.59556453474, Loss: 3.4741370677948 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2905 , TFLOPS: 70.01164533625409, Tokens per sec: 173496.56956264886, Loss: 3.4982309341430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2906 , TFLOPS: 69.64818633764932, Tokens per sec: 172595.8781257923, Loss: 3.4988794326782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2907 , TFLOPS: 67.44986999442074, Tokens per sec: 167148.20806273544, Loss: 3.5350654125213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2908 , TFLOPS: 69.69778198964558, Tokens per sec: 172718.78161485112, Loss: 3.50077748298645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2909 , TFLOPS: 70.62517476204607, Tokens per sec: 175016.963065607, Loss: 3.5272626876831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2910 , TFLOPS: 66.43801874321488, Tokens per sec: 164640.72919763034, Loss: 3.4960074424743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2911 , TFLOPS: 68.85203058423771, Tokens per sec: 170622.91646504196, Loss: 3.4943926334381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2912 , TFLOPS: 70.6055018987584, Tokens per sec: 174968.21154323537, Loss: 3.511594533920288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2913 , TFLOPS: 67.83482441534161, Tokens per sec: 168102.16752400756, Loss: 3.4906415939331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2914 , TFLOPS: 68.73299533440355, Tokens per sec: 170327.93400313766, Loss: 3.4825892448425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2915 , TFLOPS: 70.55385684837016, Tokens per sec: 174840.22941920135, Loss: 3.5053844451904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2916 , TFLOPS: 68.96482417639395, Tokens per sec: 170902.431411645, Loss: 3.523916006088257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2917 , TFLOPS: 66.67344875838975, Tokens per sec: 165224.15070999085, Loss: 3.5200748443603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2918 , TFLOPS: 69.64998968861586, Tokens per sec: 172600.3470281429, Loss: 3.5047287940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2919 , TFLOPS: 70.58631555589557, Tokens per sec: 174920.6656720711, Loss: 3.4853148460388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2920 , TFLOPS: 68.72345961415493, Tokens per sec: 170304.30343791525, Loss: 3.4903531074523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2921 , TFLOPS: 67.94457044887622, Tokens per sec: 168374.13028462994, Loss: 3.5013535022735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2922 , TFLOPS: 70.51528778265806, Tokens per sec: 174744.65102563344, Loss: 3.5260848999023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2923 , TFLOPS: 68.9707228396717, Tokens per sec: 170917.04894903867, Loss: 3.4952235221862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2924 , TFLOPS: 67.3370089347262, Tokens per sec: 166868.5259241403, Loss: 3.5184450149536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2925 , TFLOPS: 69.71978967625026, Tokens per sec: 172773.31908660434, Loss: 3.50766658782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2926 , TFLOPS: 70.55809606391023, Tokens per sec: 174850.7346622978, Loss: 3.4800539016723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2927 , TFLOPS: 67.28030185218482, Tokens per sec: 166727.99952680196, Loss: 3.5018041133880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2928 , TFLOPS: 68.77582802289174, Tokens per sec: 170434.07812362153, Loss: 3.5148515701293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2929 , TFLOPS: 70.56973729057013, Tokens per sec: 174879.582904348, Loss: 3.495307445526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2930 , TFLOPS: 68.14747026346356, Tokens per sec: 168876.9383764371, Loss: 3.4928600788116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2931 , TFLOPS: 68.1746323711881, Tokens per sec: 168944.24907153164, Loss: 3.5003230571746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2932 , TFLOPS: 69.47146478839889, Tokens per sec: 172157.942659264, Loss: 3.4973912239074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2933 , TFLOPS: 68.69509127843361, Tokens per sec: 170234.00357696766, Loss: 3.4840524196624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2934 , TFLOPS: 69.67708045204613, Tokens per sec: 172667.48092421706, Loss: 3.5180952548980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2935 , TFLOPS: 68.92586460203299, Tokens per sec: 170805.88529462714, Loss: 3.490739583969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2936 , TFLOPS: 69.79523884353237, Tokens per sec: 172960.29043454133, Loss: 3.4825730323791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2937 , TFLOPS: 68.29183353651644, Tokens per sec: 169234.68646998945, Loss: 3.469911813735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2938 , TFLOPS: 69.14161050940005, Tokens per sec: 171340.52741945643, Loss: 3.5117805004119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2939 , TFLOPS: 69.57419361399344, Tokens per sec: 172412.51600560168, Loss: 3.486420154571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2940 , TFLOPS: 68.80361480366673, Tokens per sec: 170502.93682734857, Loss: 3.5038204193115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2941 , TFLOPS: 68.0722464856683, Tokens per sec: 168690.52556847726, Loss: 3.5090792179107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2942 , TFLOPS: 70.58920221933418, Tokens per sec: 174927.81914206425, Loss: 3.4957828521728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2943 , TFLOPS: 67.96031401910267, Tokens per sec: 168413.1445859485, Loss: 3.4881675243377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2944 , TFLOPS: 68.01462172942911, Tokens per sec: 168547.72507462557, Loss: 3.508652448654175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2945 , TFLOPS: 69.68584637933061, Tokens per sec: 172689.2038576746, Loss: 3.50708270072937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2946 , TFLOPS: 70.59529268797611, Tokens per sec: 174942.91199427948, Loss: 3.5148258209228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2947 , TFLOPS: 67.171808380848, Tokens per sec: 166459.14075328683, Loss: 3.5049521923065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2948 , TFLOPS: 69.66368259591178, Tokens per sec: 172634.27956082043, Loss: 3.496471881866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2949 , TFLOPS: 70.55679024183247, Tokens per sec: 174847.49869134094, Loss: 3.485750675201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2950 , TFLOPS: 67.07239364353481, Tokens per sec: 166212.78008278736, Loss: 3.501569986343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2951 , TFLOPS: 67.89656625931816, Tokens per sec: 168255.17061480705, Loss: 3.5081732273101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2952 , TFLOPS: 70.52321587673312, Tokens per sec: 174764.29771610312, Loss: 3.498516321182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2953 , TFLOPS: 69.57962630920476, Tokens per sec: 172425.97882279605, Loss: 3.503706216812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2954 , TFLOPS: 68.10667330978659, Tokens per sec: 168775.8390310701, Loss: 3.4831130504608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2955 , TFLOPS: 69.66582490853642, Tokens per sec: 172639.5884474995, Loss: 3.5046682357788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2956 , TFLOPS: 70.61221909063927, Tokens per sec: 174984.85748467594, Loss: 3.4956295490264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2957 , TFLOPS: 67.46307461362183, Tokens per sec: 167180.93056372463, Loss: 3.4995169639587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2958 , TFLOPS: 67.34409643204866, Tokens per sec: 166886.08952325766, Loss: 3.4800069332122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2959 , TFLOPS: 70.57393734140675, Tokens per sec: 174889.99109299493, Loss: 3.4687845706939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2960 , TFLOPS: 68.69730899515149, Tokens per sec: 170239.49932329622, Loss: 3.5007107257843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2961 , TFLOPS: 68.10200033549695, Tokens per sec: 168764.25888600977, Loss: 3.503328800201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2962 , TFLOPS: 69.63009632334652, Tokens per sec: 172551.04907757038, Loss: 3.5356388092041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2963 , TFLOPS: 69.00378324637961, Tokens per sec: 170998.97627876536, Loss: 3.512686014175415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2964 , TFLOPS: 67.91469020942966, Tokens per sec: 168300.0837597013, Loss: 3.5033228397369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2965 , TFLOPS: 68.67678277381792, Tokens per sec: 170188.6330856817, Loss: 3.4850778579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2966 , TFLOPS: 70.6298445902968, Tokens per sec: 175028.53541443637, Loss: 3.5010948181152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2967 , TFLOPS: 67.97670959803806, Tokens per sec: 168453.77463667252, Loss: 3.5022401809692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2968 , TFLOPS: 69.17272437137997, Tokens per sec: 171417.63099691746, Loss: 3.510624647140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2969 , TFLOPS: 69.60704659586521, Tokens per sec: 172493.92931373458, Loss: 3.495560884475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2970 , TFLOPS: 68.12051487699436, Tokens per sec: 168810.13995938547, Loss: 3.4773447513580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2971 , TFLOPS: 68.74301842777498, Tokens per sec: 170352.7723326469, Loss: 3.4941678047180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2972 , TFLOPS: 69.0450470755634, Tokens per sec: 171101.23259306882, Loss: 3.51212215423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2973 , TFLOPS: 70.52397534358617, Tokens per sec: 174766.17975862714, Loss: 3.4799962043762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2974 , TFLOPS: 68.01679180378606, Tokens per sec: 168553.10275793556, Loss: 3.4911861419677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2975 , TFLOPS: 69.61576136059288, Tokens per sec: 172515.52546074038, Loss: 3.4671263694763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2976 , TFLOPS: 69.52113731787817, Tokens per sec: 172281.0366591945, Loss: 3.4916141033172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2977 , TFLOPS: 67.39948471266693, Tokens per sec: 167023.34778415255, Loss: 3.50770902633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2978 , TFLOPS: 68.74898587072967, Tokens per sec: 170367.56031365675, Loss: 3.516660213470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2979 , TFLOPS: 70.59689720793664, Tokens per sec: 174946.8881714941, Loss: 3.5047414302825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2980 , TFLOPS: 67.91471670241864, Tokens per sec: 168300.1494122464, Loss: 3.4970269203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2981 , TFLOPS: 68.07474708444174, Tokens per sec: 168696.72232770151, Loss: 3.473705530166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2982 , TFLOPS: 70.62417608271828, Tokens per sec: 175014.4882282191, Loss: 3.481769323348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2983 , TFLOPS: 69.00607715307322, Tokens per sec: 171004.6608322468, Loss: 3.5075125694274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2984 , TFLOPS: 67.12839866979233, Tokens per sec: 166351.5667073164, Loss: 3.483323335647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2985 , TFLOPS: 69.68102212765042, Tokens per sec: 172677.2488305759, Loss: 3.492000102996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2986 , TFLOPS: 70.59149802715172, Tokens per sec: 174933.50840674023, Loss: 3.4991538524627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2987 , TFLOPS: 67.97197732309499, Tokens per sec: 168442.04753806046, Loss: 3.5121538639068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2988 , TFLOPS: 68.3524694161943, Tokens per sec: 169384.94885942477, Loss: 3.485381603240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2989 , TFLOPS: 70.62453970406, Tokens per sec: 175015.38932196013, Loss: 3.4932539463043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2990 , TFLOPS: 68.81848134370401, Tokens per sec: 170539.77775124548, Loss: 3.4725301265716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2991 , TFLOPS: 68.04579473174775, Tokens per sec: 168624.97520836085, Loss: 3.5046627521514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2992 , TFLOPS: 69.65441033069743, Tokens per sec: 172611.30186619493, Loss: 3.5010154247283936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2993 , TFLOPS: 70.57366892688562, Tokens per sec: 174889.32593224326, Loss: 3.5089070796966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2994 , TFLOPS: 67.77706715481654, Tokens per sec: 167959.03866993895, Loss: 3.511951208114624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2995 , TFLOPS: 68.62783566660632, Tokens per sec: 170067.3367620424, Loss: 3.4926531314849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2996 , TFLOPS: 70.64282202906737, Tokens per sec: 175060.69493729275, Loss: 3.507476806640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2997 , TFLOPS: 67.20661477605468, Tokens per sec: 166545.3948348499, Loss: 3.502288341522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2998 , TFLOPS: 67.97138522641394, Tokens per sec: 168440.58025726, Loss: 3.4931085109710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2999 , TFLOPS: 70.60083708702733, Tokens per sec: 174956.65162589433, Loss: 3.4889276027679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3000 , TFLOPS: 68.16852567774015, Tokens per sec: 168929.11601245982, Loss: 3.494131565093994 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/model.pt +[rank0]:[2024-08-25 02:50:57,115] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063444109982810915, 'preprocessing_with_comm': 0.0015481349983019754, 'state_converting': 1.0000540609980817, : 1.009367912000016}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000, took 4.32s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001000 + eval ppl=25.515811920166016, eval loss=3.2392983436584473 +------------------------------------------------------------------ +iteration: 3001 , TFLOPS: 63.89593511160918, Tokens per sec: 158341.165924882, Loss: 3.501107931137085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3002 , TFLOPS: 69.10033964694594, Tokens per sec: 171238.25367593428, Loss: 3.5204339027404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3003 , TFLOPS: 69.79579989252191, Tokens per sec: 172961.68077574202, Loss: 3.478092908859253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3004 , TFLOPS: 67.40251838789669, Tokens per sec: 167030.86556555878, Loss: 3.510280132293701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3005 , TFLOPS: 69.64610255409103, Tokens per sec: 172590.71428058716, Loss: 3.499790668487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3006 , TFLOPS: 69.51580301922782, Tokens per sec: 172267.81767951744, Loss: 3.498436212539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3007 , TFLOPS: 68.07210850443772, Tokens per sec: 168690.18363578795, Loss: 3.49940824508667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3008 , TFLOPS: 68.7069600591932, Tokens per sec: 170263.41572314405, Loss: 3.5014359951019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3009 , TFLOPS: 70.53970333287592, Tokens per sec: 174805.15544866817, Loss: 3.499251127243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3010 , TFLOPS: 67.11717739695042, Tokens per sec: 166323.75915708893, Loss: 3.50815749168396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3011 , TFLOPS: 68.89817317053979, Tokens per sec: 170737.262876924, Loss: 3.4899230003356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3012 , TFLOPS: 69.62511810549022, Tokens per sec: 172538.7125053265, Loss: 3.515861988067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3013 , TFLOPS: 68.65874597634782, Tokens per sec: 170143.93591463353, Loss: 3.5039539337158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3014 , TFLOPS: 67.44654944691334, Tokens per sec: 167139.97938022358, Loss: 3.4693477153778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3015 , TFLOPS: 69.7188154922989, Tokens per sec: 172770.90495145763, Loss: 3.5026938915252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3016 , TFLOPS: 70.54549127809514, Tokens per sec: 174819.49861451623, Loss: 3.4902381896972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3017 , TFLOPS: 66.8131014596106, Tokens per sec: 165570.22548763113, Loss: 3.5027406215667725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3018 , TFLOPS: 68.77922679996013, Tokens per sec: 170442.50066760322, Loss: 3.4985544681549072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3019 , TFLOPS: 70.55428000515629, Tokens per sec: 174841.27804833194, Loss: 3.488737106323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3020 , TFLOPS: 68.56422788910125, Tokens per sec: 169909.70968240968, Loss: 3.485158681869507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3021 , TFLOPS: 67.96903319899599, Tokens per sec: 168434.7516742211, Loss: 3.4672298431396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3022 , TFLOPS: 70.5601601119153, Tokens per sec: 174855.84960062802, Loss: 3.5102062225341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3023 , TFLOPS: 69.77205088055662, Tokens per sec: 172902.8281079224, Loss: 3.4869911670684814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3024 , TFLOPS: 67.16795220859218, Tokens per sec: 166449.58473364444, Loss: 3.509448528289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3025 , TFLOPS: 69.15205204326703, Tokens per sec: 171366.4026906087, Loss: 3.5009925365448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3026 , TFLOPS: 70.45714995491183, Tokens per sec: 174600.57908406784, Loss: 3.504580497741699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3027 , TFLOPS: 68.65602413747102, Tokens per sec: 170137.19089806153, Loss: 3.497993230819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3028 , TFLOPS: 67.71178074945104, Tokens per sec: 167797.25176555198, Loss: 3.508969306945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3029 , TFLOPS: 70.5772447595288, Tokens per sec: 174898.18723944356, Loss: 3.497542381286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3030 , TFLOPS: 68.17349188219006, Tokens per sec: 168941.42281415933, Loss: 3.4880800247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3031 , TFLOPS: 68.98601510669421, Tokens per sec: 170954.94487130275, Loss: 3.501286029815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3032 , TFLOPS: 69.52541429007863, Tokens per sec: 172291.6354674546, Loss: 3.4925026893615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3033 , TFLOPS: 70.59787786022486, Tokens per sec: 174949.31833589208, Loss: 3.501654624938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3034 , TFLOPS: 66.45002270847404, Tokens per sec: 164670.47634588848, Loss: 3.5199689865112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3035 , TFLOPS: 68.8133775996591, Tokens per sec: 170527.1301112784, Loss: 3.50720477104187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3036 , TFLOPS: 70.59128648841767, Tokens per sec: 174932.9841904543, Loss: 3.509826421737671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3037 , TFLOPS: 68.19817024147326, Tokens per sec: 169002.5785070088, Loss: 3.5120792388916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3038 , TFLOPS: 68.40877493561364, Tokens per sec: 169524.48013911088, Loss: 3.5050337314605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3039 , TFLOPS: 69.5763194173949, Tokens per sec: 172417.78398060706, Loss: 3.495851516723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3040 , TFLOPS: 69.66398850809075, Tokens per sec: 172635.03764489896, Loss: 3.5009660720825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3041 , TFLOPS: 68.14485160194974, Tokens per sec: 168870.44904473645, Loss: 3.494096279144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3042 , TFLOPS: 69.99363848778755, Tokens per sec: 173451.94660853065, Loss: 3.478147506713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3043 , TFLOPS: 68.82885575589385, Tokens per sec: 170565.48668748778, Loss: 3.4883010387420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3044 , TFLOPS: 68.1690172906654, Tokens per sec: 168930.33428344422, Loss: 3.506239414215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3045 , TFLOPS: 68.77225293359155, Tokens per sec: 170425.21865850704, Loss: 3.4808547496795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3046 , TFLOPS: 70.57938670522316, Tokens per sec: 174903.49521682897, Loss: 3.495790958404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3047 , TFLOPS: 67.8630038684438, Tokens per sec: 168171.99931301782, Loss: 3.5009987354278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3048 , TFLOPS: 68.75716103921522, Tokens per sec: 170387.8192817331, Loss: 3.5023703575134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3049 , TFLOPS: 70.60260289575332, Tokens per sec: 174961.02749443724, Loss: 3.5105862617492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3050 , TFLOPS: 67.93835766941439, Tokens per sec: 168358.73433272933, Loss: 3.510585308074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3051 , TFLOPS: 68.46905525536195, Tokens per sec: 169673.86141187264, Loss: 3.4862561225891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3052 , TFLOPS: 69.59754414144777, Tokens per sec: 172470.3811849064, Loss: 3.485462188720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3053 , TFLOPS: 70.59381058861166, Tokens per sec: 174939.2391887876, Loss: 3.494271755218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3054 , TFLOPS: 66.22097048059118, Tokens per sec: 164102.85969300888, Loss: 3.474003791809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3055 , TFLOPS: 69.16101098477886, Tokens per sec: 171388.60393458387, Loss: 3.4994261264801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3056 , TFLOPS: 70.60035886114997, Tokens per sec: 174955.46652948932, Loss: 3.4954724311828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3057 , TFLOPS: 66.89552154373955, Tokens per sec: 165774.47153542444, Loss: 3.4874725341796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3058 , TFLOPS: 68.92776248507177, Tokens per sec: 170810.5884578663, Loss: 3.491236686706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3059 , TFLOPS: 69.9120998847527, Tokens per sec: 173249.8849679917, Loss: 3.4865195751190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3060 , TFLOPS: 69.64153898912102, Tokens per sec: 172579.40526387325, Loss: 3.4907515048980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3061 , TFLOPS: 67.07143122524914, Tokens per sec: 166210.3951042563, Loss: 3.4990546703338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3062 , TFLOPS: 69.759429948573, Tokens per sec: 172871.55204815656, Loss: 3.5105929374694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3063 , TFLOPS: 69.80052212098568, Tokens per sec: 172973.38297807166, Loss: 3.495558738708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3064 , TFLOPS: 68.15694965962447, Tokens per sec: 168900.4293643655, Loss: 3.5026888847351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3065 , TFLOPS: 67.3576986922579, Tokens per sec: 166919.79742246878, Loss: 3.500495672225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3066 , TFLOPS: 70.63612306416363, Tokens per sec: 175044.09416430935, Loss: 3.4952025413513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3067 , TFLOPS: 68.68304984607644, Tokens per sec: 170204.16358111333, Loss: 3.5109331607818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3068 , TFLOPS: 68.6702809284205, Tokens per sec: 170172.52079654948, Loss: 3.5131611824035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3069 , TFLOPS: 69.19408092469577, Tokens per sec: 171470.5548886526, Loss: 3.495049476623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3070 , TFLOPS: 68.85302998889019, Tokens per sec: 170625.39309986428, Loss: 3.5085134506225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3071 , TFLOPS: 68.14942854278105, Tokens per sec: 168881.7912083081, Loss: 3.506457567214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3072 , TFLOPS: 65.74725631992546, Tokens per sec: 162928.94381895653, Loss: 3.4839861392974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3073 , TFLOPS: 70.58197098553887, Tokens per sec: 174909.8993481333, Loss: 3.5060620307922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3074 , TFLOPS: 67.79003331191235, Tokens per sec: 167991.1702355625, Loss: 3.4868264198303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3075 , TFLOPS: 68.33521899754473, Tokens per sec: 169342.20042172066, Loss: 3.4845590591430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3076 , TFLOPS: 69.58487518263209, Tokens per sec: 172438.98610936012, Loss: 3.4739737510681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3077 , TFLOPS: 68.2182386235845, Tokens per sec: 169052.3101685952, Loss: 3.4935901165008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3078 , TFLOPS: 69.02580768963492, Tokens per sec: 171053.5552753447, Loss: 3.5085694789886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3079 , TFLOPS: 69.60655761537919, Tokens per sec: 172492.7175662246, Loss: 3.5005359649658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3080 , TFLOPS: 68.78333281327603, Tokens per sec: 170452.6758209147, Loss: 3.4913580417633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3081 , TFLOPS: 67.94024128038403, Tokens per sec: 168363.402128209, Loss: 3.479247808456421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3082 , TFLOPS: 69.5321980983842, Tokens per sec: 172308.44649173398, Loss: 3.509099245071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3083 , TFLOPS: 70.59839414170968, Tokens per sec: 174950.5977382846, Loss: 3.4839673042297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3084 , TFLOPS: 67.36103594234179, Tokens per sec: 166928.06749580512, Loss: 3.503307342529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3085 , TFLOPS: 68.34197776165205, Tokens per sec: 169358.9493837179, Loss: 3.514073610305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3086 , TFLOPS: 70.5063696511589, Tokens per sec: 174722.55091336524, Loss: 3.498436450958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3087 , TFLOPS: 67.86420631584585, Tokens per sec: 168174.97911014062, Loss: 3.50706148147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3088 , TFLOPS: 69.55945041227714, Tokens per sec: 172375.98072765092, Loss: 3.481722354888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3089 , TFLOPS: 69.96889579082128, Tokens per sec: 173390.63147981517, Loss: 3.5123891830444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3090 , TFLOPS: 69.78212510798058, Tokens per sec: 172927.79315324672, Loss: 3.502788782119751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3091 , TFLOPS: 67.51709472467053, Tokens per sec: 167314.7983497098, Loss: 3.4869987964630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3092 , TFLOPS: 69.08592080107385, Tokens per sec: 171202.52218749627, Loss: 3.4898574352264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3093 , TFLOPS: 70.58051116207898, Tokens per sec: 174906.28174478686, Loss: 3.510010242462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3094 , TFLOPS: 67.02782719646493, Tokens per sec: 166102.33951754877, Loss: 3.4859416484832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3095 , TFLOPS: 68.04423782407792, Tokens per sec: 168621.1170196475, Loss: 3.489525079727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3096 , TFLOPS: 70.61893022289857, Tokens per sec: 175001.4884097067, Loss: 3.474142074584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3097 , TFLOPS: 68.21528731404824, Tokens per sec: 169044.99649845078, Loss: 3.4969024658203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3098 , TFLOPS: 67.23500196157599, Tokens per sec: 166615.74140767835, Loss: 3.498689889907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3099 , TFLOPS: 69.70664811624115, Tokens per sec: 172740.75285323942, Loss: 3.492800235748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3100 , TFLOPS: 70.57469009270874, Tokens per sec: 174891.8564936437, Loss: 3.5057897567749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3101 , TFLOPS: 67.24869018183377, Tokens per sec: 166649.66232535907, Loss: 3.517974376678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3102 , TFLOPS: 69.65620518777249, Tokens per sec: 172615.7497197468, Loss: 3.5056581497192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3103 , TFLOPS: 70.58175191970454, Tokens per sec: 174909.35647886374, Loss: 3.527163505554199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3104 , TFLOPS: 68.20321890985963, Tokens per sec: 169015.0896634269, Loss: 3.505427837371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3105 , TFLOPS: 68.06613046727377, Tokens per sec: 168675.36940116054, Loss: 3.489225387573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3106 , TFLOPS: 69.57354527995957, Tokens per sec: 172410.90936244585, Loss: 3.4872522354125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3107 , TFLOPS: 69.69259596617051, Tokens per sec: 172705.93007739226, Loss: 3.5069408416748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3108 , TFLOPS: 68.86498807802757, Tokens per sec: 170655.02656203898, Loss: 3.484102725982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3109 , TFLOPS: 68.7255624912671, Tokens per sec: 170309.51459905022, Loss: 3.500349283218384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3110 , TFLOPS: 69.80300945149845, Tokens per sec: 172979.54685708438, Loss: 3.506666898727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3111 , TFLOPS: 68.12242084748209, Tokens per sec: 168814.86316421404, Loss: 3.5226287841796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3112 , TFLOPS: 68.77500192675389, Tokens per sec: 170432.03096638992, Loss: 3.509162425994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3113 , TFLOPS: 69.58077588687337, Tokens per sec: 172428.82760289533, Loss: 3.5169355869293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3114 , TFLOPS: 68.0067982456096, Tokens per sec: 168528.33761989296, Loss: 3.4743752479553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3115 , TFLOPS: 67.96350888682986, Tokens per sec: 168421.06182012023, Loss: 3.489865779876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3116 , TFLOPS: 70.59648035520954, Tokens per sec: 174945.8551645162, Loss: 3.494499921798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3117 , TFLOPS: 67.986176186132, Tokens per sec: 168477.23388480005, Loss: 3.487351655960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3118 , TFLOPS: 68.39045091746718, Tokens per sec: 169479.07120358673, Loss: 3.5006442070007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3119 , TFLOPS: 69.63509746349466, Tokens per sec: 172563.44245377852, Loss: 3.523635149002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3120 , TFLOPS: 70.52528925190941, Tokens per sec: 174769.43576819106, Loss: 3.5203146934509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3121 , TFLOPS: 67.12415351065671, Tokens per sec: 166341.0467353355, Loss: 3.500624656677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3122 , TFLOPS: 69.01737253160013, Tokens per sec: 171032.65202452513, Loss: 3.4964613914489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3123 , TFLOPS: 70.59719013367364, Tokens per sec: 174947.61407373883, Loss: 3.4925880432128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3124 , TFLOPS: 67.17330750081007, Tokens per sec: 166462.8557376945, Loss: 3.4961953163146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3125 , TFLOPS: 68.93865458982478, Tokens per sec: 170837.58029330868, Loss: 3.5197932720184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3126 , TFLOPS: 69.97832207929319, Tokens per sec: 173413.99086103996, Loss: 3.490584373474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3127 , TFLOPS: 69.7141033944446, Tokens per sec: 172759.22785389426, Loss: 3.536369800567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3128 , TFLOPS: 67.85108024174596, Tokens per sec: 168142.45125256467, Loss: 3.4943432807922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3129 , TFLOPS: 69.63371848152876, Tokens per sec: 172560.02518455964, Loss: 3.518151044845581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3130 , TFLOPS: 70.61557729462461, Tokens per sec: 174993.17948408832, Loss: 3.5001723766326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3131 , TFLOPS: 66.5667601528657, Tokens per sec: 164959.76459278213, Loss: 3.486799478530884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3132 , TFLOPS: 68.78912069157745, Tokens per sec: 170467.01882093307, Loss: 3.4777586460113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3133 , TFLOPS: 70.57429874282143, Tokens per sec: 174890.88668551215, Loss: 3.5012364387512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3134 , TFLOPS: 68.06932588404615, Tokens per sec: 168683.28799592672, Loss: 3.507239580154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3135 , TFLOPS: 67.44768627053445, Tokens per sec: 167142.79655438816, Loss: 3.4673585891723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3136 , TFLOPS: 70.57170344437584, Tokens per sec: 174884.4552500683, Loss: 3.4728353023529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3137 , TFLOPS: 68.86227907066579, Tokens per sec: 170648.3133433747, Loss: 3.476872444152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3138 , TFLOPS: 68.07195055964041, Tokens per sec: 168689.79223118117, Loss: 3.491208076477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3139 , TFLOPS: 68.74592586779907, Tokens per sec: 170359.97728930757, Loss: 3.5007753372192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3140 , TFLOPS: 70.59157434138945, Tokens per sec: 174933.697521828, Loss: 3.525283098220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3141 , TFLOPS: 68.77063795984019, Tokens per sec: 170421.21657564715, Loss: 3.500121831893921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3142 , TFLOPS: 68.12162331071903, Tokens per sec: 168812.88678025923, Loss: 3.5050392150878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3143 , TFLOPS: 69.6167155577414, Tokens per sec: 172517.89006638786, Loss: 3.491025924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3144 , TFLOPS: 68.02734047209414, Tokens per sec: 168579.24352003468, Loss: 3.509554386138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3145 , TFLOPS: 68.95674059673487, Tokens per sec: 170882.39941077, Loss: 3.515239715576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3146 , TFLOPS: 68.76230820107138, Tokens per sec: 170400.57451581932, Loss: 3.499356746673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3147 , TFLOPS: 69.72766530801758, Tokens per sec: 172792.8357696962, Loss: 3.506032705307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3148 , TFLOPS: 68.57220896166338, Tokens per sec: 169929.48765940638, Loss: 3.490145683288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3149 , TFLOPS: 69.63138467231632, Tokens per sec: 172554.2417482409, Loss: 3.462099552154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3150 , TFLOPS: 69.57296608480647, Tokens per sec: 172409.4740530528, Loss: 3.4807236194610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3151 , TFLOPS: 67.2267902834207, Tokens per sec: 166595.39196461852, Loss: 3.5222105979919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3152 , TFLOPS: 68.87568238251625, Tokens per sec: 170681.52822663682, Loss: 3.511080265045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3153 , TFLOPS: 70.5639307135274, Tokens per sec: 174865.19356678883, Loss: 3.5025086402893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3154 , TFLOPS: 67.15137980522633, Tokens per sec: 166408.5164925029, Loss: 3.5035791397094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3155 , TFLOPS: 67.90684736669265, Tokens per sec: 168280.64833143848, Loss: 3.5156238079071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3156 , TFLOPS: 70.54007729725427, Tokens per sec: 174806.08217359334, Loss: 3.507791042327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3157 , TFLOPS: 69.82199290875644, Tokens per sec: 173026.58995537012, Loss: 3.4822187423706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3158 , TFLOPS: 67.3034167673601, Tokens per sec: 166785.2808329244, Loss: 3.4977049827575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3159 , TFLOPS: 69.1345280283032, Tokens per sec: 171322.97625109868, Loss: 3.4907989501953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3160 , TFLOPS: 70.58563782283285, Tokens per sec: 174918.9861748833, Loss: 3.488248825073242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3161 , TFLOPS: 67.43603859498556, Tokens per sec: 167113.93233128064, Loss: 3.521958112716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3162 , TFLOPS: 67.95530316574104, Tokens per sec: 168400.72713932733, Loss: 3.4734585285186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3163 , TFLOPS: 70.57744918601884, Tokens per sec: 174898.69383080534, Loss: 3.492636203765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3164 , TFLOPS: 68.0752288638237, Tokens per sec: 168697.9162300824, Loss: 3.5003135204315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3165 , TFLOPS: 68.78900838939113, Tokens per sec: 170466.74052374423, Loss: 3.494865894317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3166 , TFLOPS: 69.731183318058, Tokens per sec: 172801.55378611718, Loss: 3.500147819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3167 , TFLOPS: 70.61357770986437, Tokens per sec: 174988.22429278, Loss: 3.4864954948425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3168 , TFLOPS: 68.08663095343145, Tokens per sec: 168726.17186417428, Loss: 3.5096726417541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3169 , TFLOPS: 69.57765201714248, Tokens per sec: 172421.08630957955, Loss: 3.4899911880493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3170 , TFLOPS: 70.59938641519493, Tokens per sec: 174953.05670128897, Loss: 3.481689929962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3171 , TFLOPS: 68.00580614353059, Tokens per sec: 168525.8790816521, Loss: 3.497488021850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3172 , TFLOPS: 68.06101792928494, Tokens per sec: 168662.69996883167, Loss: 3.5034499168395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3173 , TFLOPS: 70.52336687631262, Tokens per sec: 174764.67190969508, Loss: 3.5026304721832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3174 , TFLOPS: 68.88782251779651, Tokens per sec: 170711.61281920763, Loss: 3.455594062805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3175 , TFLOPS: 68.23429668172145, Tokens per sec: 169092.1038055997, Loss: 3.5179898738861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3176 , TFLOPS: 68.97271601376055, Tokens per sec: 170921.9882539971, Loss: 3.4832231998443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3177 , TFLOPS: 69.7984355706839, Tokens per sec: 172968.21227657137, Loss: 3.506666421890259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3178 , TFLOPS: 67.97698052893061, Tokens per sec: 168454.44603326975, Loss: 3.510725736618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3179 , TFLOPS: 68.92730992265282, Tokens per sec: 170809.4669583383, Loss: 3.49050235748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3180 , TFLOPS: 69.63289518492405, Tokens per sec: 172557.98496487958, Loss: 3.5032193660736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3181 , TFLOPS: 68.77423879827586, Tokens per sec: 170430.13984995583, Loss: 3.524721145629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3182 , TFLOPS: 68.00014343797203, Tokens per sec: 168511.84627347946, Loss: 3.507192850112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3183 , TFLOPS: 69.61660592886554, Tokens per sec: 172517.6183939569, Loss: 3.4961817264556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3184 , TFLOPS: 68.90827742400494, Tokens per sec: 170762.30233008607, Loss: 3.5205156803131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3185 , TFLOPS: 68.22941372862701, Tokens per sec: 169080.00330992998, Loss: 3.497535467147827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3186 , TFLOPS: 69.74275952264638, Tokens per sec: 172830.24089631136, Loss: 3.5257551670074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3187 , TFLOPS: 69.64650332983722, Tokens per sec: 172591.7074470353, Loss: 3.504457950592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3188 , TFLOPS: 68.83061124488553, Tokens per sec: 170569.83698259224, Loss: 3.4948649406433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3189 , TFLOPS: 68.88917044789164, Tokens per sec: 170714.95313847015, Loss: 3.5157971382141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3190 , TFLOPS: 70.55107558808463, Tokens per sec: 174833.33714983272, Loss: 3.497014284133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3191 , TFLOPS: 67.01953803986656, Tokens per sec: 166081.798074372, Loss: 3.4961276054382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3192 , TFLOPS: 68.83870497421519, Tokens per sec: 170589.89413546905, Loss: 3.489102363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3193 , TFLOPS: 69.98493196803587, Tokens per sec: 173430.37089348366, Loss: 3.4945929050445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3194 , TFLOPS: 68.91612196644583, Tokens per sec: 170781.74197040225, Loss: 3.4882380962371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3195 , TFLOPS: 67.16462386329879, Tokens per sec: 166441.33672735989, Loss: 3.5108683109283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3196 , TFLOPS: 69.61516489422027, Tokens per sec: 172514.04735136282, Loss: 3.474419116973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3197 , TFLOPS: 69.76746983942165, Tokens per sec: 172891.4757833466, Loss: 3.502168655395508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3198 , TFLOPS: 67.00897930062094, Tokens per sec: 166055.63235538022, Loss: 3.493211507797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3199 , TFLOPS: 68.90632129815967, Tokens per sec: 170757.4548347564, Loss: 3.5222713947296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3200 , TFLOPS: 70.59417603402485, Tokens per sec: 174940.14480277864, Loss: 3.5209877490997314 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/optimizer.pt +[rank0]:[2024-08-25 03:16:40,164] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006385560998751316, 'preprocessing_with_comm': 0.001390164004988037, 'state_converting': 1.006747680999979, : 1.0159159190006903}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200, took 4.38s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001200 + eval ppl=22.924354553222656, eval loss=3.132199764251709 +------------------------------------------------------------------ +iteration: 3201 , TFLOPS: 64.86616764834915, Tokens per sec: 160745.50903086073, Loss: 3.518141984939575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3202 , TFLOPS: 65.66397082441195, Tokens per sec: 162722.55318641933, Loss: 3.4863333702087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3203 , TFLOPS: 70.54066114647858, Tokens per sec: 174807.52901628747, Loss: 3.511878252029419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3204 , TFLOPS: 69.6811346007224, Tokens per sec: 172677.52755123822, Loss: 3.485107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3205 , TFLOPS: 67.18877158456384, Tokens per sec: 166501.1774407458, Loss: 3.4719183444976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3206 , TFLOPS: 69.1208855075008, Tokens per sec: 171289.16858170266, Loss: 3.501995086669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3207 , TFLOPS: 70.54057897409587, Tokens per sec: 174807.32538407107, Loss: 3.502103567123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3208 , TFLOPS: 68.06182661219158, Tokens per sec: 168664.7039741581, Loss: 3.4982593059539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3209 , TFLOPS: 67.85501921880805, Tokens per sec: 168152.21247163866, Loss: 3.494503974914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3210 , TFLOPS: 70.33902077373403, Tokens per sec: 174307.84196010572, Loss: 3.4943854808807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3211 , TFLOPS: 68.80546302087122, Tokens per sec: 170507.51691317916, Loss: 3.4652557373046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3212 , TFLOPS: 67.9217590533683, Tokens per sec: 168317.6011336787, Loss: 3.4983150959014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3213 , TFLOPS: 68.94441587390331, Tokens per sec: 170851.85738991312, Loss: 3.508455276489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3214 , TFLOPS: 69.70612087815567, Tokens per sec: 172739.44629918353, Loss: 3.4814460277557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3215 , TFLOPS: 68.78360377800506, Tokens per sec: 170453.34730136246, Loss: 3.4978830814361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3216 , TFLOPS: 69.70228929855163, Tokens per sec: 172729.95122284137, Loss: 3.4871439933776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3217 , TFLOPS: 68.48988844560068, Tokens per sec: 169725.48835224955, Loss: 3.4866647720336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3218 , TFLOPS: 69.79267383568757, Tokens per sec: 172953.93406254298, Loss: 3.4890456199645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3219 , TFLOPS: 68.22232216428742, Tokens per sec: 169062.42963229617, Loss: 3.509840726852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3220 , TFLOPS: 70.59927690807137, Tokens per sec: 174952.7853305736, Loss: 3.479522943496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3221 , TFLOPS: 68.67604716771655, Tokens per sec: 170186.81017272882, Loss: 3.5083632469177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3222 , TFLOPS: 67.10843798538663, Tokens per sec: 166302.10193250817, Loss: 3.4933557510375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3223 , TFLOPS: 70.63987161920237, Tokens per sec: 175053.38349663277, Loss: 3.505676031112671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3224 , TFLOPS: 68.60336287932864, Tokens per sec: 170006.69049926824, Loss: 3.5060083866119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3225 , TFLOPS: 68.90791268674938, Tokens per sec: 170761.39847098768, Loss: 3.4930427074432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3226 , TFLOPS: 68.83038283531373, Tokens per sec: 170569.2709585119, Loss: 3.5041728019714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3227 , TFLOPS: 70.59404859268567, Tokens per sec: 174939.8289891013, Loss: 3.5012826919555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3228 , TFLOPS: 67.95247473899471, Tokens per sec: 168393.71798627297, Loss: 3.487947463989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3229 , TFLOPS: 67.95206138011858, Tokens per sec: 168392.69363744272, Loss: 3.5226569175720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3230 , TFLOPS: 70.61332618162601, Tokens per sec: 174987.6009780972, Loss: 3.5049376487731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3231 , TFLOPS: 68.8086921492196, Tokens per sec: 170515.51904894435, Loss: 3.512575626373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3232 , TFLOPS: 67.92994406303953, Tokens per sec: 168337.8844892972, Loss: 3.502739191055298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3233 , TFLOPS: 69.67473509784067, Tokens per sec: 172661.66887813385, Loss: 3.5019044876098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3234 , TFLOPS: 69.67924174639163, Tokens per sec: 172672.83684969143, Loss: 3.5128793716430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3235 , TFLOPS: 68.7450601163419, Tokens per sec: 170357.8318618263, Loss: 3.463977813720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3236 , TFLOPS: 68.97532673687188, Tokens per sec: 170928.45791346187, Loss: 3.518606185913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3237 , TFLOPS: 70.61701990990643, Tokens per sec: 174996.75444367388, Loss: 3.4894909858703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3238 , TFLOPS: 69.3027905652754, Tokens per sec: 171739.9493533644, Loss: 3.503053665161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3239 , TFLOPS: 67.15078538678203, Tokens per sec: 166407.04345811735, Loss: 3.49969220161438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3240 , TFLOPS: 70.60247435069834, Tokens per sec: 174960.70894563058, Loss: 3.504714012145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3241 , TFLOPS: 69.58200565783636, Tokens per sec: 172431.87511081208, Loss: 3.4926247596740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3242 , TFLOPS: 68.0236372534239, Tokens per sec: 168570.06653622663, Loss: 3.5114798545837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3243 , TFLOPS: 68.86291195024017, Tokens per sec: 170649.8816886775, Loss: 3.517728328704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3244 , TFLOPS: 69.8047357509595, Tokens per sec: 172983.8248173164, Loss: 3.495002508163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3245 , TFLOPS: 68.86588162076521, Tokens per sec: 170657.24085937472, Loss: 3.5020134449005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3246 , TFLOPS: 67.85956253537418, Tokens per sec: 168163.47131057715, Loss: 3.489335298538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3247 , TFLOPS: 70.55960667120495, Tokens per sec: 174854.47811358122, Loss: 3.4826626777648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3248 , TFLOPS: 68.40416172419653, Tokens per sec: 169513.0480930321, Loss: 3.498117208480835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3249 , TFLOPS: 67.43493957881091, Tokens per sec: 167111.20884813322, Loss: 3.4877023696899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3250 , TFLOPS: 69.58132536351783, Tokens per sec: 172430.18926655015, Loss: 3.5021815299987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3251 , TFLOPS: 69.04464456589257, Tokens per sec: 171100.23512976436, Loss: 3.4724655151367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3252 , TFLOPS: 67.91064431795894, Tokens per sec: 168290.05759494507, Loss: 3.5017058849334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3253 , TFLOPS: 69.73344964558841, Tokens per sec: 172807.16999539413, Loss: 3.4958114624023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3254 , TFLOPS: 69.48692930255015, Tokens per sec: 172196.26542888768, Loss: 3.5015714168548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3255 , TFLOPS: 69.04732650918778, Tokens per sec: 171106.88128069037, Loss: 3.5070300102233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3256 , TFLOPS: 68.69869112714119, Tokens per sec: 170242.92439862114, Loss: 3.4800524711608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3257 , TFLOPS: 69.63427140494913, Tokens per sec: 172561.39538970485, Loss: 3.4693095684051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3258 , TFLOPS: 69.04215045627244, Tokens per sec: 171094.05445137684, Loss: 3.5086863040924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3259 , TFLOPS: 67.92334957478617, Tokens per sec: 168321.54262096251, Loss: 3.4809460639953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3260 , TFLOPS: 70.54964089274476, Tokens per sec: 174829.78181673578, Loss: 3.504185438156128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3261 , TFLOPS: 68.81722992205793, Tokens per sec: 170536.67659055107, Loss: 3.5235238075256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3262 , TFLOPS: 68.03260700703326, Tokens per sec: 168592.29457377057, Loss: 3.5171549320220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3263 , TFLOPS: 69.55205997921338, Tokens per sec: 172357.66642039552, Loss: 3.486851930618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3264 , TFLOPS: 69.76261174020671, Tokens per sec: 172879.43687832818, Loss: 3.4824044704437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3265 , TFLOPS: 67.98118915451468, Tokens per sec: 168464.87547108548, Loss: 3.4844963550567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3266 , TFLOPS: 68.8850372877734, Tokens per sec: 170704.71071239194, Loss: 3.4986534118652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3267 , TFLOPS: 70.56005802384061, Tokens per sec: 174855.59661513302, Loss: 3.5111825466156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3268 , TFLOPS: 67.94487598478173, Tokens per sec: 168374.88743626134, Loss: 3.4990646839141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3269 , TFLOPS: 67.04680822703652, Tokens per sec: 166149.37660223828, Loss: 3.5008411407470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3270 , TFLOPS: 70.57507655488158, Tokens per sec: 174892.8141894803, Loss: 3.473701238632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3271 , TFLOPS: 69.82945374288082, Tokens per sec: 173045.07872420907, Loss: 3.4713587760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3272 , TFLOPS: 68.0821109982746, Tokens per sec: 168714.97091738143, Loss: 3.508563995361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3273 , TFLOPS: 69.02855922488347, Tokens per sec: 171060.37388279726, Loss: 3.5000228881835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3274 , TFLOPS: 70.58606897425949, Tokens per sec: 174920.0546156137, Loss: 3.4882264137268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3275 , TFLOPS: 68.83322038503034, Tokens per sec: 170576.3027192916, Loss: 3.47908091545105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3276 , TFLOPS: 68.6283563801729, Tokens per sec: 170068.6271476217, Loss: 3.4992878437042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3277 , TFLOPS: 70.59671343521069, Tokens per sec: 174946.43276243497, Loss: 3.50502610206604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3278 , TFLOPS: 69.01274744806823, Tokens per sec: 171021.19055803807, Loss: 3.520402431488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3279 , TFLOPS: 68.84861427630005, Tokens per sec: 170614.45047763505, Loss: 3.501666784286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3280 , TFLOPS: 68.9807082869678, Tokens per sec: 170941.79398742624, Loss: 3.505807399749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3281 , TFLOPS: 69.62411874945745, Tokens per sec: 172536.23599098914, Loss: 3.5158727169036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3282 , TFLOPS: 67.93899767256721, Tokens per sec: 168360.3203310442, Loss: 3.4865102767944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3283 , TFLOPS: 68.80233314722717, Tokens per sec: 170499.7607415052, Loss: 3.5090396404266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3284 , TFLOPS: 70.60075944405308, Tokens per sec: 174956.45921805108, Loss: 3.4822428226470947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3285 , TFLOPS: 69.30633005700622, Tokens per sec: 171748.7206037806, Loss: 3.498389720916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3286 , TFLOPS: 68.40865736631115, Tokens per sec: 169524.18878942798, Loss: 3.4916176795959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3287 , TFLOPS: 69.55878832485809, Tokens per sec: 172374.34000209154, Loss: 3.506150245666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3288 , TFLOPS: 68.80797887012923, Tokens per sec: 170513.75146478423, Loss: 3.493825674057007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3289 , TFLOPS: 66.9911712914387, Tokens per sec: 166011.50214691248, Loss: 3.49025559425354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3290 , TFLOPS: 70.62322344220652, Tokens per sec: 175012.127480089, Loss: 3.499793767929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3291 , TFLOPS: 68.91563554021081, Tokens per sec: 170780.5365526076, Loss: 3.4952898025512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3292 , TFLOPS: 68.0683917987836, Tokens per sec: 168680.9732297481, Loss: 3.4862608909606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3293 , TFLOPS: 68.88822201783849, Tokens per sec: 170712.60282432023, Loss: 3.508906126022339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3294 , TFLOPS: 70.62012241587881, Tokens per sec: 175004.44279524265, Loss: 3.4981729984283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3295 , TFLOPS: 67.83805651071874, Tokens per sec: 168110.17701239677, Loss: 3.5187814235687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3296 , TFLOPS: 68.454112410693, Tokens per sec: 169636.83139669272, Loss: 3.4823060035705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3297 , TFLOPS: 70.60659753617712, Tokens per sec: 174970.92665345338, Loss: 3.508704900741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3298 , TFLOPS: 68.15087916237319, Tokens per sec: 168885.38600345634, Loss: 3.4817683696746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3299 , TFLOPS: 68.80052610000457, Tokens per sec: 170495.28267942494, Loss: 3.4816761016845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3300 , TFLOPS: 69.77008062215629, Tokens per sec: 172897.9455905645, Loss: 3.4799554347991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3301 , TFLOPS: 69.64794576292967, Tokens per sec: 172595.2819551355, Loss: 3.491692066192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3302 , TFLOPS: 67.9470128601989, Tokens per sec: 168380.18284894136, Loss: 3.526761293411255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3303 , TFLOPS: 69.05843276504177, Tokens per sec: 171134.40380615162, Loss: 3.512979030609131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3304 , TFLOPS: 70.64472221923889, Tokens per sec: 175065.403817861, Loss: 3.4668149948120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3305 , TFLOPS: 68.1260997560498, Tokens per sec: 168823.97990491005, Loss: 3.4603118896484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3306 , TFLOPS: 67.93312253026014, Tokens per sec: 168345.76108120693, Loss: 3.497539520263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3307 , TFLOPS: 70.66954053302366, Tokens per sec: 175126.90633338262, Loss: 3.468644857406616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3308 , TFLOPS: 68.84963181985853, Tokens per sec: 170616.97206266463, Loss: 3.479649066925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3309 , TFLOPS: 66.25403863934014, Tokens per sec: 164184.8062331472, Loss: 3.4822912216186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3310 , TFLOPS: 70.5507063769147, Tokens per sec: 174832.42220388164, Loss: 3.5047507286071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3311 , TFLOPS: 69.68021425617796, Tokens per sec: 172675.24683607277, Loss: 3.485661745071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3312 , TFLOPS: 67.9947111116111, Tokens per sec: 168498.38437033692, Loss: 3.480077028274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3313 , TFLOPS: 68.6020586093069, Tokens per sec: 170003.45837447714, Loss: 3.475770950317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3314 , TFLOPS: 70.5614492440121, Tokens per sec: 174859.0442119735, Loss: 3.4982151985168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3315 , TFLOPS: 68.92336147084812, Tokens per sec: 170799.68225980838, Loss: 3.511183500289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3316 , TFLOPS: 68.75242998423707, Tokens per sec: 170376.0952063285, Loss: 3.5277152061462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3317 , TFLOPS: 69.68263195632764, Tokens per sec: 172681.2381633753, Loss: 3.484156608581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3318 , TFLOPS: 68.90378709689672, Tokens per sec: 170751.1748048615, Loss: 3.5170490741729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3319 , TFLOPS: 68.82284016601706, Tokens per sec: 170550.5793931012, Loss: 3.5112454891204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3320 , TFLOPS: 68.67167625137202, Tokens per sec: 170175.9785605309, Loss: 3.49304461479187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3321 , TFLOPS: 70.63880080124507, Tokens per sec: 175050.72989177427, Loss: 3.4911136627197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3322 , TFLOPS: 69.10242140322175, Tokens per sec: 171243.41249731014, Loss: 3.4965994358062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3323 , TFLOPS: 68.36043486648809, Tokens per sec: 169404.68812271956, Loss: 3.4790618419647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3324 , TFLOPS: 68.94217754192003, Tokens per sec: 170846.31055668593, Loss: 3.4959921836853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3325 , TFLOPS: 68.91345172171606, Tokens per sec: 170775.12480981014, Loss: 3.523549795150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3326 , TFLOPS: 67.87499908162472, Tokens per sec: 168201.72477266195, Loss: 3.4998791217803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3327 , TFLOPS: 70.61899171354905, Tokens per sec: 175001.64079031235, Loss: 3.4945690631866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3328 , TFLOPS: 67.91774716177436, Tokens per sec: 168307.65922436252, Loss: 3.4833807945251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3329 , TFLOPS: 67.93908292756839, Tokens per sec: 168360.53160232876, Loss: 3.515691041946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3330 , TFLOPS: 70.01153176590378, Tokens per sec: 173496.28812281, Loss: 3.489205837249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3331 , TFLOPS: 68.90034026636228, Tokens per sec: 170742.63317909744, Loss: 3.474632978439331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3332 , TFLOPS: 68.68620971534816, Tokens per sec: 170211.99408525697, Loss: 3.480283260345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3333 , TFLOPS: 68.32491784851956, Tokens per sec: 169316.67303967106, Loss: 3.504045009613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3334 , TFLOPS: 70.58308842024381, Tokens per sec: 174912.66847442774, Loss: 3.5000813007354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3335 , TFLOPS: 69.48923679336446, Tokens per sec: 172201.98364531665, Loss: 3.488802909851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3336 , TFLOPS: 68.22066511286421, Tokens per sec: 169058.3232763302, Loss: 3.496387004852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3337 , TFLOPS: 70.55008658057132, Tokens per sec: 174830.8862802665, Loss: 3.531069040298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3338 , TFLOPS: 69.71948300749843, Tokens per sec: 172772.55912765494, Loss: 3.52492618560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3339 , TFLOPS: 68.0000160024485, Tokens per sec: 168511.53047421397, Loss: 3.501016855239868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3340 , TFLOPS: 68.58453625296177, Tokens per sec: 169960.03604520703, Loss: 3.501312255859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3341 , TFLOPS: 70.58658718157724, Tokens per sec: 174921.33879043235, Loss: 3.492079973220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3342 , TFLOPS: 68.01760057201132, Tokens per sec: 168555.1069746909, Loss: 3.4696457386016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3343 , TFLOPS: 67.43374282999649, Tokens per sec: 167108.24317273835, Loss: 3.5113017559051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3344 , TFLOPS: 70.58065838936234, Tokens per sec: 174906.64659021347, Loss: 3.501852035522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3345 , TFLOPS: 68.85851433232322, Tokens per sec: 170638.98390704166, Loss: 3.4901437759399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3346 , TFLOPS: 68.66953431936838, Tokens per sec: 170170.67061707255, Loss: 3.4973177909851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3347 , TFLOPS: 69.9636982554513, Tokens per sec: 173377.75141461258, Loss: 3.524054527282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3348 , TFLOPS: 68.8175762748229, Tokens per sec: 170537.53489085654, Loss: 3.4973034858703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3349 , TFLOPS: 68.11450612906111, Tokens per sec: 168795.24962008864, Loss: 3.4967846870422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3350 , TFLOPS: 68.61903888022456, Tokens per sec: 170045.5373563424, Loss: 3.4859018325805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3351 , TFLOPS: 69.81046999384503, Tokens per sec: 172998.03490286582, Loss: 3.4677486419677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3352 , TFLOPS: 69.69985324940922, Tokens per sec: 172723.9144247135, Loss: 3.513148307800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3353 , TFLOPS: 67.25334672758247, Tokens per sec: 166661.2017586852, Loss: 3.492563247680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3354 , TFLOPS: 69.51784763976593, Tokens per sec: 172272.8844744444, Loss: 3.4993367195129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3355 , TFLOPS: 69.7017427457339, Tokens per sec: 172728.5968047513, Loss: 3.485858678817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3356 , TFLOPS: 67.97285822677381, Tokens per sec: 168444.23051441612, Loss: 3.4893064498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3357 , TFLOPS: 69.6383907096323, Tokens per sec: 172571.60348048861, Loss: 3.50547456741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3358 , TFLOPS: 69.75843920580054, Tokens per sec: 172869.09687842708, Loss: 3.503901243209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3359 , TFLOPS: 68.75939608001046, Tokens per sec: 170393.3579590338, Loss: 3.487144947052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3360 , TFLOPS: 68.76137095899044, Tokens per sec: 170398.25192669706, Loss: 3.496323823928833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3361 , TFLOPS: 69.5667648644878, Tokens per sec: 172394.10674598234, Loss: 3.490516185760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3362 , TFLOPS: 68.92591260912073, Tokens per sec: 170806.00426147899, Loss: 3.502774477005005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3363 , TFLOPS: 68.10248439292073, Tokens per sec: 168765.4584336292, Loss: 3.5015227794647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3364 , TFLOPS: 70.57121743858328, Tokens per sec: 174883.25087417633, Loss: 3.5120255947113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3365 , TFLOPS: 67.92979135319419, Tokens per sec: 168337.50605747808, Loss: 3.510190725326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3366 , TFLOPS: 68.73331090247224, Tokens per sec: 170328.71601557452, Loss: 3.489483594894409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3367 , TFLOPS: 69.61519191589747, Tokens per sec: 172514.11431405562, Loss: 3.5207788944244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3368 , TFLOPS: 69.77117483275158, Tokens per sec: 172900.65716495688, Loss: 3.4652271270751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3369 , TFLOPS: 67.84682357913465, Tokens per sec: 168131.90277370374, Loss: 3.5041165351867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3370 , TFLOPS: 69.21071961715523, Tokens per sec: 171511.7874014704, Loss: 3.4934580326080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3371 , TFLOPS: 70.56952575048744, Tokens per sec: 174879.05868472002, Loss: 3.514564275741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3372 , TFLOPS: 68.09396004216688, Tokens per sec: 168744.3341534264, Loss: 3.485759973526001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3373 , TFLOPS: 67.0074434828102, Tokens per sec: 166051.82642966093, Loss: 3.513422727584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3374 , TFLOPS: 70.59712454827644, Tokens per sec: 174947.45154589976, Loss: 3.5064141750335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3375 , TFLOPS: 69.7539639663698, Tokens per sec: 172858.00674212936, Loss: 3.514425039291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3376 , TFLOPS: 67.08457757990287, Tokens per sec: 166242.9732192788, Loss: 3.490323543548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3377 , TFLOPS: 70.59133155381944, Tokens per sec: 174933.0958674843, Loss: 3.509195327758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3378 , TFLOPS: 69.73784023160398, Tokens per sec: 172818.05035120368, Loss: 3.5006487369537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3379 , TFLOPS: 68.63197695178269, Tokens per sec: 170077.59932290966, Loss: 3.491438627243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3380 , TFLOPS: 68.83222452635742, Tokens per sec: 170573.8348717972, Loss: 3.5132803916931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3381 , TFLOPS: 70.56665905582992, Tokens per sec: 174871.9546995662, Loss: 3.488279342651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3382 , TFLOPS: 68.9436688762513, Tokens per sec: 170850.00624744283, Loss: 3.5308592319488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3383 , TFLOPS: 67.08785298308568, Tokens per sec: 166251.09002918124, Loss: 3.4834203720092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3384 , TFLOPS: 70.53331534746685, Tokens per sec: 174789.32531712984, Loss: 3.4810233116149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3385 , TFLOPS: 68.76107444978098, Tokens per sec: 170397.51714421282, Loss: 3.493243455886841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3386 , TFLOPS: 68.85647438888488, Tokens per sec: 170633.92870248313, Loss: 3.5097708702087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3387 , TFLOPS: 69.3340488382621, Tokens per sec: 171817.4107972085, Loss: 3.4728057384490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3388 , TFLOPS: 70.5718261955745, Tokens per sec: 174884.7594410607, Loss: 3.4835991859436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3389 , TFLOPS: 68.4142565480677, Tokens per sec: 169538.0641786215, Loss: 3.5157268047332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3390 , TFLOPS: 68.26411493702703, Tokens per sec: 169165.9967269989, Loss: 3.4829604625701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3391 , TFLOPS: 69.65997700290852, Tokens per sec: 172625.09669315358, Loss: 3.4771344661712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3392 , TFLOPS: 68.73969919661191, Tokens per sec: 170344.54691218134, Loss: 3.487302780151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3393 , TFLOPS: 67.0594135400691, Tokens per sec: 166180.6139565229, Loss: 3.4912569522857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3394 , TFLOPS: 69.49885546193094, Tokens per sec: 172225.8197656082, Loss: 3.5042009353637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3395 , TFLOPS: 69.84852780645785, Tokens per sec: 173092.34635493453, Loss: 3.489643096923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3396 , TFLOPS: 68.77345256646674, Tokens per sec: 170428.19148092225, Loss: 3.5000834465026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3397 , TFLOPS: 70.0114433873824, Tokens per sec: 173496.0691110983, Loss: 3.5104427337646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3398 , TFLOPS: 68.81194733828234, Tokens per sec: 170523.58576603048, Loss: 3.5168848037719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3399 , TFLOPS: 68.82139925008518, Tokens per sec: 170547.008644692, Loss: 3.479058265686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3400 , TFLOPS: 68.85472569338272, Tokens per sec: 170629.59524239405, Loss: 3.4865708351135254 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/model.pt +[rank0]:[2024-08-25 03:42:21,058] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006397235003532842, 'preprocessing_with_comm': 0.0012457120028557256, 'state_converting': 0.9836131289994228, : 0.9926937609998276}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400, took 4.33s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001400 + eval ppl=25.628904342651367, eval loss=3.243720769882202 +------------------------------------------------------------------ +iteration: 3401 , TFLOPS: 66.75776070667048, Tokens per sec: 165433.0850055583, Loss: 3.5232574939727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3402 , TFLOPS: 67.41121517443385, Tokens per sec: 167052.41716062799, Loss: 3.494370460510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3403 , TFLOPS: 68.10073218437546, Tokens per sec: 168761.11626783235, Loss: 3.495239019393921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3404 , TFLOPS: 69.69367749997704, Tokens per sec: 172708.61023730563, Loss: 3.5120456218719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3405 , TFLOPS: 68.67155437407585, Tokens per sec: 170175.67653516508, Loss: 3.5149264335632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3406 , TFLOPS: 67.87541061580836, Tokens per sec: 168202.74459970323, Loss: 3.5373196601867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3407 , TFLOPS: 69.06020130868215, Tokens per sec: 171138.78645211365, Loss: 3.4958529472351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3408 , TFLOPS: 70.51812469120637, Tokens per sec: 174751.6811975274, Loss: 3.477672815322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3409 , TFLOPS: 67.85547112450558, Tokens per sec: 168153.33234373864, Loss: 3.4732871055603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3410 , TFLOPS: 68.24096785586299, Tokens per sec: 169108.63571001295, Loss: 3.488196849822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3411 , TFLOPS: 70.49368905419026, Tokens per sec: 174691.12699719387, Loss: 3.457470655441284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3412 , TFLOPS: 68.96092181690412, Tokens per sec: 170892.76093494883, Loss: 3.4921436309814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3413 , TFLOPS: 67.91784907448192, Tokens per sec: 168307.91177527848, Loss: 3.5070459842681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3414 , TFLOPS: 69.56716716823686, Tokens per sec: 172395.10369898993, Loss: 3.484574556350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3415 , TFLOPS: 69.73069000191461, Tokens per sec: 172800.33129437058, Loss: 3.498934745788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3416 , TFLOPS: 67.88550774575943, Tokens per sec: 168227.76640001265, Loss: 3.5130012035369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3417 , TFLOPS: 68.60702892043251, Tokens per sec: 170015.77535297137, Loss: 3.512829065322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3418 , TFLOPS: 70.5351501666272, Tokens per sec: 174793.8722011312, Loss: 3.4819605350494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3419 , TFLOPS: 68.10778381639855, Tokens per sec: 168778.59098878695, Loss: 3.482531785964966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3420 , TFLOPS: 67.06486804193712, Tokens per sec: 166194.1308130148, Loss: 3.4899158477783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3421 , TFLOPS: 70.5919077429844, Tokens per sec: 174934.52372770754, Loss: 3.530236005783081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3422 , TFLOPS: 68.79069289760591, Tokens per sec: 170470.9149206644, Loss: 3.5074172019958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3423 , TFLOPS: 67.97485531313416, Tokens per sec: 168449.17951441434, Loss: 3.4918103218078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3424 , TFLOPS: 68.39823616721443, Tokens per sec: 169498.36391007647, Loss: 3.483755350112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3425 , TFLOPS: 70.48066234776906, Tokens per sec: 174658.84538367196, Loss: 3.484372138977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3426 , TFLOPS: 66.78107936201202, Tokens per sec: 165490.8712022566, Loss: 3.4952633380889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3427 , TFLOPS: 68.83786881038631, Tokens per sec: 170587.82202939, Loss: 3.4737071990966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3428 , TFLOPS: 70.54981918413597, Tokens per sec: 174830.22364244363, Loss: 3.5135116577148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3429 , TFLOPS: 67.8518434644361, Tokens per sec: 168144.34260246676, Loss: 3.4846079349517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3430 , TFLOPS: 68.86265431498119, Tokens per sec: 170649.2432401248, Loss: 3.5062968730926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3431 , TFLOPS: 68.9268648590804, Tokens per sec: 170808.36404177806, Loss: 3.4973137378692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3432 , TFLOPS: 69.00389850824585, Tokens per sec: 170999.26191036703, Loss: 3.5134201049804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3433 , TFLOPS: 68.049320044981, Tokens per sec: 168633.7113228982, Loss: 3.4795236587524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3434 , TFLOPS: 69.6299540972735, Tokens per sec: 172550.69662569393, Loss: 3.4938602447509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3435 , TFLOPS: 69.64635820569191, Tokens per sec: 172591.34781341648, Loss: 3.5011258125305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3436 , TFLOPS: 67.43175766726957, Tokens per sec: 167103.3237208173, Loss: 3.4734950065612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3437 , TFLOPS: 68.66623146927981, Tokens per sec: 170162.48579071413, Loss: 3.484473705291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3438 , TFLOPS: 70.58345814918857, Tokens per sec: 174913.58470348187, Loss: 3.4984982013702393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3439 , TFLOPS: 67.16958743701967, Tokens per sec: 166453.63700982303, Loss: 3.515852451324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3440 , TFLOPS: 67.98627812104398, Tokens per sec: 168477.486490741, Loss: 3.5024075508117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3441 , TFLOPS: 70.58168861188761, Tokens per sec: 174909.19959511945, Loss: 3.5057296752929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3442 , TFLOPS: 67.7060944128942, Tokens per sec: 167783.16039716246, Loss: 3.489610433578491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3443 , TFLOPS: 67.95396967553995, Tokens per sec: 168397.42260371294, Loss: 3.477598190307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3444 , TFLOPS: 69.68888963585506, Tokens per sec: 172696.74538258457, Loss: 3.5174965858459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3445 , TFLOPS: 70.54073783954853, Tokens per sec: 174807.7190701632, Loss: 3.519420862197876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3446 , TFLOPS: 67.22545660968943, Tokens per sec: 166592.08697419614, Loss: 3.498995780944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3447 , TFLOPS: 69.17750632589333, Tokens per sec: 171429.4812069774, Loss: 3.498037576675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3448 , TFLOPS: 70.57900234113193, Tokens per sec: 174902.5427202697, Loss: 3.502751350402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3449 , TFLOPS: 68.8127641798606, Tokens per sec: 170525.6099894439, Loss: 3.5032060146331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3450 , TFLOPS: 67.42631421145215, Tokens per sec: 167089.83423765557, Loss: 3.4974687099456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3451 , TFLOPS: 69.58928324542627, Tokens per sec: 172449.90977455233, Loss: 3.4845805168151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3452 , TFLOPS: 69.03004312017187, Tokens per sec: 171064.05113878645, Loss: 3.5040698051452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3453 , TFLOPS: 68.054433038861, Tokens per sec: 168646.3818849756, Loss: 3.493943929672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3454 , TFLOPS: 68.6620638765587, Tokens per sec: 170152.15803685298, Loss: 3.4885847568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3455 , TFLOPS: 70.558737694791, Tokens per sec: 174852.32469430185, Loss: 3.4692471027374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3456 , TFLOPS: 67.17076454196201, Tokens per sec: 166456.55400553896, Loss: 3.469130277633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3457 , TFLOPS: 67.88564085431072, Tokens per sec: 168228.0962576659, Loss: 3.4972617626190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3458 , TFLOPS: 70.53072963015948, Tokens per sec: 174782.9176248019, Loss: 3.5031487941741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3459 , TFLOPS: 69.0314154154639, Tokens per sec: 171067.45183769093, Loss: 3.5186803340911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3460 , TFLOPS: 67.44815686940223, Tokens per sec: 167143.9627502233, Loss: 3.506767511367798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3461 , TFLOPS: 69.53765873331612, Tokens per sec: 172321.97854663388, Loss: 3.5056707859039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3462 , TFLOPS: 69.71423695925401, Tokens per sec: 172759.55884220533, Loss: 3.5049729347229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3463 , TFLOPS: 68.2199756553143, Tokens per sec: 169056.6147245715, Loss: 3.5039024353027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3464 , TFLOPS: 68.14591862881345, Tokens per sec: 168873.09325484742, Loss: 3.496321678161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3465 , TFLOPS: 70.55913734471007, Tokens per sec: 174853.31507082633, Loss: 3.499380588531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3466 , TFLOPS: 68.13181412929306, Tokens per sec: 168838.14075129706, Loss: 3.4990158081054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3467 , TFLOPS: 68.2103121889215, Tokens per sec: 169032.66759033146, Loss: 3.512091636657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3468 , TFLOPS: 69.61835166498717, Tokens per sec: 172521.944520375, Loss: 3.4862334728240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3469 , TFLOPS: 68.77497239077003, Tokens per sec: 170431.95777296828, Loss: 3.500218391418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3470 , TFLOPS: 67.95959526015918, Tokens per sec: 168411.3634221087, Loss: 3.5130014419555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3471 , TFLOPS: 69.68997695296889, Tokens per sec: 172699.4398741705, Loss: 3.4826669692993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3472 , TFLOPS: 68.83019708416427, Tokens per sec: 170568.81064670137, Loss: 3.499041795730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3473 , TFLOPS: 67.53358093242123, Tokens per sec: 167355.65298862176, Loss: 3.492701768875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3474 , TFLOPS: 69.6676534117924, Tokens per sec: 172644.1196799941, Loss: 3.482961654663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3475 , TFLOPS: 70.60545878317544, Tokens per sec: 174968.1046980712, Loss: 3.462435007095337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3476 , TFLOPS: 67.07344287374683, Tokens per sec: 166215.38019083499, Loss: 3.486948251724243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3477 , TFLOPS: 68.06784132246267, Tokens per sec: 168679.6090887849, Loss: 3.4773597717285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3478 , TFLOPS: 70.600216447207, Tokens per sec: 174955.11361205045, Loss: 3.506596803665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3479 , TFLOPS: 68.39703285299373, Tokens per sec: 169495.38196488158, Loss: 3.503181219100952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3480 , TFLOPS: 67.98459367703224, Tokens per sec: 168473.3122529231, Loss: 3.5071613788604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3481 , TFLOPS: 69.9303283764947, Tokens per sec: 173295.05717856248, Loss: 3.51365327835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3482 , TFLOPS: 69.72056124434803, Tokens per sec: 172775.23111734586, Loss: 3.506746768951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3483 , TFLOPS: 68.01688296021904, Tokens per sec: 168553.32865361814, Loss: 3.4954075813293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3484 , TFLOPS: 68.7951163086194, Tokens per sec: 170481.87662043475, Loss: 3.4912073612213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3485 , TFLOPS: 70.57992322814427, Tokens per sec: 174904.82477973038, Loss: 3.4773244857788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3486 , TFLOPS: 68.18880398095928, Tokens per sec: 168979.36788167636, Loss: 3.497169017791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3487 , TFLOPS: 67.82700947822033, Tokens per sec: 168082.80124893464, Loss: 3.5122313499450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3488 , TFLOPS: 70.6067745806788, Tokens per sec: 174971.3653892316, Loss: 3.4816107749938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3489 , TFLOPS: 68.81064288788936, Tokens per sec: 170520.35319425957, Loss: 3.473283290863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3490 , TFLOPS: 67.07380320952463, Tokens per sec: 166216.2731425866, Loss: 3.495948314666748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3491 , TFLOPS: 69.67058534459898, Tokens per sec: 172651.38533246788, Loss: 3.4854159355163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3492 , TFLOPS: 69.73296962438964, Tokens per sec: 172805.98044998487, Loss: 3.500192880630493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3493 , TFLOPS: 68.07485495701877, Tokens per sec: 168696.98964783055, Loss: 3.472933053970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3494 , TFLOPS: 68.01032226250504, Tokens per sec: 168537.0705219624, Loss: 3.5276570320129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3495 , TFLOPS: 70.58042207513341, Tokens per sec: 174906.06097752202, Loss: 3.4826250076293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3496 , TFLOPS: 68.82034693638778, Tokens per sec: 170544.40089542788, Loss: 3.4912779331207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3497 , TFLOPS: 68.23485732006314, Tokens per sec: 169093.4931291698, Loss: 3.50010347366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3498 , TFLOPS: 68.86227982310476, Tokens per sec: 170648.31520800132, Loss: 3.5126562118530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3499 , TFLOPS: 69.03278077130564, Tokens per sec: 171070.83533987313, Loss: 3.5075716972351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3500 , TFLOPS: 67.97165295581776, Tokens per sec: 168441.2437202155, Loss: 3.4900989532470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3501 , TFLOPS: 69.71234786864638, Tokens per sec: 172754.87746757912, Loss: 3.4898602962493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3502 , TFLOPS: 68.79941894038589, Tokens per sec: 170492.53901592613, Loss: 3.4967403411865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3503 , TFLOPS: 67.99274584769995, Tokens per sec: 168493.51422987343, Loss: 3.4998233318328857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3504 , TFLOPS: 68.79366347567493, Tokens per sec: 170478.2763403556, Loss: 3.485755443572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3505 , TFLOPS: 69.60364515161986, Tokens per sec: 172485.50016019458, Loss: 3.503188371658325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3506 , TFLOPS: 68.9860586401762, Tokens per sec: 170955.0527520667, Loss: 3.482452869415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3507 , TFLOPS: 67.43686969658836, Tokens per sec: 167115.99189260576, Loss: 3.486419677734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3508 , TFLOPS: 70.55172008092342, Tokens per sec: 174834.93427408396, Loss: 3.480936288833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3509 , TFLOPS: 67.90836110530235, Tokens per sec: 168284.3995424654, Loss: 3.4969568252563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3510 , TFLOPS: 68.03459914294638, Tokens per sec: 168597.23130601516, Loss: 3.5006661415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3511 , TFLOPS: 69.63510072407743, Tokens per sec: 172563.45053386185, Loss: 3.5263290405273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3512 , TFLOPS: 69.49467654314137, Tokens per sec: 172215.46394450174, Loss: 3.4981818199157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3513 , TFLOPS: 67.5565768892799, Tokens per sec: 167412.6395029313, Loss: 3.5105397701263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3514 , TFLOPS: 68.7583637800492, Tokens per sec: 170390.79980601254, Loss: 3.4954347610473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3515 , TFLOPS: 70.60360258453703, Tokens per sec: 174963.5048333681, Loss: 3.5054690837860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3516 , TFLOPS: 68.0758180530217, Tokens per sec: 168699.37630581958, Loss: 3.5153839588165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3517 , TFLOPS: 68.01552625608946, Tokens per sec: 168549.96659133176, Loss: 3.5036988258361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3518 , TFLOPS: 69.62071657953555, Tokens per sec: 172527.8050391427, Loss: 3.484086513519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3519 , TFLOPS: 68.82520370914472, Tokens per sec: 170556.43651333725, Loss: 3.5240702629089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3520 , TFLOPS: 67.73787572348448, Tokens per sec: 167861.91798580554, Loss: 3.5079381465911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3521 , TFLOPS: 69.62343735682816, Tokens per sec: 172534.5474249922, Loss: 3.4988889694213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3522 , TFLOPS: 70.57139350613868, Tokens per sec: 174883.68718897406, Loss: 3.4843599796295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3523 , TFLOPS: 67.65563974491475, Tokens per sec: 167658.12817186888, Loss: 3.4858906269073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3524 , TFLOPS: 68.73216827711742, Tokens per sec: 170325.88446407474, Loss: 3.521199941635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3525 , TFLOPS: 70.60187101508411, Tokens per sec: 174959.21381351564, Loss: 3.5046873092651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3526 , TFLOPS: 68.69584918415009, Tokens per sec: 170235.88175082332, Loss: 3.4919393062591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3527 , TFLOPS: 66.06549122721961, Tokens per sec: 163717.56497570046, Loss: 3.4899072647094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3528 , TFLOPS: 70.59700346940707, Tokens per sec: 174947.1514991235, Loss: 3.4866886138916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3529 , TFLOPS: 69.83145341281781, Tokens per sec: 173050.03412659475, Loss: 3.487621307373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3530 , TFLOPS: 68.29374303636067, Tokens per sec: 169239.41842095222, Loss: 3.511054277420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3531 , TFLOPS: 68.23048577957981, Tokens per sec: 169082.65997028718, Loss: 3.489455461502075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3532 , TFLOPS: 70.54180327098416, Tokens per sec: 174810.3593266277, Loss: 3.4992151260375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3533 , TFLOPS: 67.58371749811062, Tokens per sec: 167479.8969213987, Loss: 3.4962916374206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3534 , TFLOPS: 68.19998807727211, Tokens per sec: 169007.08330436944, Loss: 3.4829957485198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3535 , TFLOPS: 69.61304190623726, Tokens per sec: 172508.78635327448, Loss: 3.500549793243408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3536 , TFLOPS: 69.02652340409368, Tokens per sec: 171055.328894616, Loss: 3.4883086681365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3537 , TFLOPS: 67.8749003430341, Tokens per sec: 168201.48008755743, Loss: 3.501777410507202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3538 , TFLOPS: 69.64208264695382, Tokens per sec: 172580.75250787183, Loss: 3.4820382595062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3539 , TFLOPS: 68.73053611803452, Tokens per sec: 170321.83979407014, Loss: 3.4809865951538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3540 , TFLOPS: 67.46154558090595, Tokens per sec: 167177.14145221782, Loss: 3.497596263885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3541 , TFLOPS: 69.74991590358744, Tokens per sec: 172847.97519662374, Loss: 3.488612174987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3542 , TFLOPS: 69.64806050230699, Tokens per sec: 172595.5662919521, Loss: 3.501319646835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3543 , TFLOPS: 68.27342355959239, Tokens per sec: 169189.0645191453, Loss: 3.5098607540130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3544 , TFLOPS: 68.10229315017118, Tokens per sec: 168764.9845130287, Loss: 3.4838826656341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3545 , TFLOPS: 70.57726410322311, Tokens per sec: 174898.23517524896, Loss: 3.50581693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3546 , TFLOPS: 67.79989670858394, Tokens per sec: 168015.61281905754, Loss: 3.492610216140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3547 , TFLOPS: 67.97132811165812, Tokens per sec: 168440.4387206034, Loss: 3.50158429145813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3548 , TFLOPS: 69.95648502788849, Tokens per sec: 173359.87624210934, Loss: 3.495161533355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3549 , TFLOPS: 68.87173644764546, Tokens per sec: 170671.7497653464, Loss: 3.481285810470581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3550 , TFLOPS: 68.0993754968345, Tokens per sec: 168757.75424665434, Loss: 3.510847806930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3551 , TFLOPS: 68.7927865458833, Tokens per sec: 170476.10321173014, Loss: 3.501702070236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3552 , TFLOPS: 70.58356698610768, Tokens per sec: 174913.8544133568, Loss: 3.49876070022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3553 , TFLOPS: 67.00079511163142, Tokens per sec: 166035.35103350136, Loss: 3.5072805881500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3554 , TFLOPS: 67.92580521483244, Tokens per sec: 168327.62796750828, Loss: 3.4962754249572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3555 , TFLOPS: 70.58962561192199, Tokens per sec: 174928.86835553736, Loss: 3.5097851753234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3556 , TFLOPS: 68.97868368153381, Tokens per sec: 170936.77679213212, Loss: 3.4980013370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3557 , TFLOPS: 67.92223798998899, Tokens per sec: 168318.78799138393, Loss: 3.499957799911499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3558 , TFLOPS: 69.32417682377704, Tokens per sec: 171792.94685782233, Loss: 3.475618839263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3559 , TFLOPS: 69.64675285493746, Tokens per sec: 172592.32579772116, Loss: 3.523569345474243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3560 , TFLOPS: 67.06743104227827, Tokens per sec: 166200.48221019795, Loss: 3.49218487739563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3561 , TFLOPS: 68.70815011245392, Tokens per sec: 170266.36480621932, Loss: 3.474769115447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3562 , TFLOPS: 70.5401259439323, Tokens per sec: 174806.20272542033, Loss: 3.515367031097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3563 , TFLOPS: 68.29429045969613, Tokens per sec: 169240.7749962812, Loss: 3.5260748863220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3564 , TFLOPS: 68.06281080327136, Tokens per sec: 168667.1429080714, Loss: 3.5088131427764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3565 , TFLOPS: 69.94140582983336, Tokens per sec: 173322.50832821793, Loss: 3.4959897994995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3566 , TFLOPS: 68.80802402145267, Tokens per sec: 170513.86335473752, Loss: 3.4917774200439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3567 , TFLOPS: 67.84244037021512, Tokens per sec: 168121.0406991508, Loss: 3.497920036315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3568 , TFLOPS: 68.66124362768728, Tokens per sec: 170150.12536979176, Loss: 3.4990053176879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3569 , TFLOPS: 69.74218254127135, Tokens per sec: 172828.8110729043, Loss: 3.499613046646118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3570 , TFLOPS: 68.06723679931089, Tokens per sec: 168678.11101382106, Loss: 3.4879837036132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3571 , TFLOPS: 68.86581255723031, Tokens per sec: 170657.06971232727, Loss: 3.493042469024658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3572 , TFLOPS: 69.33371591718709, Tokens per sec: 171816.5857821103, Loss: 3.4802792072296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3573 , TFLOPS: 67.90883190042557, Tokens per sec: 168285.56622464312, Loss: 3.4932985305786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3574 , TFLOPS: 67.94500149602996, Tokens per sec: 168375.1984669607, Loss: 3.5202131271362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3575 , TFLOPS: 70.54961172130193, Tokens per sec: 174829.70952668693, Loss: 3.486191987991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3576 , TFLOPS: 67.62583933648922, Tokens per sec: 167584.2794770044, Loss: 3.51316237449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3577 , TFLOPS: 68.10612244181952, Tokens per sec: 168774.4739195646, Loss: 3.4939491748809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3578 , TFLOPS: 69.64294191608774, Tokens per sec: 172582.881871442, Loss: 3.4989566802978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3579 , TFLOPS: 65.82952456531497, Tokens per sec: 163132.8135936271, Loss: 3.5020055770874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3580 , TFLOPS: 67.44171893658613, Tokens per sec: 167128.0088435081, Loss: 3.4937806129455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3581 , TFLOPS: 69.19521234453441, Tokens per sec: 171473.35867164892, Loss: 3.507673978805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3582 , TFLOPS: 70.53657680158521, Tokens per sec: 174797.40755971393, Loss: 3.5143966674804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3583 , TFLOPS: 68.09510105088404, Tokens per sec: 168747.16169872, Loss: 3.5093994140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3584 , TFLOPS: 68.19387729754048, Tokens per sec: 168991.9401190365, Loss: 3.5145580768585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3585 , TFLOPS: 69.68237104913584, Tokens per sec: 172680.59160661275, Loss: 3.483166456222534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3586 , TFLOPS: 68.59847138683709, Tokens per sec: 169994.5688420318, Loss: 3.5025827884674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3587 , TFLOPS: 67.99629766354381, Tokens per sec: 168502.3160207993, Loss: 3.5042765140533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3588 , TFLOPS: 69.71115145015168, Tokens per sec: 172751.9126107529, Loss: 3.4926862716674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3589 , TFLOPS: 70.57876688468244, Tokens per sec: 174901.9592332502, Loss: 3.52782940864563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3590 , TFLOPS: 67.50906101051187, Tokens per sec: 167294.88992103795, Loss: 3.489168405532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3591 , TFLOPS: 68.84641797078744, Tokens per sec: 170609.0077906318, Loss: 3.472200870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3592 , TFLOPS: 70.54370166798424, Tokens per sec: 174815.06376351957, Loss: 3.504223346710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3593 , TFLOPS: 68.99593341009577, Tokens per sec: 170979.52351972315, Loss: 3.50075101852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3594 , TFLOPS: 67.23265803296657, Tokens per sec: 166609.93289437913, Loss: 3.494971752166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3595 , TFLOPS: 70.00800029758514, Tokens per sec: 173487.53675529303, Loss: 3.5064971446990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3596 , TFLOPS: 69.65545651528528, Tokens per sec: 172613.8944268504, Loss: 3.515336751937866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3597 , TFLOPS: 67.36851985076729, Tokens per sec: 166946.61344530407, Loss: 3.479766368865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3598 , TFLOPS: 68.98349983892773, Tokens per sec: 170948.71176069777, Loss: 3.4607746601104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3599 , TFLOPS: 70.61028625690366, Tokens per sec: 174980.06770975987, Loss: 3.5131123065948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3600 , TFLOPS: 67.48675458273173, Tokens per sec: 167239.6121357429, Loss: 3.5081546306610107 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/optimizer.pt +[rank0]:[2024-08-25 04:08:06,327] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006325962000119034, 'preprocessing_with_comm': 0.001255507995665539, 'state_converting': 0.9894228970006225, : 0.9983790540063637}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600, took 4.39s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001600 + eval ppl=24.751224517822266, eval loss=3.2088749408721924 +------------------------------------------------------------------ +iteration: 3601 , TFLOPS: 61.86316931711325, Tokens per sec: 153303.7483584831, Loss: 3.5151329040527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3602 , TFLOPS: 70.5438320294541, Tokens per sec: 174815.38681360238, Loss: 3.4734818935394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3603 , TFLOPS: 68.79876531496693, Tokens per sec: 170490.91926013515, Loss: 3.4916439056396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3604 , TFLOPS: 67.92874368644983, Tokens per sec: 168334.90982387553, Loss: 3.5243115425109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3605 , TFLOPS: 68.58278049140867, Tokens per sec: 169955.68507466535, Loss: 3.5076987743377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3606 , TFLOPS: 69.74007550970026, Tokens per sec: 172823.58961656244, Loss: 3.506838083267212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3607 , TFLOPS: 67.40365283524018, Tokens per sec: 167033.67685104566, Loss: 3.4878413677215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3608 , TFLOPS: 69.63983591200483, Tokens per sec: 172575.18485116994, Loss: 3.508525848388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3609 , TFLOPS: 69.49903298860175, Tokens per sec: 172226.25969625465, Loss: 3.4840433597564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3610 , TFLOPS: 68.13933730431529, Tokens per sec: 168856.78400774827, Loss: 3.494692087173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3611 , TFLOPS: 67.90428412682338, Tokens per sec: 168274.2963406783, Loss: 3.5109784603118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3612 , TFLOPS: 69.52264250322797, Tokens per sec: 172284.76667430115, Loss: 3.5167236328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3613 , TFLOPS: 68.81602214244406, Tokens per sec: 170533.68357961992, Loss: 3.4856314659118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3614 , TFLOPS: 67.96475122850322, Tokens per sec: 168424.14047964202, Loss: 3.5046520233154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3615 , TFLOPS: 70.4938499894446, Tokens per sec: 174691.5258124826, Loss: 3.4973833560943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3616 , TFLOPS: 68.61974390734626, Tokens per sec: 170047.28449121508, Loss: 3.4895172119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3617 , TFLOPS: 67.9748004704137, Tokens per sec: 168449.04360811156, Loss: 3.499061107635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3618 , TFLOPS: 69.02687076887437, Tokens per sec: 171056.18970280793, Loss: 3.4991486072540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3619 , TFLOPS: 70.53067946486209, Tokens per sec: 174782.7933096688, Loss: 3.468411684036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3620 , TFLOPS: 67.90410809577845, Tokens per sec: 168273.8601163575, Loss: 3.4943063259124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3621 , TFLOPS: 68.36029044498949, Tokens per sec: 169404.3302303367, Loss: 3.5049843788146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3622 , TFLOPS: 70.59673374974739, Tokens per sec: 174946.48310409475, Loss: 3.4955246448516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3623 , TFLOPS: 68.03107668342693, Tokens per sec: 168588.502263295, Loss: 3.466796398162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3624 , TFLOPS: 68.0537055696062, Tokens per sec: 168644.5791360248, Loss: 3.5033113956451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3625 , TFLOPS: 69.28854717464027, Tokens per sec: 171704.65266233156, Loss: 3.518655776977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3626 , TFLOPS: 70.53147881027371, Tokens per sec: 174784.77417565396, Loss: 3.4978771209716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3627 , TFLOPS: 67.26550685919969, Tokens per sec: 166691.33590438176, Loss: 3.4696576595306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3628 , TFLOPS: 69.66232728494789, Tokens per sec: 172630.9209509521, Loss: 3.502150297164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3629 , TFLOPS: 70.60134168090492, Tokens per sec: 174957.90206510862, Loss: 3.4972622394561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3630 , TFLOPS: 67.85382653246269, Tokens per sec: 168149.25686348963, Loss: 3.495065212249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3631 , TFLOPS: 67.6424878767762, Tokens per sec: 167625.53639382368, Loss: 3.4740447998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3632 , TFLOPS: 70.54217142088626, Tokens per sec: 174811.27164264017, Loss: 3.505642890930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3633 , TFLOPS: 68.86599636869296, Tokens per sec: 170657.52521738023, Loss: 3.4965763092041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3634 , TFLOPS: 68.8247775886864, Tokens per sec: 170555.3805399005, Loss: 3.5076191425323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3635 , TFLOPS: 68.94761758996385, Tokens per sec: 170859.7915950093, Loss: 3.4986836910247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3636 , TFLOPS: 69.80477171858662, Tokens per sec: 172983.91394905862, Loss: 3.493561029434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3637 , TFLOPS: 68.09716441378717, Tokens per sec: 168752.27493929217, Loss: 3.487244129180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3638 , TFLOPS: 68.55910375915596, Tokens per sec: 169897.01152393574, Loss: 3.496695041656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3639 , TFLOPS: 70.61823266674816, Tokens per sec: 174999.75978872384, Loss: 3.508535385131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3640 , TFLOPS: 69.66938193384722, Tokens per sec: 172648.40314805837, Loss: 3.5266966819763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3641 , TFLOPS: 67.46447471174942, Tokens per sec: 167184.40016111833, Loss: 3.4919347763061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3642 , TFLOPS: 69.65167289246526, Tokens per sec: 172604.51819270191, Loss: 3.49503231048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3643 , TFLOPS: 69.66644513365824, Tokens per sec: 172641.12543367484, Loss: 3.515523910522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3644 , TFLOPS: 67.48015861933116, Tokens per sec: 167223.26661183097, Loss: 3.4866158962249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3645 , TFLOPS: 69.67650469438782, Tokens per sec: 172666.0541333148, Loss: 3.490044355392456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3646 , TFLOPS: 69.57020590929488, Tokens per sec: 172402.63403407668, Loss: 3.5114357471466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3647 , TFLOPS: 67.9528916263939, Tokens per sec: 168394.75107917213, Loss: 3.5224573612213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3648 , TFLOPS: 69.1010596495934, Tokens per sec: 171240.03792180953, Loss: 3.5253050327301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3649 , TFLOPS: 70.55358381026883, Tokens per sec: 174839.55280070865, Loss: 3.4875388145446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3650 , TFLOPS: 67.96285293618675, Tokens per sec: 168419.43630216765, Loss: 3.504239082336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3651 , TFLOPS: 68.26217942239575, Tokens per sec: 169161.20030852806, Loss: 3.4895036220550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3652 , TFLOPS: 69.88335199096163, Tokens per sec: 173178.64452033548, Loss: 3.4753007888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3653 , TFLOPS: 67.85584931311482, Tokens per sec: 168154.26953677327, Loss: 3.4825408458709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3654 , TFLOPS: 68.75974536048814, Tokens per sec: 170394.2235145339, Loss: 3.5052032470703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3655 , TFLOPS: 69.7984508827104, Tokens per sec: 172968.25022145978, Loss: 3.496999740600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3656 , TFLOPS: 69.8009955173605, Tokens per sec: 172974.5561064374, Loss: 3.4910995960235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3657 , TFLOPS: 67.9382351032584, Tokens per sec: 168358.43060029307, Loss: 3.521801710128784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3658 , TFLOPS: 68.11635364217466, Tokens per sec: 168799.8279611044, Loss: 3.4850242137908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3659 , TFLOPS: 70.5871807132607, Tokens per sec: 174922.80962732687, Loss: 3.4760193824768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3660 , TFLOPS: 68.76051359697226, Tokens per sec: 170396.12728916976, Loss: 3.4945762157440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3661 , TFLOPS: 67.23437311267055, Tokens per sec: 166614.18305081726, Loss: 3.4903767108917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3662 , TFLOPS: 70.59157420962819, Tokens per sec: 174933.6971953091, Loss: 3.527442216873169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3663 , TFLOPS: 69.79467936764448, Tokens per sec: 172958.90399165975, Loss: 3.491151809692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3664 , TFLOPS: 66.28358142602868, Tokens per sec: 164258.016512968, Loss: 3.501654863357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3665 , TFLOPS: 69.65436696679153, Tokens per sec: 172611.19440565922, Loss: 3.491004228591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3666 , TFLOPS: 70.56723415567784, Tokens per sec: 174873.37986034184, Loss: 3.4840891361236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3667 , TFLOPS: 68.03322562857369, Tokens per sec: 168593.82758609453, Loss: 3.4923460483551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3668 , TFLOPS: 67.90414791426318, Tokens per sec: 168273.95879094905, Loss: 3.4915478229522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3669 , TFLOPS: 70.58585100287833, Tokens per sec: 174919.51445851967, Loss: 3.498101234436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3670 , TFLOPS: 68.95589154901104, Tokens per sec: 170880.29537698, Loss: 3.511369228363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3671 , TFLOPS: 67.94903492168336, Tokens per sec: 168385.19374004833, Loss: 3.5117363929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3672 , TFLOPS: 69.59423004828989, Tokens per sec: 172462.16849698368, Loss: 3.5126328468322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3673 , TFLOPS: 69.73965374020214, Tokens per sec: 172822.54442528437, Loss: 3.492046356201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3674 , TFLOPS: 68.26063488109916, Tokens per sec: 169157.37276504986, Loss: 3.496655225753784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3675 , TFLOPS: 68.73077395685867, Tokens per sec: 170322.4291848765, Loss: 3.500196695327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3676 , TFLOPS: 70.53825862128926, Tokens per sec: 174801.5752942083, Loss: 3.5160040855407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3677 , TFLOPS: 68.44535617230137, Tokens per sec: 169615.132473379, Loss: 3.4995739459991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3678 , TFLOPS: 67.74435974097436, Tokens per sec: 167877.98609541898, Loss: 3.49611496925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3679 , TFLOPS: 69.60077164470296, Tokens per sec: 172478.37929351162, Loss: 3.503059148788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3680 , TFLOPS: 69.77510002512115, Tokens per sec: 172910.3842240443, Loss: 3.479921340942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3681 , TFLOPS: 67.3438957619075, Tokens per sec: 166885.5922405422, Loss: 3.473097085952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3682 , TFLOPS: 70.5344346155671, Tokens per sec: 174792.09898677978, Loss: 3.514709711074829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3683 , TFLOPS: 68.72807059142444, Tokens per sec: 170315.72994753017, Loss: 3.488194704055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3684 , TFLOPS: 67.93158655939112, Tokens per sec: 168341.95477619237, Loss: 3.4843220710754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3685 , TFLOPS: 69.0387027601693, Tokens per sec: 171085.5106806382, Loss: 3.5057594776153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3686 , TFLOPS: 69.61696835721658, Tokens per sec: 172518.51653133554, Loss: 3.480618476867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3687 , TFLOPS: 68.817695781692, Tokens per sec: 170537.83104204296, Loss: 3.517382860183716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3688 , TFLOPS: 68.34685876153482, Tokens per sec: 169371.04503911393, Loss: 3.47546124458313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3689 , TFLOPS: 70.61882891607398, Tokens per sec: 175001.23736023554, Loss: 3.506683826446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3690 , TFLOPS: 67.86825255693186, Tokens per sec: 168185.006141282, Loss: 3.5122175216674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3691 , TFLOPS: 67.86729314126468, Tokens per sec: 168182.62860357034, Loss: 3.502988815307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3692 , TFLOPS: 69.57076837427641, Tokens per sec: 172404.02788426157, Loss: 3.529818534851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3693 , TFLOPS: 70.56439435729145, Tokens per sec: 174866.34252711074, Loss: 3.5086421966552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3694 , TFLOPS: 66.67552182440903, Tokens per sec: 165229.28799595367, Loss: 3.5226423740386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3695 , TFLOPS: 69.75224138106702, Tokens per sec: 172853.73798599068, Loss: 3.4953787326812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3696 , TFLOPS: 70.60187852821585, Tokens per sec: 174959.2324318837, Loss: 3.483686685562134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3697 , TFLOPS: 67.88555523716694, Tokens per sec: 168227.8840889521, Loss: 3.506175994873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3698 , TFLOPS: 67.8296844386249, Tokens per sec: 168089.43009548838, Loss: 3.487412452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3699 , TFLOPS: 70.60702393416378, Tokens per sec: 174971.98331463596, Loss: 3.4895198345184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3700 , TFLOPS: 69.47266458958681, Tokens per sec: 172160.9158987767, Loss: 3.4913833141326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3701 , TFLOPS: 67.19563931977866, Tokens per sec: 166518.1964451516, Loss: 3.5046515464782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3702 , TFLOPS: 70.00817592663546, Tokens per sec: 173487.97198342692, Loss: 3.495396614074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3703 , TFLOPS: 69.79983503499689, Tokens per sec: 172971.68030330265, Loss: 3.4952845573425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3704 , TFLOPS: 68.01213933276003, Tokens per sec: 168541.57342222103, Loss: 3.4934744834899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3705 , TFLOPS: 67.83923825507192, Tokens per sec: 168113.10550508386, Loss: 3.517256259918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3706 , TFLOPS: 70.58442301396948, Tokens per sec: 174915.97574469761, Loss: 3.495467185974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3707 , TFLOPS: 69.72307479605519, Tokens per sec: 172781.45997536677, Loss: 3.4949147701263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3708 , TFLOPS: 67.40458498452146, Tokens per sec: 167035.98681964938, Loss: 3.4942641258239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3709 , TFLOPS: 69.5821001708369, Tokens per sec: 172432.10932443885, Loss: 3.526318073272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3710 , TFLOPS: 69.79036466826783, Tokens per sec: 172948.21169130105, Loss: 3.4840798377990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3711 , TFLOPS: 67.15840353018181, Tokens per sec: 166425.9220566698, Loss: 3.534554958343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3712 , TFLOPS: 68.6328251325613, Tokens per sec: 170079.70120831387, Loss: 3.5033042430877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3713 , TFLOPS: 70.49735562327987, Tokens per sec: 174700.21315930388, Loss: 3.489417552947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3714 , TFLOPS: 67.26235529840237, Tokens per sec: 166683.52598955293, Loss: 3.4904942512512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3715 , TFLOPS: 69.71101371170771, Tokens per sec: 172751.57127971557, Loss: 3.5126612186431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3716 , TFLOPS: 69.54110716604876, Tokens per sec: 172330.5241428216, Loss: 3.5092620849609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3717 , TFLOPS: 68.04008196790956, Tokens per sec: 168610.81835025692, Loss: 3.51564621925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3718 , TFLOPS: 68.719329455838, Tokens per sec: 170294.06845063137, Loss: 3.5102672576904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3719 , TFLOPS: 69.96832347627104, Tokens per sec: 173389.213221314, Loss: 3.5037288665771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3720 , TFLOPS: 68.7800096165182, Tokens per sec: 170444.44057326845, Loss: 3.489471435546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3721 , TFLOPS: 68.14944644693088, Tokens per sec: 168881.83557676367, Loss: 3.4981555938720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3722 , TFLOPS: 69.76287448043797, Tokens per sec: 172880.08797756446, Loss: 3.503718137741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3723 , TFLOPS: 69.83459207312895, Tokens per sec: 173057.81207259774, Loss: 3.5132975578308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3724 , TFLOPS: 67.78476636521246, Tokens per sec: 167978.11816143806, Loss: 3.495147705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3725 , TFLOPS: 68.14927043205006, Tokens per sec: 168881.39939249938, Loss: 3.5040700435638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3726 , TFLOPS: 70.54906143556485, Tokens per sec: 174828.34585801148, Loss: 3.513427257537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3727 , TFLOPS: 68.80119570043297, Tokens per sec: 170496.94202304905, Loss: 3.5085935592651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3728 , TFLOPS: 67.31498856956289, Tokens per sec: 166813.95703352214, Loss: 3.494673252105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3729 , TFLOPS: 70.58771385366897, Tokens per sec: 174924.13080798884, Loss: 3.5179715156555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3730 , TFLOPS: 69.41334722460041, Tokens per sec: 172013.9209339932, Loss: 3.49458909034729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3731 , TFLOPS: 66.1329105176174, Tokens per sec: 163884.63740415414, Loss: 3.5098297595977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3732 , TFLOPS: 69.61931321123626, Tokens per sec: 172524.3273379033, Loss: 3.4954142570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3733 , TFLOPS: 70.5452900352916, Tokens per sec: 174818.99991268033, Loss: 3.474931478500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3734 , TFLOPS: 67.16329361777993, Tokens per sec: 166438.04023242556, Loss: 3.5161736011505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3735 , TFLOPS: 69.16273835515501, Tokens per sec: 171392.88454866156, Loss: 3.465790271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3736 , TFLOPS: 70.6184470987526, Tokens per sec: 175000.2911748525, Loss: 3.5134105682373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3737 , TFLOPS: 68.88430211610766, Tokens per sec: 170702.88887601838, Loss: 3.4948668479919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3738 , TFLOPS: 67.19984906175016, Tokens per sec: 166528.62864949834, Loss: 3.519984245300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3739 , TFLOPS: 69.61405200164953, Tokens per sec: 172511.28948097728, Loss: 3.503657579421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3740 , TFLOPS: 69.69643088419328, Tokens per sec: 172715.43342670376, Loss: 3.496685266494751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3741 , TFLOPS: 68.15435425631945, Tokens per sec: 168893.99766906843, Loss: 3.483053207397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3742 , TFLOPS: 69.49380296794007, Tokens per sec: 172213.29912892068, Loss: 3.52010178565979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3743 , TFLOPS: 70.6132082209786, Tokens per sec: 174987.30865861854, Loss: 3.4955692291259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3744 , TFLOPS: 68.3029662566562, Tokens per sec: 169262.27457686618, Loss: 3.503598690032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3745 , TFLOPS: 67.89859812854141, Tokens per sec: 168260.20581057112, Loss: 3.477567195892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3746 , TFLOPS: 69.6467889256593, Tokens per sec: 172592.41518494338, Loss: 3.5065412521362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3747 , TFLOPS: 69.67224206554961, Tokens per sec: 172655.4908694864, Loss: 3.501680374145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3748 , TFLOPS: 68.12435166172348, Tokens per sec: 168819.64793460094, Loss: 3.504013776779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3749 , TFLOPS: 68.90689777837594, Tokens per sec: 170758.88341623684, Loss: 3.494973659515381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3750 , TFLOPS: 69.80405586188523, Tokens per sec: 172982.13997729443, Loss: 3.512542247772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3751 , TFLOPS: 67.79789146618086, Tokens per sec: 168010.643607487, Loss: 3.512730121612549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3752 , TFLOPS: 69.0310252336025, Tokens per sec: 171066.48492405633, Loss: 3.4987142086029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3753 , TFLOPS: 69.5488901738845, Tokens per sec: 172349.81129360764, Loss: 3.5146732330322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3754 , TFLOPS: 68.79604629117847, Tokens per sec: 170484.18121966216, Loss: 3.490391969680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3755 , TFLOPS: 68.3169110063797, Tokens per sec: 169296.83120282198, Loss: 3.4855213165283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3756 , TFLOPS: 70.57968540395709, Tokens per sec: 174904.23542519604, Loss: 3.4911327362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3757 , TFLOPS: 67.88701747767428, Tokens per sec: 168231.50768200835, Loss: 3.4789674282073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3758 , TFLOPS: 67.94348739658119, Tokens per sec: 168371.4463617348, Loss: 3.492744207382202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3759 , TFLOPS: 69.6430491269637, Tokens per sec: 172583.1475518029, Loss: 3.5007407665252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3760 , TFLOPS: 69.68206854312625, Tokens per sec: 172679.8419633972, Loss: 3.4982078075408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3761 , TFLOPS: 67.37686767072499, Tokens per sec: 166967.30026274754, Loss: 3.490849018096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3762 , TFLOPS: 69.67715360993468, Tokens per sec: 172667.66221752387, Loss: 3.4882164001464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3763 , TFLOPS: 70.58656335321685, Tokens per sec: 174921.2797411304, Loss: 3.4748005867004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3764 , TFLOPS: 67.15701764884274, Tokens per sec: 166422.48769004413, Loss: 3.495941638946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3765 , TFLOPS: 68.04872646011884, Tokens per sec: 168632.24035422096, Loss: 3.5043840408325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3766 , TFLOPS: 70.60970834565447, Tokens per sec: 174978.6355820222, Loss: 3.4973907470703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3767 , TFLOPS: 69.70694588893538, Tokens per sec: 172741.4907667781, Loss: 3.468245267868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3768 , TFLOPS: 67.98458637885933, Tokens per sec: 168473.29416724667, Loss: 3.48272967338562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3769 , TFLOPS: 69.54934421108354, Tokens per sec: 172350.9364478032, Loss: 3.4734530448913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3770 , TFLOPS: 69.77986283512473, Tokens per sec: 172922.1869919, Loss: 3.5243635177612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3771 , TFLOPS: 66.94078167372354, Tokens per sec: 165886.63112333964, Loss: 3.493492603302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3772 , TFLOPS: 68.80617174995642, Tokens per sec: 170509.2732219253, Loss: 3.491168975830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3773 , TFLOPS: 70.56247508764055, Tokens per sec: 174861.58636549083, Loss: 3.5021233558654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3774 , TFLOPS: 68.82168617067212, Tokens per sec: 170547.71966551334, Loss: 3.493232250213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3775 , TFLOPS: 67.809224864499, Tokens per sec: 168038.72901699622, Loss: 3.515321731567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3776 , TFLOPS: 70.62014440176581, Tokens per sec: 175004.49727869258, Loss: 3.4855880737304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3777 , TFLOPS: 69.79199449436724, Tokens per sec: 172952.25057991553, Loss: 3.5097672939300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3778 , TFLOPS: 67.36983171426259, Tokens per sec: 166949.86438755912, Loss: 3.4925320148468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3779 , TFLOPS: 68.70460361388908, Tokens per sec: 170257.57619209657, Loss: 3.5091519355773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3780 , TFLOPS: 70.58638837694332, Tokens per sec: 174920.84613064936, Loss: 3.5107009410858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3781 , TFLOPS: 66.47588781857195, Tokens per sec: 164734.5729379882, Loss: 3.495718002319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3782 , TFLOPS: 69.61211352013954, Tokens per sec: 172506.48571025417, Loss: 3.5003418922424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3783 , TFLOPS: 69.52680700923229, Tokens per sec: 172295.08677893807, Loss: 3.500958204269409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3784 , TFLOPS: 67.6186006735478, Tokens per sec: 167566.34127283105, Loss: 3.4897873401641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3785 , TFLOPS: 68.89869312594502, Tokens per sec: 170738.55138369597, Loss: 3.5073981285095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3786 , TFLOPS: 68.99628939575699, Tokens per sec: 170980.4056914067, Loss: 3.4985034465789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3787 , TFLOPS: 69.69980183669342, Tokens per sec: 172723.78701834023, Loss: 3.505908966064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3788 , TFLOPS: 67.98907590530007, Tokens per sec: 168484.419708329, Loss: 3.522087335586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3789 , TFLOPS: 69.68066092130289, Tokens per sec: 172676.35372145654, Loss: 3.5082340240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3790 , TFLOPS: 69.54740383187618, Tokens per sec: 172346.12797437693, Loss: 3.5282866954803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3791 , TFLOPS: 68.40824836512842, Tokens per sec: 169523.175239442, Loss: 3.508361339569092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3792 , TFLOPS: 68.17463245023188, Tokens per sec: 168944.24926741087, Loss: 3.5310921669006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3793 , TFLOPS: 70.58478243370209, Tokens per sec: 174916.86642638835, Loss: 3.4846158027648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3794 , TFLOPS: 68.01222063010911, Tokens per sec: 168541.77488600768, Loss: 3.5132415294647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3795 , TFLOPS: 67.82556255982013, Tokens per sec: 168079.21562574772, Loss: 3.4849040508270264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3796 , TFLOPS: 70.6092348681841, Tokens per sec: 174977.46225269279, Loss: 3.4984240531921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3797 , TFLOPS: 69.40486896880901, Tokens per sec: 171992.91088218516, Loss: 3.498731851577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3798 , TFLOPS: 67.25607971076084, Tokens per sec: 166667.97439206418, Loss: 3.492252826690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3799 , TFLOPS: 69.65296190253666, Tokens per sec: 172607.712501655, Loss: 3.503603219985962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3800 , TFLOPS: 70.55264768367852, Tokens per sec: 174837.23297589496, Loss: 3.5058534145355225 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/optimizer.pt +[rank0]:[2024-08-25 04:34:08,263] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006396809003490489, 'preprocessing_with_comm': 0.0012793109999620356, 'state_converting': 1.0042545460019028, : 1.0133179179974832}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800, took 4.39s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0001800 + eval ppl=24.69380760192871, eval loss=3.206552505493164 +------------------------------------------------------------------ +iteration: 3801 , TFLOPS: 63.86220286866449, Tokens per sec: 158257.57371095175, Loss: 3.495128870010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3802 , TFLOPS: 69.52218035186806, Tokens per sec: 172283.62141232114, Loss: 3.5075247287750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3803 , TFLOPS: 70.5392656143572, Tokens per sec: 174804.07073396014, Loss: 3.4876718521118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3804 , TFLOPS: 68.2117870033287, Tokens per sec: 169036.32234289666, Loss: 3.5169119834899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3805 , TFLOPS: 66.82541280260624, Tokens per sec: 165600.7343517807, Loss: 3.4846677780151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3806 , TFLOPS: 70.52043520924738, Tokens per sec: 174757.40691575763, Loss: 3.4994020462036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3807 , TFLOPS: 68.79802837179014, Tokens per sec: 170489.09303376207, Loss: 3.5183699131011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3808 , TFLOPS: 67.95786981406141, Tokens per sec: 168407.08757660486, Loss: 3.4968388080596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3809 , TFLOPS: 69.39114766786399, Tokens per sec: 171958.9079869173, Loss: 3.4769678115844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3810 , TFLOPS: 69.69934194238529, Tokens per sec: 172722.6473495831, Loss: 3.510335922241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3811 , TFLOPS: 67.92808309034997, Tokens per sec: 168333.27279396943, Loss: 3.4881937503814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3812 , TFLOPS: 69.05426267054325, Tokens per sec: 171124.06985261047, Loss: 3.5132369995117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3813 , TFLOPS: 69.48893152014209, Tokens per sec: 172201.22714464297, Loss: 3.511763095855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3814 , TFLOPS: 68.80252483690711, Tokens per sec: 170500.23576964845, Loss: 3.509678363800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3815 , TFLOPS: 68.19229222768385, Tokens per sec: 168988.01214132193, Loss: 3.497206211090088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3816 , TFLOPS: 69.50105243590885, Tokens per sec: 172231.26410914212, Loss: 3.5042672157287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3817 , TFLOPS: 69.02856969189747, Tokens per sec: 171060.39982121097, Loss: 3.4931561946868896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3818 , TFLOPS: 68.06738673239056, Tokens per sec: 168678.48256450897, Loss: 3.5124568939208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3819 , TFLOPS: 69.6796420049066, Tokens per sec: 172673.82873438366, Loss: 3.532850503921509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3820 , TFLOPS: 69.5553693627035, Tokens per sec: 172365.8674372427, Loss: 3.4899370670318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3821 , TFLOPS: 67.37109987213292, Tokens per sec: 166953.00702246092, Loss: 3.4907922744750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3822 , TFLOPS: 69.60162767121764, Tokens per sec: 172480.5006215141, Loss: 3.490147590637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3823 , TFLOPS: 69.36632604447814, Tokens per sec: 171897.3972698396, Loss: 3.491901397705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3824 , TFLOPS: 67.8552176328983, Tokens per sec: 168152.7041636114, Loss: 3.4991180896759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3825 , TFLOPS: 67.92828770152886, Tokens per sec: 168333.77984301047, Loss: 3.5108625888824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3826 , TFLOPS: 70.58496298246799, Tokens per sec: 174917.3138461193, Loss: 3.4654510021209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3827 , TFLOPS: 68.70194605295548, Tokens per sec: 170250.99046334857, Loss: 3.512434244155884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3828 , TFLOPS: 67.92154292710978, Tokens per sec: 168317.06554900177, Loss: 3.498011827468872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3829 , TFLOPS: 69.63367747887713, Tokens per sec: 172559.92357547212, Loss: 3.524467706680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3830 , TFLOPS: 69.77482510047608, Tokens per sec: 172909.70293048836, Loss: 3.4935646057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3831 , TFLOPS: 67.14621494872925, Tokens per sec: 166395.71740914724, Loss: 3.5092382431030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3832 , TFLOPS: 68.80904763619598, Tokens per sec: 170516.39998483195, Loss: 3.474175453186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3833 , TFLOPS: 70.59135585607984, Tokens per sec: 174933.15609116267, Loss: 3.484511137008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3834 , TFLOPS: 68.76127387564144, Tokens per sec: 170398.0113434636, Loss: 3.504239559173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3835 , TFLOPS: 67.5043566283807, Tokens per sec: 167283.23194388647, Loss: 3.494128704071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3836 , TFLOPS: 70.57996557864989, Tokens per sec: 174904.9297289486, Loss: 3.528871536254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3837 , TFLOPS: 69.80106566868726, Tokens per sec: 172974.7299491529, Loss: 3.5022318363189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3838 , TFLOPS: 67.39136520529402, Tokens per sec: 167003.22675044485, Loss: 3.495631694793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3839 , TFLOPS: 69.64608473446188, Tokens per sec: 172590.67012158313, Loss: 3.4905753135681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3840 , TFLOPS: 70.6176332156656, Tokens per sec: 174998.27428290632, Loss: 3.4874160289764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3841 , TFLOPS: 67.24891863567977, Tokens per sec: 166650.2284591557, Loss: 3.5019102096557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3842 , TFLOPS: 68.71789984573184, Tokens per sec: 170290.52571930378, Loss: 3.503171920776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3843 , TFLOPS: 70.60240349242298, Tokens per sec: 174960.53335101856, Loss: 3.4825518131256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3844 , TFLOPS: 68.2706735361471, Tokens per sec: 169182.24965810782, Loss: 3.5205557346343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3845 , TFLOPS: 68.53546019913375, Tokens per sec: 169838.42017764755, Loss: 3.501624584197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3846 , TFLOPS: 69.39462840016157, Tokens per sec: 171967.533624985, Loss: 3.4768319129943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3847 , TFLOPS: 69.66912082336121, Tokens per sec: 172647.75608751035, Loss: 3.4986889362335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3848 , TFLOPS: 68.00797052736297, Tokens per sec: 168531.2426632153, Loss: 3.5085103511810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3849 , TFLOPS: 68.76173507530771, Tokens per sec: 170399.154247042, Loss: 3.4739232063293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3850 , TFLOPS: 70.59137485924178, Tokens per sec: 174933.20318309133, Loss: 3.488051652908325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3851 , TFLOPS: 68.46912533179427, Tokens per sec: 169674.03506899148, Loss: 3.4909896850585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3852 , TFLOPS: 68.15660280006846, Tokens per sec: 168899.569808176, Loss: 3.490891933441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3853 , TFLOPS: 69.59073003262709, Tokens per sec: 172453.4950726127, Loss: 3.4983768463134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3854 , TFLOPS: 68.76837323840776, Tokens per sec: 170415.60434645793, Loss: 3.4860570430755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3855 , TFLOPS: 67.93333578392895, Tokens per sec: 168346.2895472901, Loss: 3.5033273696899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3856 , TFLOPS: 70.57513566433455, Tokens per sec: 174892.9606692162, Loss: 3.5038816928863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3857 , TFLOPS: 68.82447450216787, Tokens per sec: 170554.62945811986, Loss: 3.496208667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3858 , TFLOPS: 67.14396046067115, Tokens per sec: 166390.1305393869, Loss: 3.490581750869751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3859 , TFLOPS: 69.6731517286989, Tokens per sec: 172657.74511497802, Loss: 3.4965949058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3860 , TFLOPS: 69.47454796910962, Tokens per sec: 172165.5831207056, Loss: 3.484623432159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3861 , TFLOPS: 68.01664738883787, Tokens per sec: 168552.7448817853, Loss: 3.4955174922943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3862 , TFLOPS: 69.12244991197616, Tokens per sec: 171293.04534832592, Loss: 3.4823391437530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3863 , TFLOPS: 70.58928897552279, Tokens per sec: 174928.03413345662, Loss: 3.482248544692993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3864 , TFLOPS: 68.06613624280408, Tokens per sec: 168675.38371356088, Loss: 3.491990327835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3865 , TFLOPS: 68.17310731799037, Tokens per sec: 168940.46982170924, Loss: 3.5033762454986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3866 , TFLOPS: 69.6701906452166, Tokens per sec: 172650.40722391862, Loss: 3.5005035400390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3867 , TFLOPS: 69.69538727594541, Tokens per sec: 172712.8472505026, Loss: 3.488872528076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3868 , TFLOPS: 67.16084299411156, Tokens per sec: 166431.96731701685, Loss: 3.502690553665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3869 , TFLOPS: 69.60720681186054, Tokens per sec: 172494.32634662025, Loss: 3.5216498374938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3870 , TFLOPS: 70.59823576949861, Tokens per sec: 174950.2052744994, Loss: 3.5114963054656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3871 , TFLOPS: 68.00413470752962, Tokens per sec: 168521.73707911783, Loss: 3.502012252807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3872 , TFLOPS: 66.91361716015436, Tokens per sec: 165819.31446629704, Loss: 3.510814905166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3873 , TFLOPS: 70.59394248136594, Tokens per sec: 174939.5660335619, Loss: 3.505208730697632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3874 , TFLOPS: 68.9509549738244, Tokens per sec: 170868.06199985888, Loss: 3.4845540523529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3875 , TFLOPS: 67.08988159127229, Tokens per sec: 166256.11714373727, Loss: 3.503012180328369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3876 , TFLOPS: 70.54292137047896, Tokens per sec: 174813.13010034463, Loss: 3.494234800338745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3877 , TFLOPS: 69.75989423011153, Tokens per sec: 172872.70258895386, Loss: 3.5194311141967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3878 , TFLOPS: 67.95670362892128, Tokens per sec: 168404.1976413617, Loss: 3.515101671218872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3879 , TFLOPS: 68.57385359385958, Tokens per sec: 169933.56323915976, Loss: 3.4994969367980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3880 , TFLOPS: 70.55451199789701, Tokens per sec: 174841.85295189955, Loss: 3.457775592803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3881 , TFLOPS: 68.84044367222685, Tokens per sec: 170594.2028206753, Loss: 3.5130064487457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3882 , TFLOPS: 68.2352274812458, Tokens per sec: 169094.41042935706, Loss: 3.508861780166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3883 , TFLOPS: 69.63848811463039, Tokens per sec: 172571.84486080395, Loss: 3.499845504760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3884 , TFLOPS: 68.96076972709173, Tokens per sec: 170892.38403963964, Loss: 3.5189483165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3885 , TFLOPS: 67.9214034387745, Tokens per sec: 168316.71988154107, Loss: 3.509289026260376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3886 , TFLOPS: 68.73542642059812, Tokens per sec: 170333.95850253402, Loss: 3.495509624481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3887 , TFLOPS: 70.52254579894844, Tokens per sec: 174762.6371895375, Loss: 3.5187337398529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3888 , TFLOPS: 67.51628195547826, Tokens per sec: 167312.78421811745, Loss: 3.4893198013305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3889 , TFLOPS: 69.59614283249097, Tokens per sec: 172466.90858694454, Loss: 3.5176355838775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3890 , TFLOPS: 69.61140350411034, Tokens per sec: 172504.7262123191, Loss: 3.511289358139038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3891 , TFLOPS: 67.98616012240616, Tokens per sec: 168477.19407711786, Loss: 3.5170652866363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3892 , TFLOPS: 67.6583109062649, Tokens per sec: 167664.74760394724, Loss: 3.487527847290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3893 , TFLOPS: 70.57448116653244, Tokens per sec: 174891.3387515637, Loss: 3.4860520362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3894 , TFLOPS: 67.89556682653568, Tokens per sec: 168252.69391027544, Loss: 3.499793767929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3895 , TFLOPS: 68.81634529814201, Tokens per sec: 170534.48439503805, Loss: 3.4745469093322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3896 , TFLOPS: 69.89448755286513, Tokens per sec: 173206.23966941575, Loss: 3.4980854988098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3897 , TFLOPS: 68.86949383781909, Tokens per sec: 170666.1923311528, Loss: 3.4953203201293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3898 , TFLOPS: 67.85855385903342, Tokens per sec: 168160.9716994899, Loss: 3.515934467315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3899 , TFLOPS: 68.84770505731589, Tokens per sec: 170612.19733283354, Loss: 3.512495279312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3900 , TFLOPS: 70.57050039359761, Tokens per sec: 174881.47395771285, Loss: 3.472766399383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3901 , TFLOPS: 69.40140490385656, Tokens per sec: 171984.32654763473, Loss: 3.5002241134643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3902 , TFLOPS: 67.52149373174895, Tokens per sec: 167325.69957384074, Loss: 3.505319595336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3903 , TFLOPS: 70.59769511783013, Tokens per sec: 174948.8654801072, Loss: 3.4715936183929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3904 , TFLOPS: 69.81399146701041, Tokens per sec: 173006.7615012919, Loss: 3.480095624923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3905 , TFLOPS: 66.44966073697773, Tokens per sec: 164669.57934064616, Loss: 3.51524019241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3906 , TFLOPS: 69.64957156437092, Tokens per sec: 172599.31087020374, Loss: 3.5070478916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3907 , TFLOPS: 70.6256941750188, Tokens per sec: 175018.25022817147, Loss: 3.50325083732605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3908 , TFLOPS: 67.70070537171765, Tokens per sec: 167769.80575947504, Loss: 3.5023353099823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3909 , TFLOPS: 68.95734338334633, Tokens per sec: 170883.89318239564, Loss: 3.505763292312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3910 , TFLOPS: 70.56853229391844, Tokens per sec: 174876.59678990376, Loss: 3.505610942840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3911 , TFLOPS: 68.91688804945149, Tokens per sec: 170783.6404084813, Loss: 3.506258964538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3912 , TFLOPS: 67.19250410916096, Tokens per sec: 166510.42704786902, Loss: 3.5037448406219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3913 , TFLOPS: 69.76064272268431, Tokens per sec: 172874.55743600248, Loss: 3.4807472229003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3914 , TFLOPS: 68.95375546056968, Tokens per sec: 170875.0019145128, Loss: 3.4961330890655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3915 , TFLOPS: 68.84176489077782, Tokens per sec: 170597.47694578886, Loss: 3.4911837577819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3916 , TFLOPS: 68.71807464493568, Tokens per sec: 170290.95889098666, Loss: 3.5035548210144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3917 , TFLOPS: 69.79866837990602, Tokens per sec: 172968.78920346982, Loss: 3.4692087173461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3918 , TFLOPS: 68.74197994319015, Tokens per sec: 170350.1988534472, Loss: 3.504103899002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3919 , TFLOPS: 68.16066290631325, Tokens per sec: 168909.6311987082, Loss: 3.488335132598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3920 , TFLOPS: 69.47684991486467, Tokens per sec: 172171.28759586686, Loss: 3.487936019897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3921 , TFLOPS: 69.75598712616181, Tokens per sec: 172863.02035496343, Loss: 3.5270769596099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3922 , TFLOPS: 67.25786399763972, Tokens per sec: 166672.39605150544, Loss: 3.5209548473358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3923 , TFLOPS: 69.61807751651973, Tokens per sec: 172521.26515027272, Loss: 3.512441396713257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3924 , TFLOPS: 69.81363982664794, Tokens per sec: 173005.89009773734, Loss: 3.5009617805480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3925 , TFLOPS: 67.07855808373044, Tokens per sec: 166228.05624466127, Loss: 3.4955286979675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3926 , TFLOPS: 69.68106584571196, Tokens per sec: 172677.35716874833, Loss: 3.473937511444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3927 , TFLOPS: 69.6197322263678, Tokens per sec: 172525.3657035576, Loss: 3.499997615814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3928 , TFLOPS: 68.07783527027595, Tokens per sec: 168704.37519239043, Loss: 3.4942052364349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3929 , TFLOPS: 68.708158841904, Tokens per sec: 170266.38643875826, Loss: 3.485642433166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3930 , TFLOPS: 70.58124407729483, Tokens per sec: 174908.09798942902, Loss: 3.506057024002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3931 , TFLOPS: 67.61610893125075, Tokens per sec: 167560.16646093084, Loss: 3.4828310012817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3932 , TFLOPS: 67.87273913917198, Tokens per sec: 168196.12438631084, Loss: 3.488384246826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3933 , TFLOPS: 69.62909647643507, Tokens per sec: 172548.57134678154, Loss: 3.5019478797912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3934 , TFLOPS: 69.72157892463275, Tokens per sec: 172777.75304119813, Loss: 3.5042004585266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3935 , TFLOPS: 67.90108790144042, Tokens per sec: 168266.375742086, Loss: 3.495126247406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3936 , TFLOPS: 69.6526203403577, Tokens per sec: 172606.8660729494, Loss: 3.4851529598236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3937 , TFLOPS: 70.52600200579003, Tokens per sec: 174771.20205082357, Loss: 3.5142898559570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3938 , TFLOPS: 66.85872504065645, Tokens per sec: 165683.2857473151, Loss: 3.4969167709350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3939 , TFLOPS: 67.85369368778267, Tokens per sec: 168148.92765973846, Loss: 3.512714147567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3940 , TFLOPS: 70.54943624190848, Tokens per sec: 174829.27466941933, Loss: 3.5024545192718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3941 , TFLOPS: 69.62814041978015, Tokens per sec: 172546.20213307234, Loss: 3.48913836479187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3942 , TFLOPS: 67.08490109286902, Tokens per sec: 166243.7749200468, Loss: 3.501509666442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3943 , TFLOPS: 70.59288271262017, Tokens per sec: 174936.93980986672, Loss: 3.5023348331451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3944 , TFLOPS: 69.81465194969618, Tokens per sec: 173008.39825014543, Loss: 3.4742116928100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3945 , TFLOPS: 67.74659109979294, Tokens per sec: 167883.51564837588, Loss: 3.491554021835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3946 , TFLOPS: 69.14737699436705, Tokens per sec: 171354.81740443537, Loss: 3.490877151489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3947 , TFLOPS: 70.58906638197676, Tokens per sec: 174927.48252212885, Loss: 3.4921438694000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3948 , TFLOPS: 68.86914720415498, Tokens per sec: 170665.3333347483, Loss: 3.5083985328674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3949 , TFLOPS: 67.34720597785422, Tokens per sec: 166893.79532030856, Loss: 3.496398448944092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3950 , TFLOPS: 70.56366634392168, Tokens per sec: 174864.53842978316, Loss: 3.513622760772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3951 , TFLOPS: 68.92833306924489, Tokens per sec: 170812.00242830237, Loss: 3.516666889190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3952 , TFLOPS: 67.93068847371593, Tokens per sec: 168339.72922095694, Loss: 3.5103752613067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3953 , TFLOPS: 69.391303098927, Tokens per sec: 171959.29316221358, Loss: 3.507624864578247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3954 , TFLOPS: 70.58361689726898, Tokens per sec: 174913.97809871266, Loss: 3.4986929893493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3955 , TFLOPS: 67.43261571772857, Tokens per sec: 167105.45006437675, Loss: 3.4858529567718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3956 , TFLOPS: 69.7299259516431, Tokens per sec: 172798.4378936307, Loss: 3.4907732009887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3957 , TFLOPS: 69.46886409133204, Tokens per sec: 172151.4978454419, Loss: 3.49943208694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3958 , TFLOPS: 68.11328194911559, Tokens per sec: 168792.21596732907, Loss: 3.499337911605835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3959 , TFLOPS: 67.59771176351639, Tokens per sec: 167514.5762526698, Loss: 3.5112357139587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3960 , TFLOPS: 69.585011215892, Tokens per sec: 172439.32321476348, Loss: 3.4883713722229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3961 , TFLOPS: 69.65261192825704, Tokens per sec: 172606.8452268373, Loss: 3.479470729827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3962 , TFLOPS: 67.99614466386221, Tokens per sec: 168501.936870734, Loss: 3.50476336479187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3963 , TFLOPS: 69.93615434276887, Tokens per sec: 173309.49456477305, Loss: 3.482927083969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3964 , TFLOPS: 68.7268439030958, Tokens per sec: 170312.69007872095, Loss: 3.497404098510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3965 , TFLOPS: 67.88451080589574, Tokens per sec: 168225.29587320847, Loss: 3.5066397190093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3966 , TFLOPS: 68.87222893792799, Tokens per sec: 170672.97021051878, Loss: 3.5050227642059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3967 , TFLOPS: 70.54536622144171, Tokens per sec: 174819.1887103529, Loss: 3.4882891178131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3968 , TFLOPS: 68.76651155218964, Tokens per sec: 170410.99088292776, Loss: 3.48948073387146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3969 , TFLOPS: 67.48157441845012, Tokens per sec: 167226.77511801088, Loss: 3.513763189315796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3970 , TFLOPS: 70.58511192221478, Tokens per sec: 174917.6829352189, Loss: 3.486644744873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3971 , TFLOPS: 68.78581081435459, Tokens per sec: 170458.81658056172, Loss: 3.5021588802337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3972 , TFLOPS: 67.46150779263274, Tokens per sec: 167177.04780871412, Loss: 3.523125648498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3973 , TFLOPS: 69.58484452631735, Tokens per sec: 172438.91013963515, Loss: 3.4870166778564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3974 , TFLOPS: 70.56952675703761, Tokens per sec: 174879.06117906224, Loss: 3.484621524810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3975 , TFLOPS: 66.2937076696033, Tokens per sec: 164283.11046004374, Loss: 3.484060525894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3976 , TFLOPS: 69.66792794164371, Tokens per sec: 172644.7999952076, Loss: 3.504615545272827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3977 , TFLOPS: 70.58382305453712, Tokens per sec: 174914.4889791332, Loss: 3.5202856063842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3978 , TFLOPS: 67.9635715197736, Tokens per sec: 168421.21703145438, Loss: 3.485740900039673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3979 , TFLOPS: 67.83107601801252, Tokens per sec: 168092.87858250603, Loss: 3.493593215942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3980 , TFLOPS: 69.89489950618275, Tokens per sec: 173207.26053511724, Loss: 3.488919258117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3981 , TFLOPS: 69.74264288171534, Tokens per sec: 172829.95184723518, Loss: 3.5141358375549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3982 , TFLOPS: 67.00390294268874, Tokens per sec: 166043.05258121723, Loss: 3.5127487182617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3983 , TFLOPS: 69.6449645499147, Tokens per sec: 172587.89418087673, Loss: 3.508697032928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3984 , TFLOPS: 69.75398072007863, Tokens per sec: 172858.04825966543, Loss: 3.478985071182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3985 , TFLOPS: 68.95147731560279, Tokens per sec: 170869.35642032642, Loss: 3.4732449054718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3986 , TFLOPS: 67.9827153052731, Tokens per sec: 168468.6574407842, Loss: 3.49347186088562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3987 , TFLOPS: 70.56337712902024, Tokens per sec: 174863.82172339776, Loss: 3.4934334754943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3988 , TFLOPS: 69.73757861694716, Tokens per sec: 172817.40204126487, Loss: 3.498542547225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3989 , TFLOPS: 67.14263764966023, Tokens per sec: 166386.85246798221, Loss: 3.509911060333252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3990 , TFLOPS: 69.54822705412937, Tokens per sec: 172348.16800980575, Loss: 3.4957685470581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3991 , TFLOPS: 69.70768348031835, Tokens per sec: 172743.31859947747, Loss: 3.4843645095825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3992 , TFLOPS: 67.09671346945258, Tokens per sec: 166273.04729046157, Loss: 3.5033767223358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3993 , TFLOPS: 69.6040420530657, Tokens per sec: 172486.48372569968, Loss: 3.5160818099975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3994 , TFLOPS: 69.66913326415668, Tokens per sec: 172647.78691717202, Loss: 3.5004701614379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3995 , TFLOPS: 68.01915328110371, Tokens per sec: 168558.95475886686, Loss: 3.509683847427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3996 , TFLOPS: 69.09719366567447, Tokens per sec: 171230.45758778515, Loss: 3.495612382888794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3997 , TFLOPS: 69.5180810911642, Tokens per sec: 172273.46299272613, Loss: 3.5005555152893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3998 , TFLOPS: 68.97547024619247, Tokens per sec: 170928.81354536716, Loss: 3.4938406944274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3999 , TFLOPS: 68.25356406743077, Tokens per sec: 169139.85050986515, Loss: 3.5086686611175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4000 , TFLOPS: 69.66258472391858, Tokens per sec: 172631.55891308072, Loss: 3.5349323749542236 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/model.pt +[rank0]:[2024-08-25 04:59:50,747] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006379272002959624, 'preprocessing_with_comm': 0.001404552996973507, 'state_converting': 0.9798528190003708, : 0.9890012309988379}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000, took 4.18s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002000 + eval ppl=22.67022705078125, eval loss=3.1210525035858154 +------------------------------------------------------------------ +iteration: 4001 , TFLOPS: 63.68305063882831, Tokens per sec: 157813.61475017038, Loss: 3.506965160369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4002 , TFLOPS: 68.76739920953754, Tokens per sec: 170413.1905956194, Loss: 3.4852144718170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4003 , TFLOPS: 70.52410349970954, Tokens per sec: 174766.49734361842, Loss: 3.5015337467193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4004 , TFLOPS: 70.56116743892302, Tokens per sec: 174858.34586791936, Loss: 3.4949536323547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4005 , TFLOPS: 67.56659932040601, Tokens per sec: 167437.47619132375, Loss: 3.505267858505249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4006 , TFLOPS: 68.21183393296371, Tokens per sec: 169036.43863970196, Loss: 3.513152837753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4007 , TFLOPS: 70.52868386769853, Tokens per sec: 174777.84800006432, Loss: 3.5004711151123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4008 , TFLOPS: 67.9012295444982, Tokens per sec: 168266.72674918643, Loss: 3.5198683738708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4009 , TFLOPS: 67.40685972075637, Tokens per sec: 167041.62386662237, Loss: 3.4835245609283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4010 , TFLOPS: 70.59230169781145, Tokens per sec: 174935.49999116667, Loss: 3.4905166625976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4011 , TFLOPS: 69.75461984894348, Tokens per sec: 172859.63209139838, Loss: 3.4904634952545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4012 , TFLOPS: 67.87948418870046, Tokens per sec: 168212.83936207095, Loss: 3.492873191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4013 , TFLOPS: 69.65779894955527, Tokens per sec: 172619.69923701187, Loss: 3.491363525390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4014 , TFLOPS: 70.59159916148023, Tokens per sec: 174933.75902874712, Loss: 3.4981093406677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4015 , TFLOPS: 67.56111312325645, Tokens per sec: 167423.8807904326, Loss: 3.50714111328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4016 , TFLOPS: 67.67584037536946, Tokens per sec: 167708.1875594188, Loss: 3.5030219554901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4017 , TFLOPS: 70.61324153248722, Tokens per sec: 174987.39120820645, Loss: 3.4883534908294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4018 , TFLOPS: 68.73656370163314, Tokens per sec: 170336.77681022068, Loss: 3.4937047958374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4019 , TFLOPS: 68.00073878707848, Tokens per sec: 168513.32161414833, Loss: 3.504669189453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4020 , TFLOPS: 69.47692689244411, Tokens per sec: 172171.4783547885, Loss: 3.5009593963623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4021 , TFLOPS: 68.95151935980843, Tokens per sec: 170869.46061049958, Loss: 3.501850128173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4022 , TFLOPS: 67.32584929514934, Tokens per sec: 166840.87110793308, Loss: 3.4646668434143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4023 , TFLOPS: 68.91123650503211, Tokens per sec: 170769.63525884107, Loss: 3.510518789291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4024 , TFLOPS: 70.57096930681907, Tokens per sec: 174882.63597633067, Loss: 3.514841318130493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4025 , TFLOPS: 68.77946399300177, Tokens per sec: 170443.08845808933, Loss: 3.492262840270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4026 , TFLOPS: 68.33240645577551, Tokens per sec: 169335.23063338947, Loss: 3.4915311336517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4027 , TFLOPS: 68.85865694402383, Tokens per sec: 170639.3373145458, Loss: 3.5036580562591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4028 , TFLOPS: 68.9396163213991, Tokens per sec: 170839.96357009333, Loss: 3.492802619934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4029 , TFLOPS: 67.05086190207803, Tokens per sec: 166159.42205554032, Loss: 3.4976234436035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4030 , TFLOPS: 69.54821863977185, Tokens per sec: 172348.1471581009, Loss: 3.47194766998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4031 , TFLOPS: 69.65100221266083, Tokens per sec: 172602.85617426506, Loss: 3.51981258392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4032 , TFLOPS: 68.03078631665706, Tokens per sec: 168587.7827024527, Loss: 3.519540309906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4033 , TFLOPS: 69.65631218286896, Tokens per sec: 172616.0148653823, Loss: 3.509507894515991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4034 , TFLOPS: 69.65511678889119, Tokens per sec: 172613.05254742192, Loss: 3.4934864044189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4035 , TFLOPS: 67.21053251215034, Tokens per sec: 166555.1034164692, Loss: 3.5074472427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4036 , TFLOPS: 68.70568000387448, Tokens per sec: 170260.2436050545, Loss: 3.484488010406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4037 , TFLOPS: 70.52572084071559, Tokens per sec: 174770.50529279624, Loss: 3.5000627040863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4038 , TFLOPS: 67.7917913128274, Tokens per sec: 167995.52675548976, Loss: 3.4877333641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4039 , TFLOPS: 68.85666505498654, Tokens per sec: 170634.40119408665, Loss: 3.5132369995117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4040 , TFLOPS: 69.89942492708285, Tokens per sec: 173218.47502662393, Loss: 3.501166343688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4041 , TFLOPS: 69.70857404225004, Tokens per sec: 172745.5255100477, Loss: 3.472184419631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4042 , TFLOPS: 67.30440543106748, Tokens per sec: 166787.73085050157, Loss: 3.487412452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4043 , TFLOPS: 69.00149591321734, Tokens per sec: 170993.3080152187, Loss: 3.5014233589172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4044 , TFLOPS: 70.5874146446978, Tokens per sec: 174923.3893351977, Loss: 3.493669033050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4045 , TFLOPS: 68.01764150252275, Tokens per sec: 168555.20840500708, Loss: 3.50022029876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4046 , TFLOPS: 68.03137708836796, Tokens per sec: 168589.24669983122, Loss: 3.4708094596862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4047 , TFLOPS: 70.54061876797377, Tokens per sec: 174807.42399768415, Loss: 3.512159824371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4048 , TFLOPS: 68.94195972975241, Tokens per sec: 170845.77079414055, Loss: 3.512892246246338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4049 , TFLOPS: 66.15323224542219, Tokens per sec: 163934.99688427753, Loss: 3.502277135848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4050 , TFLOPS: 70.57696670446877, Tokens per sec: 174897.4981883745, Loss: 3.4855380058288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4051 , TFLOPS: 70.58907615113476, Tokens per sec: 174927.50673117852, Loss: 3.4871408939361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4052 , TFLOPS: 67.39984311156509, Tokens per sec: 167024.23593610307, Loss: 3.5051968097686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4053 , TFLOPS: 68.7616005629077, Tokens per sec: 170398.820910497, Loss: 3.5151143074035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4054 , TFLOPS: 70.55164106999217, Tokens per sec: 174834.73847629255, Loss: 3.528914213180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4055 , TFLOPS: 68.05344392026798, Tokens per sec: 168643.93074014166, Loss: 3.4734981060028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4056 , TFLOPS: 68.22865634664646, Tokens per sec: 169078.12643394968, Loss: 3.5107297897338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4057 , TFLOPS: 69.52246762209704, Tokens per sec: 172284.33329959397, Loss: 3.5126399993896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4058 , TFLOPS: 69.74145099033483, Tokens per sec: 172826.9982090966, Loss: 3.4932522773742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4059 , TFLOPS: 68.15955412319617, Tokens per sec: 168906.88351200154, Loss: 3.514150619506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4060 , TFLOPS: 68.68445376088592, Tokens per sec: 170207.6426366652, Loss: 3.5019752979278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4061 , TFLOPS: 69.78421655157193, Tokens per sec: 172932.97598085765, Loss: 3.494044065475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4062 , TFLOPS: 68.18518774097086, Tokens per sec: 168970.40644062334, Loss: 3.5208892822265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4063 , TFLOPS: 68.76156709126025, Tokens per sec: 170398.73796406738, Loss: 3.490123987197876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4064 , TFLOPS: 69.6904873888638, Tokens per sec: 172700.70479054726, Loss: 3.5182251930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4065 , TFLOPS: 68.76397448615869, Tokens per sec: 170404.7037538222, Loss: 3.5064077377319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4066 , TFLOPS: 68.00802378130653, Tokens per sec: 168531.37463235387, Loss: 3.486947536468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4067 , TFLOPS: 69.4849832757864, Tokens per sec: 172191.4429601946, Loss: 3.508481502532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4068 , TFLOPS: 69.04774047745727, Tokens per sec: 171107.90713966452, Loss: 3.5032339096069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4069 , TFLOPS: 68.02896058386953, Tokens per sec: 168583.25833548416, Loss: 3.4830780029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4070 , TFLOPS: 69.84179320306241, Tokens per sec: 173075.65726584278, Loss: 3.517753839492798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4071 , TFLOPS: 69.44936710444847, Tokens per sec: 172103.182164174, Loss: 3.5065600872039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4072 , TFLOPS: 67.9746055192688, Tokens per sec: 168448.5604976989, Loss: 3.4747047424316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4073 , TFLOPS: 68.94702300038412, Tokens per sec: 170858.31813653125, Loss: 3.5009119510650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4074 , TFLOPS: 70.6427594976997, Tokens per sec: 175060.53997767525, Loss: 3.4916858673095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4075 , TFLOPS: 68.47317514091063, Tokens per sec: 169684.0709421051, Loss: 3.478070020675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4076 , TFLOPS: 67.34379875101276, Tokens per sec: 166885.3518368584, Loss: 3.486499786376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4077 , TFLOPS: 70.60408169230604, Tokens per sec: 174964.69211519777, Loss: 3.5044336318969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4078 , TFLOPS: 69.68637151303871, Tokens per sec: 172690.50519685153, Loss: 3.5162851810455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4079 , TFLOPS: 66.21198981629881, Tokens per sec: 164080.60461759055, Loss: 3.4782192707061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4080 , TFLOPS: 69.65249310529381, Tokens per sec: 172606.550770445, Loss: 3.4681546688079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4081 , TFLOPS: 69.81769403751969, Tokens per sec: 173015.93687888567, Loss: 3.4850666522979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4082 , TFLOPS: 67.42006374720326, Tokens per sec: 167074.34489870133, Loss: 3.5173306465148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4083 , TFLOPS: 68.5403622657291, Tokens per sec: 169850.56803867806, Loss: 3.4974281787872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4084 , TFLOPS: 70.5707295111843, Tokens per sec: 174882.04173633203, Loss: 3.4975242614746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4085 , TFLOPS: 68.73838025966303, Tokens per sec: 170341.27844112914, Loss: 3.4947350025177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4086 , TFLOPS: 64.30509903358427, Tokens per sec: 159355.11919666314, Loss: 3.4952218532562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4087 , TFLOPS: 70.58772877989158, Tokens per sec: 174924.1677968127, Loss: 3.496110677719116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4088 , TFLOPS: 69.6593411653171, Tokens per sec: 172623.52101755876, Loss: 3.4825403690338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4089 , TFLOPS: 67.57853566180827, Tokens per sec: 167467.0557602151, Loss: 3.5177743434906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4090 , TFLOPS: 69.47475810996355, Tokens per sec: 172166.10387289053, Loss: 3.499326229095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4091 , TFLOPS: 70.61070985544843, Tokens per sec: 174981.11743361707, Loss: 3.501612901687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4092 , TFLOPS: 68.11336664311199, Tokens per sec: 168792.425848382, Loss: 3.48974347114563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4093 , TFLOPS: 68.66778314500058, Tokens per sec: 170166.3310140814, Loss: 3.489633560180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4094 , TFLOPS: 69.53620406009563, Tokens per sec: 172318.37370614804, Loss: 3.4961023330688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4095 , TFLOPS: 68.94763609559476, Tokens per sec: 170859.83745400122, Loss: 3.50579833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4096 , TFLOPS: 67.11786359077684, Tokens per sec: 166325.45962098124, Loss: 3.495065689086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4097 , TFLOPS: 69.61637678586253, Tokens per sec: 172517.05055235323, Loss: 3.4747824668884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4098 , TFLOPS: 69.7471886871716, Tokens per sec: 172841.21685391763, Loss: 3.4981577396392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4099 , TFLOPS: 67.8857730407395, Tokens per sec: 168228.42383019792, Loss: 3.5082345008850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4100 , TFLOPS: 69.56124299265299, Tokens per sec: 172380.4229392913, Loss: 3.5103187561035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4101 , TFLOPS: 69.5811233472736, Tokens per sec: 172429.68864804303, Loss: 3.5123374462127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4102 , TFLOPS: 67.42103196401594, Tokens per sec: 167076.74424662103, Loss: 3.488802194595337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4103 , TFLOPS: 68.89040925739265, Tokens per sec: 170718.02304487975, Loss: 3.504733085632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4104 , TFLOPS: 69.63933657788793, Tokens per sec: 172573.94744622207, Loss: 3.4986863136291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4105 , TFLOPS: 68.788299236353, Tokens per sec: 170464.9831643961, Loss: 3.505497932434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4106 , TFLOPS: 68.74736285739192, Tokens per sec: 170363.53830781614, Loss: 3.5000081062316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4107 , TFLOPS: 69.91230125174619, Tokens per sec: 173250.38397758393, Loss: 3.4726500511169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4108 , TFLOPS: 68.78870060384831, Tokens per sec: 170465.97779726377, Loss: 3.507117748260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4109 , TFLOPS: 68.06422568517597, Tokens per sec: 168670.64914129314, Loss: 3.5107975006103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4110 , TFLOPS: 68.47125547910596, Tokens per sec: 169679.31380868494, Loss: 3.474431276321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4111 , TFLOPS: 70.60718563211378, Tokens per sec: 174972.38401996857, Loss: 3.513899326324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4112 , TFLOPS: 67.8587062227455, Tokens per sec: 168161.34927355265, Loss: 3.5162367820739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4113 , TFLOPS: 68.67670093111366, Tokens per sec: 170188.43027044483, Loss: 3.514863967895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4114 , TFLOPS: 70.5965464853988, Tokens per sec: 174946.01904241007, Loss: 3.499042510986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4115 , TFLOPS: 69.06712696540004, Tokens per sec: 171155.94899238463, Loss: 3.5056285858154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4116 , TFLOPS: 66.94006733177163, Tokens per sec: 165884.86090529157, Loss: 3.4997506141662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4117 , TFLOPS: 70.55725539035096, Tokens per sec: 174848.65138061007, Loss: 3.4846391677856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4118 , TFLOPS: 70.60370520352697, Tokens per sec: 174963.7591345296, Loss: 3.492593288421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4119 , TFLOPS: 65.75444882483907, Tokens per sec: 162946.7676384523, Loss: 3.488662004470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4120 , TFLOPS: 69.59455496460342, Tokens per sec: 172462.9736754011, Loss: 3.5090510845184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4121 , TFLOPS: 70.56147924339808, Tokens per sec: 174859.11855375668, Loss: 3.487184524536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4122 , TFLOPS: 68.21711812199126, Tokens per sec: 169049.53344221372, Loss: 3.518080949783325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4123 , TFLOPS: 67.28852069111785, Tokens per sec: 166748.36671505784, Loss: 3.4952666759490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4124 , TFLOPS: 70.53096624125787, Tokens per sec: 174783.50397316855, Loss: 3.494602680206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4125 , TFLOPS: 69.67873428146427, Tokens per sec: 172671.57929569954, Loss: 3.475402593612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4126 , TFLOPS: 68.16946745494194, Tokens per sec: 168931.4498401112, Loss: 3.4776816368103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4127 , TFLOPS: 69.54955839529781, Tokens per sec: 172351.4672198805, Loss: 3.469325304031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4128 , TFLOPS: 69.82925094636433, Tokens per sec: 173044.57617210128, Loss: 3.5089950561523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4129 , TFLOPS: 68.08047063487709, Tokens per sec: 168710.90591618145, Loss: 3.482879638671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4130 , TFLOPS: 68.85680690677303, Tokens per sec: 170634.75271843982, Loss: 3.4902095794677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4131 , TFLOPS: 69.51475857023559, Tokens per sec: 172265.22941985913, Loss: 3.5137429237365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4132 , TFLOPS: 68.0064830850473, Tokens per sec: 168527.55661730177, Loss: 3.4956068992614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4133 , TFLOPS: 68.63218027372106, Tokens per sec: 170078.1031770707, Loss: 3.4844934940338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4134 , TFLOPS: 69.65734666615707, Tokens per sec: 172618.578428928, Loss: 3.494548797607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4135 , TFLOPS: 69.02750945826229, Tokens per sec: 171057.7724454686, Loss: 3.502824068069458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4136 , TFLOPS: 67.16586098560398, Tokens per sec: 166444.4024527125, Loss: 3.502830982208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4137 , TFLOPS: 70.48173196097659, Tokens per sec: 174661.49600302786, Loss: 3.4900026321411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4138 , TFLOPS: 69.56106224165806, Tokens per sec: 172379.97501841447, Loss: 3.477834463119507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4139 , TFLOPS: 67.89220805897529, Tokens per sec: 168244.37051426197, Loss: 3.4808509349823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4140 , TFLOPS: 69.00888662579895, Tokens per sec: 171011.6230151501, Loss: 3.483339548110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4141 , TFLOPS: 69.57939747357838, Tokens per sec: 172425.41174290556, Loss: 3.4985690116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4142 , TFLOPS: 69.04475593948003, Tokens per sec: 171100.5111257829, Loss: 3.4965462684631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4143 , TFLOPS: 66.7685485007064, Tokens per sec: 165459.81834755294, Loss: 3.499011993408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4144 , TFLOPS: 70.54381593754289, Tokens per sec: 174815.34693607374, Loss: 3.5056569576263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4145 , TFLOPS: 68.82833766835701, Tokens per sec: 170564.20280949943, Loss: 3.485447406768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4146 , TFLOPS: 67.93349807208044, Tokens per sec: 168346.691715207, Loss: 3.4935977458953857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4147 , TFLOPS: 69.70033009866819, Tokens per sec: 172725.09610970653, Loss: 3.5024678707122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4148 , TFLOPS: 69.81900070910875, Tokens per sec: 173019.17495502514, Loss: 3.489499092102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4149 , TFLOPS: 67.77758574506264, Tokens per sec: 167960.3237936962, Loss: 3.4839189052581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4150 , TFLOPS: 68.92381919455006, Tokens per sec: 170800.81654956422, Loss: 3.506535053253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4151 , TFLOPS: 70.55993782430816, Tokens per sec: 174855.29874745055, Loss: 3.484412670135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4152 , TFLOPS: 67.98379057337985, Tokens per sec: 168471.3220736033, Loss: 3.508822202682495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4153 , TFLOPS: 67.85221294276744, Tokens per sec: 168145.25821047317, Loss: 3.5040884017944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4154 , TFLOPS: 70.61453949913674, Tokens per sec: 174990.60771254674, Loss: 3.5081875324249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4155 , TFLOPS: 68.97252681598802, Tokens per sec: 170921.51940107494, Loss: 3.4720776081085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4156 , TFLOPS: 67.12105167842977, Tokens per sec: 166333.3600533821, Loss: 3.4965505599975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4157 , TFLOPS: 69.97475076083785, Tokens per sec: 173405.14074049442, Loss: 3.4719996452331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4158 , TFLOPS: 70.5816668663526, Tokens per sec: 174909.14570728823, Loss: 3.5143964290618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4159 , TFLOPS: 68.73687963267905, Tokens per sec: 170337.55972215507, Loss: 3.5050604343414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4160 , TFLOPS: 67.22332080017452, Tokens per sec: 166586.79420293937, Loss: 3.501708984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4161 , TFLOPS: 70.5720263670338, Tokens per sec: 174885.25548798757, Loss: 3.498579263687134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4162 , TFLOPS: 68.09229619793867, Tokens per sec: 168740.21096414133, Loss: 3.511543035507202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4163 , TFLOPS: 66.68306256701788, Tokens per sec: 165247.9747868198, Loss: 3.519172430038452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4164 , TFLOPS: 69.4895054953725, Tokens per sec: 172202.64951849254, Loss: 3.5078279972076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4165 , TFLOPS: 69.64812070791908, Tokens per sec: 172595.71548809105, Loss: 3.499873638153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4166 , TFLOPS: 67.61555797397673, Tokens per sec: 167558.8011281129, Loss: 3.4902684688568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4167 , TFLOPS: 69.67990142423464, Tokens per sec: 172674.47160405593, Loss: 3.5160253047943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4168 , TFLOPS: 69.64141662209761, Tokens per sec: 172579.10202490943, Loss: 3.512664318084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4169 , TFLOPS: 68.34524426832947, Tokens per sec: 169367.04414709995, Loss: 3.4824111461639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4170 , TFLOPS: 68.36122549724107, Tokens per sec: 169406.6473928206, Loss: 3.5144948959350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4171 , TFLOPS: 69.57357337549817, Tokens per sec: 172410.97898628545, Loss: 3.5209784507751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4172 , TFLOPS: 68.88846200695734, Tokens per sec: 170713.19754379368, Loss: 3.5082991123199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4173 , TFLOPS: 68.35686350389305, Tokens per sec: 169395.83789279102, Loss: 3.5133841037750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4174 , TFLOPS: 69.86148941558915, Tokens per sec: 173124.46665020098, Loss: 3.4926226139068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4175 , TFLOPS: 68.87837017659406, Tokens per sec: 170688.18887644663, Loss: 3.4999663829803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4176 , TFLOPS: 68.23410285991575, Tokens per sec: 169091.62349381394, Loss: 3.4997081756591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4177 , TFLOPS: 69.13969041240023, Tokens per sec: 171335.7692075747, Loss: 3.5029845237731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4178 , TFLOPS: 69.46629736748405, Tokens per sec: 172145.13722099861, Loss: 3.5064661502838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4179 , TFLOPS: 68.79617283015557, Tokens per sec: 170484.4947971868, Loss: 3.5173754692077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4180 , TFLOPS: 67.87439134829546, Tokens per sec: 168200.21874252468, Loss: 3.5332069396972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4181 , TFLOPS: 70.58849818239037, Tokens per sec: 174926.0744609614, Loss: 3.487738609313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4182 , TFLOPS: 68.07141586717161, Tokens per sec: 168688.46720434225, Loss: 3.5227861404418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4183 , TFLOPS: 67.15161375631551, Tokens per sec: 166409.09624907377, Loss: 3.492553234100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4184 , TFLOPS: 70.59960026190225, Tokens per sec: 174953.58663698687, Loss: 3.496567726135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4185 , TFLOPS: 70.59201222929794, Tokens per sec: 174934.7826563026, Loss: 3.500112771987915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4186 , TFLOPS: 67.23709482809699, Tokens per sec: 166620.92776146566, Loss: 3.5139708518981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4187 , TFLOPS: 69.6936410045242, Tokens per sec: 172708.51979755316, Loss: 3.508174419403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4188 , TFLOPS: 70.45281910408266, Tokens per sec: 174589.84675863056, Loss: 3.5008554458618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4189 , TFLOPS: 67.99498219469102, Tokens per sec: 168499.05614407122, Loss: 3.504098653793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4190 , TFLOPS: 67.04936707911612, Tokens per sec: 166155.71771957236, Loss: 3.4882452487945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4191 , TFLOPS: 70.63867307619341, Tokens per sec: 175050.41337502617, Loss: 3.494281053543091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4192 , TFLOPS: 69.70165345912577, Tokens per sec: 172728.37554270058, Loss: 3.4865834712982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4193 , TFLOPS: 67.4245432151649, Tokens per sec: 167085.44551376443, Loss: 3.4844703674316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4194 , TFLOPS: 69.64221405787876, Tokens per sec: 172581.07815861993, Loss: 3.4851651191711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4195 , TFLOPS: 69.8305265510907, Tokens per sec: 173047.73726113272, Loss: 3.50394606590271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4196 , TFLOPS: 67.85854835355117, Tokens per sec: 168160.95805629838, Loss: 3.500211715698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4197 , TFLOPS: 67.27332696838103, Tokens per sec: 166710.71499638926, Loss: 3.4835033416748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4198 , TFLOPS: 70.50092797002573, Tokens per sec: 174709.06582806655, Loss: 3.5009846687316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4199 , TFLOPS: 68.7656985860209, Tokens per sec: 170408.97626320602, Loss: 3.505175828933716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4200 , TFLOPS: 67.82160584635027, Tokens per sec: 168069.41045390142, Loss: 3.5160508155822754 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/model.pt +[rank0]:[2024-08-25 05:25:34,767] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006372570001985878, 'preprocessing_with_comm': 0.0012768390006385744, 'state_converting': 1.0083786690011038, : 1.017449854996812}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002200 + eval ppl=24.661441802978516, eval loss=3.2052409648895264 +------------------------------------------------------------------ +iteration: 4201 , TFLOPS: 65.33983508143173, Tokens per sec: 161919.30910881463, Loss: 3.5007827281951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4202 , TFLOPS: 68.9124276713819, Tokens per sec: 170772.58710027364, Loss: 3.502925157546997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4203 , TFLOPS: 67.2050057253202, Tokens per sec: 166541.4074298783, Loss: 3.4836061000823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4204 , TFLOPS: 69.4594658480799, Tokens per sec: 172128.20796336114, Loss: 3.50917649269104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4205 , TFLOPS: 70.56877178307231, Tokens per sec: 174877.19027040896, Loss: 3.4702529907226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4206 , TFLOPS: 66.55107593234065, Tokens per sec: 164920.89736656152, Loss: 3.5024402141571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4207 , TFLOPS: 69.57622401486327, Tokens per sec: 172417.5475626242, Loss: 3.4940378665924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4208 , TFLOPS: 69.5112616789804, Tokens per sec: 172256.5637381144, Loss: 3.473388195037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4209 , TFLOPS: 68.0683674349491, Tokens per sec: 168680.91285348232, Loss: 3.4943225383758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4210 , TFLOPS: 68.29249157166518, Tokens per sec: 169236.31715357662, Loss: 3.495635986328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4211 , TFLOPS: 70.51296312056895, Tokens per sec: 174738.89025689397, Loss: 3.505034923553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4212 , TFLOPS: 68.56387264528081, Tokens per sec: 169908.8293490893, Loss: 3.4710755348205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4213 , TFLOPS: 66.4374741869203, Tokens per sec: 164639.37972714455, Loss: 3.4934802055358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4214 , TFLOPS: 70.551824739781, Tokens per sec: 174835.19363026216, Loss: 3.488219738006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4215 , TFLOPS: 69.379671081803, Tokens per sec: 171930.46774240784, Loss: 3.503725051879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4216 , TFLOPS: 67.18662095082058, Tokens per sec: 166495.84793341815, Loss: 3.4830756187438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4217 , TFLOPS: 70.08564897618146, Tokens per sec: 173679.9587345628, Loss: 3.4818382263183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4218 , TFLOPS: 70.56350403641237, Tokens per sec: 174864.13621389543, Loss: 3.5119926929473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4219 , TFLOPS: 68.61281871341681, Tokens per sec: 170030.12309778485, Loss: 3.5017809867858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4220 , TFLOPS: 68.19137314751893, Tokens per sec: 168985.73455942824, Loss: 3.4942944049835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4221 , TFLOPS: 70.55878177832248, Tokens per sec: 174852.43393814904, Loss: 3.488101005554199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4222 , TFLOPS: 68.91283837326469, Tokens per sec: 170773.60486478175, Loss: 3.5172181129455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4223 , TFLOPS: 66.31536034296697, Tokens per sec: 164336.76816987782, Loss: 3.4850451946258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4224 , TFLOPS: 70.56119718421613, Tokens per sec: 174858.4195800324, Loss: 3.4851176738739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4225 , TFLOPS: 70.54488393407543, Tokens per sec: 174817.99354913092, Loss: 3.4831244945526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4226 , TFLOPS: 67.3456567673218, Tokens per sec: 166889.9562059496, Loss: 3.4716975688934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4227 , TFLOPS: 67.94572642517932, Tokens per sec: 168376.9949212503, Loss: 3.4783108234405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4228 , TFLOPS: 70.60714403046588, Tokens per sec: 174972.28092650225, Loss: 3.496333360671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4229 , TFLOPS: 68.37008712286543, Tokens per sec: 169428.60747730642, Loss: 3.4973394870758057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4230 , TFLOPS: 66.47032708286264, Tokens per sec: 164720.79282233494, Loss: 3.4853601455688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4231 , TFLOPS: 70.56781227475379, Tokens per sec: 174874.8125030971, Loss: 3.5141119956970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4232 , TFLOPS: 69.61969674219345, Tokens per sec: 172525.27776986465, Loss: 3.5020055770874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4233 , TFLOPS: 67.74353562271043, Tokens per sec: 167875.9438395768, Loss: 3.483813524246216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4234 , TFLOPS: 68.83858329998054, Tokens per sec: 170589.59261331204, Loss: 3.506761074066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4235 , TFLOPS: 69.77614144224546, Tokens per sec: 172912.9649703999, Loss: 3.5129780769348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4236 , TFLOPS: 68.218504442281, Tokens per sec: 169052.96889660746, Loss: 3.4991140365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4237 , TFLOPS: 69.09281000499189, Tokens per sec: 171219.5943937136, Loss: 3.497060537338257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4238 , TFLOPS: 69.53288208968227, Tokens per sec: 172310.1414975185, Loss: 3.475755214691162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4239 , TFLOPS: 68.6411590652633, Tokens per sec: 170100.3536116079, Loss: 3.486602306365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4240 , TFLOPS: 67.85014695418015, Tokens per sec: 168140.13846316646, Loss: 3.4960782527923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4241 , TFLOPS: 70.56174670003715, Tokens per sec: 174859.7813407711, Loss: 3.5096185207366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4242 , TFLOPS: 68.88628428418023, Tokens per sec: 170707.80090685605, Loss: 3.488786458969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4243 , TFLOPS: 66.83479441775697, Tokens per sec: 165623.98302759443, Loss: 3.5003583431243896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4244 , TFLOPS: 70.64496954077417, Tokens per sec: 175066.01670787012, Loss: 3.490787982940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4245 , TFLOPS: 69.53143545127834, Tokens per sec: 172306.55656819316, Loss: 3.487992525100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4246 , TFLOPS: 67.19275043068978, Tokens per sec: 166511.037459752, Loss: 3.4781150817871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4247 , TFLOPS: 68.71666736639223, Tokens per sec: 170287.47149973141, Loss: 3.4699535369873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4248 , TFLOPS: 70.56822520535742, Tokens per sec: 174875.835790621, Loss: 3.473940372467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4249 , TFLOPS: 68.05738701574118, Tokens per sec: 168653.70216509208, Loss: 3.492915630340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4250 , TFLOPS: 66.79940365072947, Tokens per sec: 165536.2808082855, Loss: 3.4761064052581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4251 , TFLOPS: 70.54921646609843, Tokens per sec: 174828.7300407517, Loss: 3.517554998397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4252 , TFLOPS: 68.74875868994883, Tokens per sec: 170366.9973346558, Loss: 3.499525547027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4253 , TFLOPS: 67.85452152331511, Tokens per sec: 168150.97912738146, Loss: 3.475965976715088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4254 , TFLOPS: 69.63692068111723, Tokens per sec: 172567.96058789128, Loss: 3.49957013130188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4255 , TFLOPS: 70.49140111489464, Tokens per sec: 174685.45723158203, Loss: 3.496429681777954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4256 , TFLOPS: 67.54775118102145, Tokens per sec: 167390.76842563483, Loss: 3.5068955421447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4257 , TFLOPS: 68.8646601120081, Tokens per sec: 170654.2138261143, Loss: 3.4943578243255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4258 , TFLOPS: 70.59429095658005, Tokens per sec: 174940.4295935302, Loss: 3.4902641773223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4259 , TFLOPS: 68.72275736942106, Tokens per sec: 170302.56319810607, Loss: 3.5006613731384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4260 , TFLOPS: 67.82230247968037, Tokens per sec: 168071.1367880339, Loss: 3.4902939796447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4261 , TFLOPS: 70.57615241316077, Tokens per sec: 174895.48028481173, Loss: 3.48500394821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4262 , TFLOPS: 69.8126715519345, Tokens per sec: 173003.49060633313, Loss: 3.4975290298461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4263 , TFLOPS: 66.51804663558497, Tokens per sec: 164839.04713072354, Loss: 3.509483814239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4264 , TFLOPS: 69.50507639865131, Tokens per sec: 172241.23593209404, Loss: 3.5039608478546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4265 , TFLOPS: 70.61592424108399, Tokens per sec: 174994.03925563395, Loss: 3.483994245529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4266 , TFLOPS: 67.23787440946228, Tokens per sec: 166622.85964996682, Loss: 3.482149839401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4267 , TFLOPS: 67.83901031901891, Tokens per sec: 168112.54065443526, Loss: 3.518101930618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4268 , TFLOPS: 70.54008389499332, Tokens per sec: 174806.0985235175, Loss: 3.5097289085388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4269 , TFLOPS: 68.97016600101689, Tokens per sec: 170915.66904151122, Loss: 3.4826278686523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4270 , TFLOPS: 67.14413005208664, Tokens per sec: 166390.55080559652, Loss: 3.511993408203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4271 , TFLOPS: 69.57040541210063, Tokens per sec: 172403.12842400637, Loss: 3.4985358715057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4272 , TFLOPS: 70.59609717927327, Tokens per sec: 174944.90561233586, Loss: 3.495189905166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4273 , TFLOPS: 67.33963562916107, Tokens per sec: 166875.03516230415, Loss: 3.4730637073516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4274 , TFLOPS: 69.59688146588591, Tokens per sec: 172468.7390018644, Loss: 3.4725074768066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4275 , TFLOPS: 69.62355837871995, Tokens per sec: 172534.8473305717, Loss: 3.5270652770996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4276 , TFLOPS: 68.12508541090665, Tokens per sec: 168821.466245906, Loss: 3.474876642227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4277 , TFLOPS: 68.01871743876376, Tokens per sec: 168557.87469353646, Loss: 3.4727866649627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4278 , TFLOPS: 69.49625639007049, Tokens per sec: 172219.3789792266, Loss: 3.4915053844451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4279 , TFLOPS: 69.64635192066542, Tokens per sec: 172591.33223842853, Loss: 3.5042972564697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4280 , TFLOPS: 67.36741823375795, Tokens per sec: 166943.88351700178, Loss: 3.485095500946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4281 , TFLOPS: 70.63678886515756, Tokens per sec: 175045.74409251634, Loss: 3.507094144821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4282 , TFLOPS: 68.81235742466548, Tokens per sec: 170524.60200526266, Loss: 3.508277416229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4283 , TFLOPS: 67.12073743241204, Tokens per sec: 166332.58131713307, Loss: 3.5043866634368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4284 , TFLOPS: 69.9453168501541, Tokens per sec: 173332.20026740717, Loss: 3.5072922706604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4285 , TFLOPS: 70.54943527355202, Tokens per sec: 174829.27226972533, Loss: 3.4968907833099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4286 , TFLOPS: 67.85802836700014, Tokens per sec: 168159.66947234372, Loss: 3.4942917823791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4287 , TFLOPS: 68.3377564547602, Tokens per sec: 169348.48852022455, Loss: 3.499999761581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4288 , TFLOPS: 70.56430074961432, Tokens per sec: 174866.1105569751, Loss: 3.4996845722198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4289 , TFLOPS: 68.74741900733868, Tokens per sec: 170363.67745356963, Loss: 3.4775567054748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4290 , TFLOPS: 67.03785668590304, Tokens per sec: 166127.19369721581, Loss: 3.523710012435913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4291 , TFLOPS: 70.5429982376575, Tokens per sec: 174813.32058568066, Loss: 3.4968504905700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4292 , TFLOPS: 70.53520269697016, Tokens per sec: 174794.00237710777, Loss: 3.519214391708374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4293 , TFLOPS: 66.95659553009907, Tokens per sec: 165925.81960147832, Loss: 3.5169341564178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4294 , TFLOPS: 68.85896290201599, Tokens per sec: 170640.09551215437, Loss: 3.4929561614990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4295 , TFLOPS: 70.57470469348745, Tokens per sec: 174891.8926759817, Loss: 3.5000736713409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4296 , TFLOPS: 69.09938562797436, Tokens per sec: 171235.88951182886, Loss: 3.4839558601379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4297 , TFLOPS: 66.39151249696994, Tokens per sec: 164525.48159634945, Loss: 3.509268045425415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4298 , TFLOPS: 70.6030905673348, Tokens per sec: 174962.2359983376, Loss: 3.508988857269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4299 , TFLOPS: 69.62322205816353, Tokens per sec: 172534.01389118412, Loss: 3.4802651405334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4300 , TFLOPS: 67.87673316650839, Tokens per sec: 168206.0220260289, Loss: 3.5242981910705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4301 , TFLOPS: 68.9575053467448, Tokens per sec: 170884.29454553875, Loss: 3.505582332611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4302 , TFLOPS: 69.78152694633745, Tokens per sec: 172926.3108428021, Loss: 3.4775376319885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4303 , TFLOPS: 68.13022661276108, Tokens per sec: 168834.2067104514, Loss: 3.4972290992736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4304 , TFLOPS: 68.03277831967264, Tokens per sec: 168592.7191053626, Loss: 3.5041136741638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4305 , TFLOPS: 70.54211782748102, Tokens per sec: 174811.13883227817, Loss: 3.4795682430267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4306 , TFLOPS: 68.0191490515152, Tokens per sec: 168558.9442774806, Loss: 3.508661985397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4307 , TFLOPS: 68.0140826859751, Tokens per sec: 168546.38926556602, Loss: 3.516275405883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4308 , TFLOPS: 69.66884313953219, Tokens per sec: 172647.06795639262, Loss: 3.4912426471710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4309 , TFLOPS: 68.96944482415071, Tokens per sec: 170913.88188579265, Loss: 3.490464687347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4310 , TFLOPS: 67.05988776545394, Tokens per sec: 166181.78913926674, Loss: 3.504470109939575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4311 , TFLOPS: 70.58676747608935, Tokens per sec: 174921.78558009444, Loss: 3.4954724311828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4312 , TFLOPS: 69.5704421441738, Tokens per sec: 172403.21945013007, Loss: 3.520935297012329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4313 , TFLOPS: 67.96605685507062, Tokens per sec: 168427.37596610267, Loss: 3.5031023025512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4314 , TFLOPS: 68.8283629640971, Tokens per sec: 170564.26549512992, Loss: 3.512629270553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4315 , TFLOPS: 70.51135737997622, Tokens per sec: 174734.91105481825, Loss: 3.5079386234283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4316 , TFLOPS: 68.10748646868652, Tokens per sec: 168777.85412840097, Loss: 3.490739583969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4317 , TFLOPS: 67.06752247549187, Tokens per sec: 166200.70879177348, Loss: 3.507488250732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4318 , TFLOPS: 70.53316182448805, Tokens per sec: 174788.9448702764, Loss: 3.4836618900299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4319 , TFLOPS: 68.55533779541443, Tokens per sec: 169887.67905092912, Loss: 3.5042166709899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4320 , TFLOPS: 67.94007202891389, Tokens per sec: 168362.9827044214, Loss: 3.49688458442688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4321 , TFLOPS: 69.69773620452179, Tokens per sec: 172718.6681542707, Loss: 3.4890546798706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4322 , TFLOPS: 70.48756147623277, Tokens per sec: 174675.9421840068, Loss: 3.5033793449401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4323 , TFLOPS: 67.34891177671742, Tokens per sec: 166898.0224778008, Loss: 3.5058164596557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4324 , TFLOPS: 68.18876128975273, Tokens per sec: 168979.2620881637, Loss: 3.4958724975585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4325 , TFLOPS: 70.63385208619245, Tokens per sec: 175038.4664307279, Loss: 3.5157675743103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4326 , TFLOPS: 67.83770953411803, Tokens per sec: 168109.31716615762, Loss: 3.531826972961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4327 , TFLOPS: 67.47795634390079, Tokens per sec: 167217.80913071384, Loss: 3.4962751865386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4328 , TFLOPS: 70.52834885520556, Tokens per sec: 174777.01780220208, Loss: 3.4781785011291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4329 , TFLOPS: 69.77940324476985, Tokens per sec: 172921.0480763726, Loss: 3.4831182956695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4330 , TFLOPS: 67.30407601587169, Tokens per sec: 166786.9145233583, Loss: 3.497002363204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4331 , TFLOPS: 69.87787848852106, Tokens per sec: 173165.0805783317, Loss: 3.51557993888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4332 , TFLOPS: 70.57335293183826, Tokens per sec: 174888.5428617062, Loss: 3.5077035427093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4333 , TFLOPS: 68.7636355139916, Tokens per sec: 170403.86374345122, Loss: 3.5258421897888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4334 , TFLOPS: 68.12949078020893, Tokens per sec: 168832.3832363285, Loss: 3.4878785610198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4335 , TFLOPS: 70.59462809257919, Tokens per sec: 174941.26505367467, Loss: 3.502640724182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4336 , TFLOPS: 69.06571459828818, Tokens per sec: 171152.44899109637, Loss: 3.5168323516845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4337 , TFLOPS: 67.17006792601839, Tokens per sec: 166454.8277144921, Loss: 3.4887354373931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4338 , TFLOPS: 69.54568406116815, Tokens per sec: 172341.86619311405, Loss: 3.4864375591278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4339 , TFLOPS: 70.5931211246751, Tokens per sec: 174937.53062120208, Loss: 3.474821090698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4340 , TFLOPS: 67.38453867589534, Tokens per sec: 166986.30985859066, Loss: 3.5022807121276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4341 , TFLOPS: 68.56779638122285, Tokens per sec: 169918.55279898888, Loss: 3.514230728149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4342 , TFLOPS: 70.51117803404837, Tokens per sec: 174734.46661585246, Loss: 3.493802070617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4343 , TFLOPS: 68.04866378387781, Tokens per sec: 168632.08503559136, Loss: 3.497584342956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4344 , TFLOPS: 68.0086518014002, Tokens per sec: 168532.9309353281, Loss: 3.4965999126434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4345 , TFLOPS: 69.45659140344884, Tokens per sec: 172121.08477291916, Loss: 3.5179014205932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4346 , TFLOPS: 69.82037119281063, Tokens per sec: 173022.57116460963, Loss: 3.523839235305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4347 , TFLOPS: 66.57521859794673, Tokens per sec: 164980.72555146756, Loss: 3.5129709243774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4348 , TFLOPS: 70.57940190066643, Tokens per sec: 174903.5328728113, Loss: 3.5072927474975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4349 , TFLOPS: 68.75817336317881, Tokens per sec: 170390.32793203156, Loss: 3.4972410202026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4350 , TFLOPS: 67.26924693167051, Tokens per sec: 166700.60421596747, Loss: 3.5134787559509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4351 , TFLOPS: 69.59670499423476, Tokens per sec: 172468.3016856729, Loss: 3.5087363719940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4352 , TFLOPS: 69.54051236185626, Tokens per sec: 172329.05015250947, Loss: 3.509397029876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4353 , TFLOPS: 68.83918333345075, Tokens per sec: 170591.07956235035, Loss: 3.5061211585998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4354 , TFLOPS: 67.92775678868165, Tokens per sec: 168332.46418249022, Loss: 3.4789769649505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4355 , TFLOPS: 70.53607017290084, Tokens per sec: 174796.15207802432, Loss: 3.4939968585968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4356 , TFLOPS: 68.00381606896529, Tokens per sec: 168520.94745765446, Loss: 3.4838480949401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4357 , TFLOPS: 67.16114740919545, Tokens per sec: 166432.72169112883, Loss: 3.4447519779205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4358 , TFLOPS: 70.58126057207507, Tokens per sec: 174908.13886531148, Loss: 3.4917092323303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4359 , TFLOPS: 70.5949859218849, Tokens per sec: 174942.15179411235, Loss: 3.5126118659973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4360 , TFLOPS: 67.90292496990679, Tokens per sec: 168270.92820011548, Loss: 3.510687828063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4361 , TFLOPS: 69.07291196041801, Tokens per sec: 171170.28484730897, Loss: 3.4822611808776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4362 , TFLOPS: 70.5983884461813, Tokens per sec: 174950.58362413786, Loss: 3.503920316696167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4363 , TFLOPS: 67.79708449376355, Tokens per sec: 168008.64384094157, Loss: 3.477659225463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4364 , TFLOPS: 67.38108094227921, Tokens per sec: 166977.7412137891, Loss: 3.4896762371063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4365 , TFLOPS: 70.53800906721818, Tokens per sec: 174800.95687172955, Loss: 3.488236904144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4366 , TFLOPS: 69.82369802826874, Tokens per sec: 173030.81542935604, Loss: 3.4967973232269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4367 , TFLOPS: 67.09323082891962, Tokens per sec: 166264.41692357635, Loss: 3.494835376739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4368 , TFLOPS: 69.93588245259753, Tokens per sec: 173308.82079097728, Loss: 3.4765846729278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4369 , TFLOPS: 70.56676889443433, Tokens per sec: 174872.22689172765, Loss: 3.4815032482147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4370 , TFLOPS: 67.4257165987708, Tokens per sec: 167088.35328759876, Loss: 3.4964799880981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4371 , TFLOPS: 68.7995919999671, Tokens per sec: 170492.96787663247, Loss: 3.5238237380981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4372 , TFLOPS: 70.55166482778588, Tokens per sec: 174834.7973507225, Loss: 3.5046005249023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4373 , TFLOPS: 68.86315050004006, Tokens per sec: 170650.47284136002, Loss: 3.502379894256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4374 , TFLOPS: 67.80751108371587, Tokens per sec: 168034.4820794271, Loss: 3.5151591300964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4375 , TFLOPS: 69.61431958289049, Tokens per sec: 172511.9525767689, Loss: 3.4964141845703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4376 , TFLOPS: 68.8263683982835, Tokens per sec: 170559.3227413243, Loss: 3.501433849334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4377 , TFLOPS: 67.2570351492208, Tokens per sec: 166670.34207381817, Loss: 3.483125925064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4378 , TFLOPS: 69.50517159700614, Tokens per sec: 172241.47184410426, Loss: 3.483412027359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4379 , TFLOPS: 70.53374796480013, Tokens per sec: 174790.39739053845, Loss: 3.4997551441192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4380 , TFLOPS: 68.04582576107472, Tokens per sec: 168625.05210245127, Loss: 3.504141092300415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4381 , TFLOPS: 68.6440067823281, Tokens per sec: 170107.41056819644, Loss: 3.497272253036499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4382 , TFLOPS: 69.67965966749267, Tokens per sec: 172673.8725042176, Loss: 3.5018482208251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4383 , TFLOPS: 68.7418401706947, Tokens per sec: 170349.85248180624, Loss: 3.5094804763793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4384 , TFLOPS: 67.2246692840659, Tokens per sec: 166590.13589456995, Loss: 3.496879816055298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4385 , TFLOPS: 70.49902510598643, Tokens per sec: 174704.35032135944, Loss: 3.473992109298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4386 , TFLOPS: 68.72945670066602, Tokens per sec: 170319.16487892333, Loss: 3.486510753631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4387 , TFLOPS: 67.21631261604492, Tokens per sec: 166569.427150651, Loss: 3.477217674255371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4388 , TFLOPS: 69.69536422892561, Tokens per sec: 172712.7901374487, Loss: 3.4807026386260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4389 , TFLOPS: 70.5406495954944, Tokens per sec: 174807.5003916762, Loss: 3.5098037719726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4390 , TFLOPS: 66.10521777022437, Tokens per sec: 163816.01172550625, Loss: 3.482534408569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4391 , TFLOPS: 68.88554843161687, Tokens per sec: 170705.97738314318, Loss: 3.511202335357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4392 , TFLOPS: 70.60582219299724, Tokens per sec: 174969.00526764253, Loss: 3.489210367202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4393 , TFLOPS: 67.9804427899003, Tokens per sec: 168463.02589735217, Loss: 3.4723706245422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4394 , TFLOPS: 67.79632436023644, Tokens per sec: 168006.76014632577, Loss: 3.4780662059783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4395 , TFLOPS: 70.56372402353176, Tokens per sec: 174864.6813662109, Loss: 3.4842891693115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4396 , TFLOPS: 69.62919565027177, Tokens per sec: 172548.8171104738, Loss: 3.5012400150299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4397 , TFLOPS: 65.75560754382376, Tokens per sec: 162949.63907173977, Loss: 3.481971263885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4398 , TFLOPS: 70.61504045558573, Tokens per sec: 174991.84913781227, Loss: 3.49371337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4399 , TFLOPS: 70.58213631377838, Tokens per sec: 174910.30904972318, Loss: 3.4705700874328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4400 , TFLOPS: 67.07341190060109, Tokens per sec: 166215.30343596783, Loss: 3.4657766819000244 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/model.pt +[rank0]:[2024-08-25 05:51:19,353] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006367144000250846, 'preprocessing_with_comm': 0.001241352001670748, 'state_converting': 0.9970175149937859, : 1.0060352079963195}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400, took 4.49s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002400 + eval ppl=25.879331588745117, eval loss=3.2534446716308594 +------------------------------------------------------------------ +iteration: 4401 , TFLOPS: 65.49020049322492, Tokens per sec: 162291.93116335565, Loss: 3.509577512741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4402 , TFLOPS: 70.52242555815978, Tokens per sec: 174762.3392196176, Loss: 3.5258193016052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4403 , TFLOPS: 69.65670269261008, Tokens per sec: 172616.982591539, Loss: 3.5019876956939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4404 , TFLOPS: 66.12037231698045, Tokens per sec: 163853.56636177894, Loss: 3.4995312690734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4405 , TFLOPS: 70.53315172307225, Tokens per sec: 174788.9198378553, Loss: 3.4817469120025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4406 , TFLOPS: 69.6999413752547, Tokens per sec: 172724.13281026646, Loss: 3.4895730018615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4407 , TFLOPS: 66.96278875655939, Tokens per sec: 165941.1670988872, Loss: 3.4946649074554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4408 , TFLOPS: 68.86616458571815, Tokens per sec: 170657.9420776993, Loss: 3.4920096397399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4409 , TFLOPS: 70.54071509477347, Tokens per sec: 174807.6627061051, Loss: 3.4997994899749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4410 , TFLOPS: 69.2306946964398, Tokens per sec: 171561.2878483742, Loss: 3.5007402896881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4411 , TFLOPS: 66.87212650161504, Tokens per sec: 165716.4960438633, Loss: 3.499986410140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4412 , TFLOPS: 70.559726973062, Tokens per sec: 174854.77623483536, Loss: 3.5202832221984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4413 , TFLOPS: 69.7999236899081, Tokens per sec: 172971.89999993876, Loss: 3.491848945617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4414 , TFLOPS: 66.54289732455787, Tokens per sec: 164900.62987552778, Loss: 3.5155410766601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4415 , TFLOPS: 69.24967042712709, Tokens per sec: 171608.3117993666, Loss: 3.5012025833129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4416 , TFLOPS: 69.6512550871312, Tokens per sec: 172603.4828250591, Loss: 3.5108323097229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4417 , TFLOPS: 67.93619157698926, Tokens per sec: 168353.36651708512, Loss: 3.499366283416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4418 , TFLOPS: 69.86512315001275, Tokens per sec: 173133.4714443916, Loss: 3.5104191303253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4419 , TFLOPS: 69.52331870169566, Tokens per sec: 172286.4423685935, Loss: 3.5040712356567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4420 , TFLOPS: 68.14179037956117, Tokens per sec: 168862.86299843612, Loss: 3.50166392326355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4421 , TFLOPS: 68.27734503676908, Tokens per sec: 169198.7823715755, Loss: 3.494868516921997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4422 , TFLOPS: 69.52567426974937, Tokens per sec: 172292.27972571785, Loss: 3.5165786743164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4423 , TFLOPS: 68.83850354159048, Tokens per sec: 170589.3949632354, Loss: 3.494666576385498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4424 , TFLOPS: 67.08402601969064, Tokens per sec: 166241.60639231352, Loss: 3.5063397884368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4425 , TFLOPS: 70.58333722057739, Tokens per sec: 174913.28502906195, Loss: 3.5071358680725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4426 , TFLOPS: 69.57428359838016, Tokens per sec: 172412.7389968245, Loss: 3.504181146621704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4427 , TFLOPS: 67.6003908337368, Tokens per sec: 167521.2152837999, Loss: 3.5106523036956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4428 , TFLOPS: 68.83466794437805, Tokens per sec: 170579.88993081704, Loss: 3.503763437271118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4429 , TFLOPS: 70.56290698118939, Tokens per sec: 174862.65664528182, Loss: 3.501638650894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4430 , TFLOPS: 67.7914258341615, Tokens per sec: 167994.6210590949, Loss: 3.4815001487731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4431 , TFLOPS: 67.84333751496781, Tokens per sec: 168123.26392267665, Loss: 3.500141143798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4432 , TFLOPS: 70.60949537941599, Tokens per sec: 174978.1078282231, Loss: 3.4828667640686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4433 , TFLOPS: 69.67944038941681, Tokens per sec: 172673.32910899015, Loss: 3.523033857345581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4434 , TFLOPS: 66.31768501544128, Tokens per sec: 164342.52896435294, Loss: 3.4757707118988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4435 , TFLOPS: 69.99304721557075, Tokens per sec: 173450.48137084345, Loss: 3.4896786212921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4436 , TFLOPS: 69.76198219960803, Tokens per sec: 172877.87680737503, Loss: 3.4846348762512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4437 , TFLOPS: 67.64302287841306, Tokens per sec: 167626.86218681512, Loss: 3.5072035789489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4438 , TFLOPS: 68.89069685229967, Tokens per sec: 170718.73573674052, Loss: 3.513007640838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4439 , TFLOPS: 70.51930618461088, Tokens per sec: 174754.60906833605, Loss: 3.4800424575805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4440 , TFLOPS: 68.98809250624157, Tokens per sec: 170960.09289622557, Loss: 3.4748289585113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4441 , TFLOPS: 66.66481273727048, Tokens per sec: 165202.74969832101, Loss: 3.515899181365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4442 , TFLOPS: 70.58946808853804, Tokens per sec: 174928.47799523934, Loss: 3.5325827598571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4443 , TFLOPS: 69.60432243256311, Tokens per sec: 172487.17853698065, Loss: 3.5131428241729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4444 , TFLOPS: 66.47817651377896, Tokens per sec: 164740.24457683176, Loss: 3.4924097061157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4445 , TFLOPS: 70.57328418289705, Tokens per sec: 174888.37249425656, Loss: 3.493277072906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4446 , TFLOPS: 70.54181091241833, Tokens per sec: 174810.37826294333, Loss: 3.476919651031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4447 , TFLOPS: 67.27791188926659, Tokens per sec: 166722.07693541443, Loss: 3.484729528427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4448 , TFLOPS: 67.89247788141611, Tokens per sec: 168245.0391639938, Loss: 3.4948208332061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4449 , TFLOPS: 70.56516126679983, Tokens per sec: 174868.24301335457, Loss: 3.499506711959839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4450 , TFLOPS: 68.81352827086, Tokens per sec: 170527.50349111206, Loss: 3.4811434745788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4451 , TFLOPS: 67.83106278230802, Tokens per sec: 168092.84578297226, Loss: 3.499189853668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4452 , TFLOPS: 69.50734080955957, Tokens per sec: 172246.84739177013, Loss: 3.481417655944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4453 , TFLOPS: 69.78333416442766, Tokens per sec: 172930.78932831116, Loss: 3.4936227798461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4454 , TFLOPS: 68.18460208392628, Tokens per sec: 168968.9551179514, Loss: 3.4920637607574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4455 , TFLOPS: 69.06205571533671, Tokens per sec: 171143.38187608338, Loss: 3.4802091121673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4456 , TFLOPS: 68.80736237202873, Tokens per sec: 170512.22371457831, Loss: 3.49259090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4457 , TFLOPS: 69.69254172018789, Tokens per sec: 172705.7956498716, Loss: 3.516876459121704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4458 , TFLOPS: 67.68867707208491, Tokens per sec: 167739.9983079586, Loss: 3.497291088104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4459 , TFLOPS: 69.59007990848283, Tokens per sec: 172451.88399336618, Loss: 3.480833053588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4460 , TFLOPS: 69.780951889358, Tokens per sec: 172924.88578825927, Loss: 3.5006494522094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4461 , TFLOPS: 66.6998822542872, Tokens per sec: 165289.6558247147, Loss: 3.5347671508789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4462 , TFLOPS: 70.58384827375485, Tokens per sec: 174914.55147513284, Loss: 3.4962105751037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4463 , TFLOPS: 68.72378980491447, Tokens per sec: 170305.12168699093, Loss: 3.492626428604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4464 , TFLOPS: 67.31709558575963, Tokens per sec: 166819.1784517647, Loss: 3.4674999713897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4465 , TFLOPS: 69.89658338419216, Tokens per sec: 173211.433370319, Loss: 3.502471923828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4466 , TFLOPS: 70.5908829634827, Tokens per sec: 174931.98421121354, Loss: 3.5041885375976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4467 , TFLOPS: 67.19298943718681, Tokens per sec: 166511.62974418033, Loss: 3.4995036125183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4468 , TFLOPS: 68.05321548727468, Tokens per sec: 168643.36465802032, Loss: 3.5039939880371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4469 , TFLOPS: 70.57751627782135, Tokens per sec: 174898.8600916827, Loss: 3.4983348846435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4470 , TFLOPS: 68.84174961975549, Tokens per sec: 170597.4391025133, Loss: 3.4945878982543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4471 , TFLOPS: 67.92356812633957, Tokens per sec: 168322.0842157873, Loss: 3.504240036010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4472 , TFLOPS: 70.56524981669848, Tokens per sec: 174868.462449758, Loss: 3.520369529724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4473 , TFLOPS: 69.7823231147127, Tokens per sec: 172928.28383574117, Loss: 3.4957163333892822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4474 , TFLOPS: 67.85030693209056, Tokens per sec: 168140.53490605144, Loss: 3.4965949058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4475 , TFLOPS: 68.84993160030126, Tokens per sec: 170617.71495162524, Loss: 3.505157470703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4476 , TFLOPS: 70.58770539297505, Tokens per sec: 174924.10984145725, Loss: 3.475903034210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4477 , TFLOPS: 69.27776629308768, Tokens per sec: 171677.9364502321, Loss: 3.4764323234558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4478 , TFLOPS: 67.04736060194877, Tokens per sec: 166150.74544811994, Loss: 3.523451805114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4479 , TFLOPS: 70.56037731337018, Tokens per sec: 174856.3878497599, Loss: 3.4838571548461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4480 , TFLOPS: 69.82501473214087, Tokens per sec: 173034.07836659809, Loss: 3.4934206008911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4481 , TFLOPS: 66.33286685265989, Tokens per sec: 164380.1512293981, Loss: 3.5018129348754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4482 , TFLOPS: 70.60563114223574, Tokens per sec: 174968.53182280957, Loss: 3.520378351211548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4483 , TFLOPS: 69.64422688347688, Tokens per sec: 172586.0661621831, Loss: 3.492196559906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4484 , TFLOPS: 68.25070674633044, Tokens per sec: 169132.7697534186, Loss: 3.5149240493774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4485 , TFLOPS: 67.41330263928302, Tokens per sec: 167057.59012847746, Loss: 3.5037124156951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4486 , TFLOPS: 70.55750166723551, Tokens per sec: 174849.26168185967, Loss: 3.482476234436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4487 , TFLOPS: 68.99692264578626, Tokens per sec: 170981.97495473747, Loss: 3.469221830368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4488 , TFLOPS: 67.27517144667033, Tokens per sec: 166715.2858167762, Loss: 3.494269371032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4489 , TFLOPS: 69.56469571190374, Tokens per sec: 172388.97915794307, Loss: 3.5019869804382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4490 , TFLOPS: 69.70071771617582, Tokens per sec: 172726.05666859, Loss: 3.4802870750427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4491 , TFLOPS: 67.45013388048154, Tokens per sec: 167148.86200146357, Loss: 3.4749693870544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4492 , TFLOPS: 70.56902709515312, Tokens per sec: 174877.82296187017, Loss: 3.5159287452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4493 , TFLOPS: 69.53507546199185, Tokens per sec: 172315.57691572153, Loss: 3.4913740158081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4494 , TFLOPS: 67.6069427262548, Tokens per sec: 167537.45159521006, Loss: 3.4925591945648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4495 , TFLOPS: 68.75881321575855, Tokens per sec: 170391.91355720974, Loss: 3.457371711730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4496 , TFLOPS: 69.65516325119751, Tokens per sec: 172613.1676861352, Loss: 3.505574941635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4497 , TFLOPS: 68.95088485953028, Tokens per sec: 170867.88824891433, Loss: 3.4787089824676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4498 , TFLOPS: 67.92086385902839, Tokens per sec: 168315.38274349054, Loss: 3.4914400577545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4499 , TFLOPS: 70.61139644218167, Tokens per sec: 174982.81887117578, Loss: 3.494363784790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4500 , TFLOPS: 68.72724553504437, Tokens per sec: 170313.6853669329, Loss: 3.4905552864074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4501 , TFLOPS: 67.25981633389966, Tokens per sec: 166677.2341558252, Loss: 3.493170976638794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4502 , TFLOPS: 69.91052481819541, Tokens per sec: 173245.9817795548, Loss: 3.5203895568847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4503 , TFLOPS: 69.69073199118351, Tokens per sec: 172701.31094204067, Loss: 3.508450508117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4504 , TFLOPS: 68.16135447383309, Tokens per sec: 168911.3449792048, Loss: 3.478052854537964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4505 , TFLOPS: 68.1563787717276, Tokens per sec: 168899.0146412686, Loss: 3.4989917278289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4506 , TFLOPS: 70.57648457773934, Tokens per sec: 174896.30342522834, Loss: 3.5068836212158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4507 , TFLOPS: 68.93924276030707, Tokens per sec: 170839.03784455612, Loss: 3.5201821327209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4508 , TFLOPS: 66.10595141647403, Tokens per sec: 163817.8297817307, Loss: 3.4659135341644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4509 , TFLOPS: 70.60177073634267, Tokens per sec: 174958.9653117477, Loss: 3.51326584815979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4510 , TFLOPS: 69.67936715339593, Tokens per sec: 172673.14762206282, Loss: 3.495569944381714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4511 , TFLOPS: 66.17277912999864, Tokens per sec: 163983.43621752507, Loss: 3.505150318145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4512 , TFLOPS: 70.56758360890116, Tokens per sec: 174874.24584392467, Loss: 3.502943754196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4513 , TFLOPS: 70.58637113621917, Tokens per sec: 174920.8034062357, Loss: 3.5044424533843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4514 , TFLOPS: 67.34222988476638, Tokens per sec: 166881.464013475, Loss: 3.490248203277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4515 , TFLOPS: 68.60234824616896, Tokens per sec: 170004.17612652745, Loss: 3.4644784927368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4516 , TFLOPS: 70.56478918073252, Tokens per sec: 174867.32094309112, Loss: 3.500487804412842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4517 , TFLOPS: 68.93755758463125, Tokens per sec: 170834.861793594, Loss: 3.5046069622039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4518 , TFLOPS: 66.85593031597196, Tokens per sec: 165676.3601116826, Loss: 3.4942216873168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4519 , TFLOPS: 70.59023149596945, Tokens per sec: 174930.3698029506, Loss: 3.516684055328369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4520 , TFLOPS: 69.79899814341638, Tokens per sec: 172969.60639377494, Loss: 3.477552652359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4521 , TFLOPS: 68.16555067125502, Tokens per sec: 168921.74361866567, Loss: 3.501398801803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4522 , TFLOPS: 68.07142472436689, Tokens per sec: 168688.48915344785, Loss: 3.5095412731170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4523 , TFLOPS: 70.66652510059387, Tokens per sec: 175119.43375964428, Loss: 3.5221734046936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4524 , TFLOPS: 69.2223109848236, Tokens per sec: 171540.51208744716, Loss: 3.520598888397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4525 , TFLOPS: 67.44309618025684, Tokens per sec: 167131.4218050401, Loss: 3.4786510467529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4526 , TFLOPS: 69.59466457895459, Tokens per sec: 172463.24531183834, Loss: 3.4898648262023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4527 , TFLOPS: 69.86575829681573, Tokens per sec: 173135.04540813653, Loss: 3.4986934661865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4528 , TFLOPS: 66.42434741830809, Tokens per sec: 164606.85014849057, Loss: 3.5262415409088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4529 , TFLOPS: 70.510160292253, Tokens per sec: 174731.94453957005, Loss: 3.492737054824829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4530 , TFLOPS: 68.7999439507542, Tokens per sec: 170493.84004945346, Loss: 3.511965751647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4531 , TFLOPS: 67.086315392321, Tokens per sec: 166247.27970988673, Loss: 3.514922618865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4532 , TFLOPS: 70.55936772381354, Tokens per sec: 174853.8859756231, Loss: 3.4897775650024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4533 , TFLOPS: 69.61175046513029, Tokens per sec: 172505.58601994746, Loss: 3.4852166175842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4534 , TFLOPS: 67.37272837464009, Tokens per sec: 166957.04263106806, Loss: 3.4823646545410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4535 , TFLOPS: 68.66866094310953, Tokens per sec: 170168.5062944928, Loss: 3.5191292762756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4536 , TFLOPS: 70.52729706780244, Tokens per sec: 174774.41135715318, Loss: 3.4998154640197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4537 , TFLOPS: 67.99158707472274, Tokens per sec: 168490.6426627865, Loss: 3.490668773651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4538 , TFLOPS: 67.91457983824569, Tokens per sec: 168299.81024774897, Loss: 3.4962525367736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4539 , TFLOPS: 70.56269617809282, Tokens per sec: 174862.13425198666, Loss: 3.4854440689086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4540 , TFLOPS: 69.81618888322774, Tokens per sec: 173012.20694074384, Loss: 3.5094895362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4541 , TFLOPS: 67.04316020528414, Tokens per sec: 166140.33640247313, Loss: 3.5120065212249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4542 , TFLOPS: 68.56557766809456, Tokens per sec: 169913.05458344554, Loss: 3.507357358932495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4543 , TFLOPS: 70.56270267660211, Tokens per sec: 174862.15035600858, Loss: 3.5045883655548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4544 , TFLOPS: 68.0449287539453, Tokens per sec: 168622.82921997108, Loss: 3.4977540969848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4545 , TFLOPS: 67.23915660462112, Tokens per sec: 166626.03707081586, Loss: 3.4907565116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4546 , TFLOPS: 70.55292996600002, Tokens per sec: 174837.93250258366, Loss: 3.523318290710449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4547 , TFLOPS: 69.83984914838241, Tokens per sec: 173070.83968419046, Loss: 3.5096983909606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4548 , TFLOPS: 66.14529008030279, Tokens per sec: 163915.31532420992, Loss: 3.4743735790252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4549 , TFLOPS: 70.58967868457603, Tokens per sec: 174928.99987542053, Loss: 3.504014492034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4550 , TFLOPS: 69.66417284686153, Tokens per sec: 172635.49445667953, Loss: 3.5147368907928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4551 , TFLOPS: 67.90442146338918, Tokens per sec: 168274.63667581722, Loss: 3.490664482116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4552 , TFLOPS: 69.01347869788655, Tokens per sec: 171023.00267564168, Loss: 3.4919469356536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4553 , TFLOPS: 70.62559071955194, Tokens per sec: 175017.99385412806, Loss: 3.4929182529449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4554 , TFLOPS: 68.14344837270247, Tokens per sec: 168866.97168808326, Loss: 3.488477945327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4555 , TFLOPS: 67.25798955530313, Tokens per sec: 166672.7071972267, Loss: 3.484562397003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4556 , TFLOPS: 70.55984973904852, Tokens per sec: 174855.0804624737, Loss: 3.5103447437286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4557 , TFLOPS: 68.8469052288034, Tokens per sec: 170610.2152696712, Loss: 3.5029094219207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4558 , TFLOPS: 67.93894160609388, Tokens per sec: 168360.18139214706, Loss: 3.5234928131103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4559 , TFLOPS: 69.47421315036092, Tokens per sec: 172164.75340296296, Loss: 3.474235773086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4560 , TFLOPS: 70.57345576394768, Tokens per sec: 174888.79769100124, Loss: 3.5048623085021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4561 , TFLOPS: 67.7013889703451, Tokens per sec: 167771.4997921785, Loss: 3.492501974105835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4562 , TFLOPS: 68.81020684197252, Tokens per sec: 170519.27262444326, Loss: 3.492875814437866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4563 , TFLOPS: 69.64358283187215, Tokens per sec: 172584.47013135848, Loss: 3.505561590194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4564 , TFLOPS: 68.88604877379234, Tokens per sec: 170707.21728617128, Loss: 3.518141269683838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4565 , TFLOPS: 67.10411223741154, Tokens per sec: 166291.38225250546, Loss: 3.488954544067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4566 , TFLOPS: 70.587997282541, Tokens per sec: 174924.83317595598, Loss: 3.4854557514190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4567 , TFLOPS: 68.72977859774349, Tokens per sec: 170319.96257534123, Loss: 3.5038959980010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4568 , TFLOPS: 67.2005337825186, Tokens per sec: 166530.3254629908, Loss: 3.4805402755737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4569 , TFLOPS: 69.83796944871123, Tokens per sec: 173066.18158133927, Loss: 3.4986777305603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4570 , TFLOPS: 68.77304302406218, Tokens per sec: 170427.17658972892, Loss: 3.500438928604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4571 , TFLOPS: 68.88313740120888, Tokens per sec: 170700.0025841953, Loss: 3.50823974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4572 , TFLOPS: 68.94824414641414, Tokens per sec: 170861.34427091398, Loss: 3.4832844734191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4573 , TFLOPS: 70.55505051222613, Tokens per sec: 174843.1874497294, Loss: 3.495509147644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4574 , TFLOPS: 67.93145862233217, Tokens per sec: 168341.63773406684, Loss: 3.4959065914154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4575 , TFLOPS: 67.35604698256905, Tokens per sec: 166915.704303909, Loss: 3.5069055557250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4576 , TFLOPS: 70.58271929739763, Tokens per sec: 174911.7537473527, Loss: 3.5096211433410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4577 , TFLOPS: 69.70526130658335, Tokens per sec: 172737.3161861377, Loss: 3.512679100036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4578 , TFLOPS: 66.91758221785047, Tokens per sec: 165829.14031605516, Loss: 3.5028798580169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4579 , TFLOPS: 70.62844555562864, Tokens per sec: 175025.06845241226, Loss: 3.5115020275115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4580 , TFLOPS: 70.55538774352452, Tokens per sec: 174844.02314603588, Loss: 3.5195488929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4581 , TFLOPS: 67.40794402133709, Tokens per sec: 167044.31088290695, Loss: 3.517714738845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4582 , TFLOPS: 67.7911252442654, Tokens per sec: 167993.8761642196, Loss: 3.5055723190307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4583 , TFLOPS: 70.55471094283696, Tokens per sec: 174842.3459593764, Loss: 3.473266124725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4584 , TFLOPS: 68.85979575422503, Tokens per sec: 170642.15941167463, Loss: 3.509110927581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4585 , TFLOPS: 66.70632964636674, Tokens per sec: 165305.63317252664, Loss: 3.5001347064971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4586 , TFLOPS: 70.60859865607095, Tokens per sec: 174975.88564899185, Loss: 3.5060970783233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4587 , TFLOPS: 69.80977028541699, Tokens per sec: 172996.30094830182, Loss: 3.507891893386841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4588 , TFLOPS: 68.13133727391876, Tokens per sec: 168836.95905114958, Loss: 3.5022339820861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4589 , TFLOPS: 67.3788249582836, Tokens per sec: 166972.1506369308, Loss: 3.5097360610961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4590 , TFLOPS: 69.681631856534, Tokens per sec: 172678.75980591658, Loss: 3.473336696624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4591 , TFLOPS: 69.80006887496653, Tokens per sec: 172972.259784507, Loss: 3.5017600059509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4592 , TFLOPS: 66.89029895008377, Tokens per sec: 165761.52937302925, Loss: 3.5053775310516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4593 , TFLOPS: 70.5071942392654, Tokens per sec: 174724.59433352904, Loss: 3.499447822570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4594 , TFLOPS: 69.78097859829143, Tokens per sec: 172924.95197593857, Loss: 3.4877877235412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4595 , TFLOPS: 66.5153813001207, Tokens per sec: 164832.44213583213, Loss: 3.511570930480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4596 , TFLOPS: 66.49529454693652, Tokens per sec: 164782.66494870323, Loss: 3.511831283569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4597 , TFLOPS: 69.66794166543825, Tokens per sec: 172644.83400428228, Loss: 3.5057873725891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4598 , TFLOPS: 67.32950547321505, Tokens per sec: 166849.93151994116, Loss: 3.505943775177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4599 , TFLOPS: 70.59657022639088, Tokens per sec: 174946.07787520371, Loss: 3.5197033882141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4600 , TFLOPS: 69.51082076960823, Tokens per sec: 172255.47111611997, Loss: 3.497769594192505 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/optimizer.pt +[rank0]:[2024-08-25 06:17:03,134] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006355983001412824, 'preprocessing_with_comm': 0.001221593003720045, 'state_converting': 1.0054559050040552, : 1.0144360799968126}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600, took 4.47s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002600 + eval ppl=25.08592414855957, eval loss=3.222306966781616 +------------------------------------------------------------------ +iteration: 4601 , TFLOPS: 64.63434399750892, Tokens per sec: 160171.02448659536, Loss: 3.4991886615753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4602 , TFLOPS: 68.68941206397359, Tokens per sec: 170219.92985791864, Loss: 3.498229503631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4603 , TFLOPS: 70.59194398049938, Tokens per sec: 174934.61352826154, Loss: 3.5147252082824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4604 , TFLOPS: 67.57488254990125, Tokens per sec: 167458.00294648597, Loss: 3.4883005619049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4605 , TFLOPS: 66.85258367422955, Tokens per sec: 165668.06676478186, Loss: 3.5032453536987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4606 , TFLOPS: 70.5549770694122, Tokens per sec: 174843.00545034607, Loss: 3.4717111587524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4607 , TFLOPS: 69.47096146661423, Tokens per sec: 172156.69537243614, Loss: 3.489657163619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4608 , TFLOPS: 67.98395188919565, Tokens per sec: 168471.72183196526, Loss: 3.502315044403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4609 , TFLOPS: 68.6855858181868, Tokens per sec: 170210.44799936318, Loss: 3.508526563644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4610 , TFLOPS: 70.58834476736824, Tokens per sec: 174925.6942816365, Loss: 3.480644702911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4611 , TFLOPS: 67.75078470051199, Tokens per sec: 167893.90785292094, Loss: 3.503284215927124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4612 , TFLOPS: 68.10305319575332, Tokens per sec: 168766.86798970719, Loss: 3.47625470161438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4613 , TFLOPS: 70.53370026660605, Tokens per sec: 174790.27918915908, Loss: 3.4973325729370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4614 , TFLOPS: 69.4137086772834, Tokens per sec: 172014.81665355896, Loss: 3.4954051971435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4615 , TFLOPS: 68.14272364748923, Tokens per sec: 168865.1757391698, Loss: 3.4979710578918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4616 , TFLOPS: 69.92027045958937, Tokens per sec: 173270.1325525001, Loss: 3.4992570877075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4617 , TFLOPS: 70.59281615639851, Tokens per sec: 174936.77487621768, Loss: 3.5064120292663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4618 , TFLOPS: 66.4595274380245, Tokens per sec: 164694.03011274693, Loss: 3.4747753143310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4619 , TFLOPS: 68.93330780825906, Tokens per sec: 170824.3303795922, Loss: 3.4967212677001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4620 , TFLOPS: 70.57778984932305, Tokens per sec: 174899.53803200033, Loss: 3.4991002082824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4621 , TFLOPS: 68.0309170320643, Tokens per sec: 168588.10662963137, Loss: 3.484323024749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4622 , TFLOPS: 67.7932455235353, Tokens per sec: 167999.1304498185, Loss: 3.4906222820281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4623 , TFLOPS: 70.60976038442233, Tokens per sec: 174978.76453982154, Loss: 3.4914638996124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4624 , TFLOPS: 68.76087332826769, Tokens per sec: 170397.01874294755, Loss: 3.513193130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4625 , TFLOPS: 67.13906842847422, Tokens per sec: 166378.0075447011, Loss: 3.5120303630828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4626 , TFLOPS: 69.4966497506443, Tokens per sec: 172220.35377006073, Loss: 3.4993762969970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4627 , TFLOPS: 70.61012008101466, Tokens per sec: 174979.65590760115, Loss: 3.487626075744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4628 , TFLOPS: 68.10424729198371, Tokens per sec: 168769.82709170665, Loss: 3.5192060470581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4629 , TFLOPS: 67.86525270613623, Tokens per sec: 168177.57218054959, Loss: 3.5065290927886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4630 , TFLOPS: 70.5584447777855, Tokens per sec: 174851.5988136947, Loss: 3.510958194732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4631 , TFLOPS: 68.84309080600973, Tokens per sec: 170600.76270979515, Loss: 3.512136697769165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4632 , TFLOPS: 67.2090704903336, Tokens per sec: 166551.48036535233, Loss: 3.5069947242736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4633 , TFLOPS: 69.56944853480395, Tokens per sec: 172400.75717665645, Loss: 3.5142264366149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4634 , TFLOPS: 70.61019493820712, Tokens per sec: 174979.84141197026, Loss: 3.519549608230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4635 , TFLOPS: 68.30174788481199, Tokens per sec: 169259.25531722145, Loss: 3.491708755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4636 , TFLOPS: 69.2235371591619, Tokens per sec: 171543.5506825318, Loss: 3.5090842247009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4637 , TFLOPS: 69.48589037208795, Tokens per sec: 172193.69084475475, Loss: 3.5050392150878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4638 , TFLOPS: 67.2307388952989, Tokens per sec: 166605.17705982822, Loss: 3.4941585063934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4639 , TFLOPS: 68.66897193947544, Tokens per sec: 170169.2769777466, Loss: 3.471250534057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4640 , TFLOPS: 70.51742872123799, Tokens per sec: 174749.95650727837, Loss: 3.512019157409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4641 , TFLOPS: 67.90128533971198, Tokens per sec: 168266.86501587255, Loss: 3.49067759513855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4642 , TFLOPS: 68.2726478387759, Tokens per sec: 169187.14219750935, Loss: 3.480705738067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4643 , TFLOPS: 70.56772195953198, Tokens per sec: 174874.58869202845, Loss: 3.473625659942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4644 , TFLOPS: 68.77413605922274, Tokens per sec: 170429.88525126455, Loss: 3.5090856552124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4645 , TFLOPS: 66.43432689522612, Tokens per sec: 164631.5803916236, Loss: 3.493602991104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4646 , TFLOPS: 70.61447035901959, Tokens per sec: 174990.43637572005, Loss: 3.4967899322509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4647 , TFLOPS: 70.64311284750342, Tokens per sec: 175061.41561741356, Loss: 3.494985342025757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4648 , TFLOPS: 67.80153308687068, Tokens per sec: 168019.66794471402, Loss: 3.5151636600494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4649 , TFLOPS: 68.89804217649481, Tokens per sec: 170736.93825925037, Loss: 3.524527072906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4650 , TFLOPS: 70.6090471891006, Tokens per sec: 174976.99716324933, Loss: 3.5156140327453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4651 , TFLOPS: 67.94995175423765, Tokens per sec: 168387.46575211524, Loss: 3.472940444946289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4652 , TFLOPS: 67.0414466832276, Tokens per sec: 166136.09010605697, Loss: 3.4729108810424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4653 , TFLOPS: 70.6122304672139, Tokens per sec: 174984.88567708113, Loss: 3.485358715057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4654 , TFLOPS: 70.51909897637098, Tokens per sec: 174754.0955834917, Loss: 3.4976251125335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4655 , TFLOPS: 67.11741446225932, Tokens per sec: 166324.34663103928, Loss: 3.5033013820648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4656 , TFLOPS: 68.85380215593179, Tokens per sec: 170627.30661485463, Loss: 3.4884607791900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4657 , TFLOPS: 70.55943757417405, Tokens per sec: 174854.0590725111, Loss: 3.4970450401306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4658 , TFLOPS: 67.25567279878885, Tokens per sec: 166666.9660193725, Loss: 3.4854607582092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4659 , TFLOPS: 68.58763522640736, Tokens per sec: 169967.7156427853, Loss: 3.481386184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4660 , TFLOPS: 70.54555087379713, Tokens per sec: 174819.64629923075, Loss: 3.4698588848114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4661 , TFLOPS: 68.90939532801042, Tokens per sec: 170765.07261935892, Loss: 3.5180273056030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4662 , TFLOPS: 68.01103072846657, Tokens per sec: 168538.82617865902, Loss: 3.505059242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4663 , TFLOPS: 69.60804961681792, Tokens per sec: 172496.4149101473, Loss: 3.4913747310638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4664 , TFLOPS: 70.5712716405996, Tokens per sec: 174883.38519274365, Loss: 3.4932687282562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4665 , TFLOPS: 66.6161555247022, Tokens per sec: 165082.17176554239, Loss: 3.491441488265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4666 , TFLOPS: 68.8761504763851, Tokens per sec: 170682.68821480867, Loss: 3.4878997802734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4667 , TFLOPS: 70.52384579758794, Tokens per sec: 174765.85872937282, Loss: 3.4908699989318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4668 , TFLOPS: 67.98469441114818, Tokens per sec: 168473.5618831592, Loss: 3.505378246307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4669 , TFLOPS: 68.88374238798568, Tokens per sec: 170701.50180807296, Loss: 3.5181844234466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4670 , TFLOPS: 69.37654347005619, Tokens per sec: 171922.71717596438, Loss: 3.4862823486328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4671 , TFLOPS: 69.01728464783582, Tokens per sec: 171032.43423887595, Loss: 3.510535955429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4672 , TFLOPS: 66.96366407446256, Tokens per sec: 165943.3362330755, Loss: 3.523786783218384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4673 , TFLOPS: 70.63005974285588, Tokens per sec: 175029.06858617876, Loss: 3.478695869445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4674 , TFLOPS: 69.659519393784, Tokens per sec: 172623.96268733323, Loss: 3.4932336807250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4675 , TFLOPS: 68.08462098986644, Tokens per sec: 168721.1909530444, Loss: 3.506432056427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4676 , TFLOPS: 68.8123092089695, Tokens per sec: 170524.48252145658, Loss: 3.494575023651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4677 , TFLOPS: 70.59086123111439, Tokens per sec: 174931.93035601082, Loss: 3.5019099712371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4678 , TFLOPS: 67.07017553878904, Tokens per sec: 166207.28337488294, Loss: 3.473184108734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4679 , TFLOPS: 68.12849859318385, Tokens per sec: 168829.92448758194, Loss: 3.5054259300231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4680 , TFLOPS: 70.57094640141511, Tokens per sec: 174882.57921421644, Loss: 3.4789886474609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4681 , TFLOPS: 68.85443878392124, Tokens per sec: 170628.88424914284, Loss: 3.4844162464141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4682 , TFLOPS: 68.28088054938338, Tokens per sec: 169207.5437613026, Loss: 3.4982879161834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4683 , TFLOPS: 69.93839217087013, Tokens per sec: 173315.0401493251, Loss: 3.459646224975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4684 , TFLOPS: 70.53353032587077, Tokens per sec: 174789.85805729657, Loss: 3.4808528423309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4685 , TFLOPS: 66.29288269142691, Tokens per sec: 164281.06607324383, Loss: 3.46118426322937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4686 , TFLOPS: 68.95598215940804, Tokens per sec: 170880.51991952537, Loss: 3.4888317584991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4687 , TFLOPS: 70.59368932335285, Tokens per sec: 174938.93868011783, Loss: 3.4912452697753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4688 , TFLOPS: 67.11156579803767, Tokens per sec: 166309.85299681447, Loss: 3.4806787967681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4689 , TFLOPS: 68.71619963607594, Tokens per sec: 170286.3124124829, Loss: 3.490112543106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4690 , TFLOPS: 70.60800893574557, Tokens per sec: 174974.42425706252, Loss: 3.500938892364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4691 , TFLOPS: 68.9660166660795, Tokens per sec: 170905.38653244937, Loss: 3.5140440464019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4692 , TFLOPS: 66.85190346794147, Tokens per sec: 165666.38113866906, Loss: 3.5087859630584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4693 , TFLOPS: 70.59400679002732, Tokens per sec: 174939.725397509, Loss: 3.4861226081848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4694 , TFLOPS: 70.54534405646129, Tokens per sec: 174819.13378308975, Loss: 3.501798629760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4695 , TFLOPS: 68.10192637449326, Tokens per sec: 168764.07560249514, Loss: 3.4997119903564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4696 , TFLOPS: 68.60071759795744, Tokens per sec: 170000.1352006286, Loss: 3.5002810955047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4697 , TFLOPS: 70.56288447604972, Tokens per sec: 174862.60087506656, Loss: 3.492906332015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4698 , TFLOPS: 68.95236194188344, Tokens per sec: 170871.5486216995, Loss: 3.475302219390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4699 , TFLOPS: 67.37064073841645, Tokens per sec: 166951.86923853378, Loss: 3.493424415588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4700 , TFLOPS: 69.52154462634175, Tokens per sec: 172282.04601443597, Loss: 3.494576930999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4701 , TFLOPS: 70.57821758361733, Tokens per sec: 174900.59800470035, Loss: 3.5079739093780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4702 , TFLOPS: 68.30473333777431, Tokens per sec: 169266.65359853688, Loss: 3.4909956455230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4703 , TFLOPS: 68.2382396988953, Tokens per sec: 169101.87503651567, Loss: 3.5021045207977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4704 , TFLOPS: 70.61343011914813, Tokens per sec: 174987.85854672673, Loss: 3.5046730041503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4705 , TFLOPS: 67.21127957491977, Tokens per sec: 166556.9547203076, Loss: 3.5166196823120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4706 , TFLOPS: 68.75337508066613, Tokens per sec: 170378.43725939072, Loss: 3.481370449066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4707 , TFLOPS: 69.44518865115683, Tokens per sec: 172092.82749662272, Loss: 3.5048773288726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4708 , TFLOPS: 67.89918939323601, Tokens per sec: 168261.67102961752, Loss: 3.5041794776916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4709 , TFLOPS: 68.81697413133651, Tokens per sec: 170536.0427129657, Loss: 3.5152158737182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4710 , TFLOPS: 70.57744626940057, Tokens per sec: 174898.68660310394, Loss: 3.4986279010772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4711 , TFLOPS: 68.14995864545465, Tokens per sec: 168883.1048611289, Loss: 3.4931468963623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4712 , TFLOPS: 67.05597888313412, Tokens per sec: 166172.10249827927, Loss: 3.498562812805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4713 , TFLOPS: 69.84284180850331, Tokens per sec: 173078.25582563877, Loss: 3.5114803314208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4714 , TFLOPS: 70.54816148045037, Tokens per sec: 174826.11567009956, Loss: 3.5045857429504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4715 , TFLOPS: 66.99314101550826, Tokens per sec: 166016.38334013984, Loss: 3.5112626552581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4716 , TFLOPS: 68.73538107004815, Tokens per sec: 170333.84611887543, Loss: 3.4982848167419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4717 , TFLOPS: 70.54417281948386, Tokens per sec: 174816.23132883728, Loss: 3.5085315704345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4718 , TFLOPS: 68.79847614485928, Tokens per sec: 170490.20266475368, Loss: 3.484081745147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4719 , TFLOPS: 67.25372602543561, Tokens per sec: 166662.1417005484, Loss: 3.4934093952178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4720 , TFLOPS: 70.54632694307828, Tokens per sec: 174821.56948440053, Loss: 3.4894697666168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4721 , TFLOPS: 70.55483719849617, Tokens per sec: 174842.6588348081, Loss: 3.4641523361206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4722 , TFLOPS: 66.87503617891319, Tokens per sec: 165723.7065447355, Loss: 3.487075090408325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4723 , TFLOPS: 68.76608739969474, Tokens per sec: 170409.93978632116, Loss: 3.509141206741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4724 , TFLOPS: 70.53534046966743, Tokens per sec: 174794.3437930285, Loss: 3.4743947982788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4725 , TFLOPS: 67.2469055104499, Tokens per sec: 166645.23971307208, Loss: 3.4947361946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4726 , TFLOPS: 68.45555237095469, Tokens per sec: 169640.3997768459, Loss: 3.4997589588165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4727 , TFLOPS: 70.54261418519863, Tokens per sec: 174812.3688613807, Loss: 3.5041730403900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4728 , TFLOPS: 68.7515170097234, Tokens per sec: 170373.83275491121, Loss: 3.469515323638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4729 , TFLOPS: 67.89162879087486, Tokens per sec: 168242.9350240975, Loss: 3.4878854751586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4730 , TFLOPS: 70.60640396711203, Tokens per sec: 174970.4469679867, Loss: 3.500910758972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4731 , TFLOPS: 69.7670441897967, Tokens per sec: 172890.4209766868, Loss: 3.5004091262817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4732 , TFLOPS: 68.42121842305966, Tokens per sec: 169555.3164717654, Loss: 3.5003581047058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4733 , TFLOPS: 68.06853027256511, Tokens per sec: 168681.31638303286, Loss: 3.4678847789764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4734 , TFLOPS: 70.62506619239899, Tokens per sec: 175016.69401806162, Loss: 3.4971120357513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4735 , TFLOPS: 67.87791769470626, Tokens per sec: 168208.95741738976, Loss: 3.485944986343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4736 , TFLOPS: 68.81333963232673, Tokens per sec: 170527.03602404633, Loss: 3.515090227127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4737 , TFLOPS: 69.5828405913434, Tokens per sec: 172433.94416801803, Loss: 3.5055084228515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4738 , TFLOPS: 68.85931367835357, Tokens per sec: 170640.9647745601, Loss: 3.4935083389282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4739 , TFLOPS: 67.20196113367632, Tokens per sec: 166533.8625963953, Loss: 3.5005455017089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4740 , TFLOPS: 69.4155397696048, Tokens per sec: 172019.35430204286, Loss: 3.520160436630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4741 , TFLOPS: 70.57327699570496, Tokens per sec: 174888.3546836028, Loss: 3.5059919357299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4742 , TFLOPS: 67.98464952199508, Tokens per sec: 168473.45064289283, Loss: 3.500929117202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4743 , TFLOPS: 68.7817680028851, Tokens per sec: 170448.7980483879, Loss: 3.5213208198547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4744 , TFLOPS: 69.67459939204389, Tokens per sec: 172661.33258422016, Loss: 3.4953765869140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4745 , TFLOPS: 68.12112248545597, Tokens per sec: 168811.64568008692, Loss: 3.501443386077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4746 , TFLOPS: 68.13768747370679, Tokens per sec: 168852.69554575655, Loss: 3.483940601348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4747 , TFLOPS: 70.55322256461051, Tokens per sec: 174838.65759417284, Loss: 3.5226752758026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4748 , TFLOPS: 68.75696649472916, Tokens per sec: 170387.3371790658, Loss: 3.5077872276306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4749 , TFLOPS: 68.06334718056779, Tokens per sec: 168668.47211009878, Loss: 3.5077245235443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4750 , TFLOPS: 69.9865326391516, Tokens per sec: 173434.33753283697, Loss: 3.4915852546691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4751 , TFLOPS: 70.55865604454095, Tokens per sec: 174852.12235598767, Loss: 3.4869751930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4752 , TFLOPS: 67.37245033058063, Tokens per sec: 166956.35360725963, Loss: 3.468808174133301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4753 , TFLOPS: 68.7394154047378, Tokens per sec: 170343.84364465502, Loss: 3.507070302963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4754 , TFLOPS: 70.590825117156, Tokens per sec: 174931.8408616437, Loss: 3.49617862701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4755 , TFLOPS: 67.02247225127199, Tokens per sec: 166089.06937346476, Loss: 3.493870496749878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4756 , TFLOPS: 68.75383124577836, Tokens per sec: 170379.56768678955, Loss: 3.4763526916503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4757 , TFLOPS: 70.58752557375072, Tokens per sec: 174923.66422961067, Loss: 3.5104618072509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4758 , TFLOPS: 68.99192649409186, Tokens per sec: 170969.5939404672, Loss: 3.506381034851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4759 , TFLOPS: 66.9829835279304, Tokens per sec: 165991.2119669823, Loss: 3.5009987354278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4760 , TFLOPS: 70.55031378872053, Tokens per sec: 174831.44932708936, Loss: 3.4915242195129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4761 , TFLOPS: 70.55169881449407, Tokens per sec: 174834.88157352936, Loss: 3.5139846801757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4762 , TFLOPS: 67.15975713787014, Tokens per sec: 166429.27644563367, Loss: 3.4956185817718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4763 , TFLOPS: 68.69370697390058, Tokens per sec: 170230.5731178411, Loss: 3.4888691902160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4764 , TFLOPS: 70.59852764259358, Tokens per sec: 174950.92856818123, Loss: 3.4842777252197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4765 , TFLOPS: 68.84469163833892, Tokens per sec: 170604.7297486531, Loss: 3.5179989337921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4766 , TFLOPS: 66.49597778827435, Tokens per sec: 164784.35809600362, Loss: 3.4950592517852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4767 , TFLOPS: 70.51714409311761, Tokens per sec: 174749.2511674417, Loss: 3.517131805419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4768 , TFLOPS: 70.59401265428735, Tokens per sec: 174939.73992979137, Loss: 3.490928888320923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4769 , TFLOPS: 68.23975768464037, Tokens per sec: 169105.6367724119, Loss: 3.498286485671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4770 , TFLOPS: 67.9834073519354, Tokens per sec: 168470.37240864863, Loss: 3.5212602615356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4771 , TFLOPS: 70.55613878288118, Tokens per sec: 174845.8843042955, Loss: 3.503046751022339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4772 , TFLOPS: 67.33966136704757, Tokens per sec: 166875.0989436222, Loss: 3.49686861038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4773 , TFLOPS: 68.88803267380597, Tokens per sec: 170712.13360894966, Loss: 3.483717203140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4774 , TFLOPS: 69.67609422025687, Tokens per sec: 172665.03693320087, Loss: 3.492339849472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4775 , TFLOPS: 67.86858832705599, Tokens per sec: 168185.83821663773, Loss: 3.4964394569396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4776 , TFLOPS: 68.65202107483519, Tokens per sec: 170127.27086787618, Loss: 3.5077786445617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4777 , TFLOPS: 69.57245747880692, Tokens per sec: 172408.2136713584, Loss: 3.4918110370635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4778 , TFLOPS: 68.9191280052212, Tokens per sec: 170789.19126563013, Loss: 3.4678354263305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4779 , TFLOPS: 67.39397314235904, Tokens per sec: 167009.68950578, Loss: 3.4839353561401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4780 , TFLOPS: 70.60642445679281, Tokens per sec: 174970.49774367278, Loss: 3.503291130065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4781 , TFLOPS: 69.63166839497136, Tokens per sec: 172554.9448442347, Loss: 3.5132558345794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4782 , TFLOPS: 67.89434729081657, Tokens per sec: 168249.67176642054, Loss: 3.498663902282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4783 , TFLOPS: 68.3250339573611, Tokens per sec: 169316.96077017082, Loss: 3.479688882827759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4784 , TFLOPS: 70.49775068246635, Tokens per sec: 174701.1921594873, Loss: 3.4926633834838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4785 , TFLOPS: 67.9019585386802, Tokens per sec: 168268.5332770748, Loss: 3.504605293273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4786 , TFLOPS: 67.3267595160157, Tokens per sec: 166843.126735509, Loss: 3.4857873916625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4787 , TFLOPS: 70.5790894955994, Tokens per sec: 174902.758698641, Loss: 3.494022846221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4788 , TFLOPS: 70.53487301649187, Tokens per sec: 174793.18539256524, Loss: 3.483381748199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4789 , TFLOPS: 66.8274577674286, Tokens per sec: 165605.80199988198, Loss: 3.5042338371276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4790 , TFLOPS: 68.75758870690592, Tokens per sec: 170388.87908938213, Loss: 3.503286600112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4791 , TFLOPS: 70.58790494860713, Tokens per sec: 174924.60436229644, Loss: 3.50930118560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4792 , TFLOPS: 65.73129658057739, Tokens per sec: 162889.3938267416, Loss: 3.4715280532836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4793 , TFLOPS: 68.83563051770025, Tokens per sec: 170582.27529354565, Loss: 3.5027823448181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4794 , TFLOPS: 70.54883750685282, Tokens per sec: 174827.79093799528, Loss: 3.4958930015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4795 , TFLOPS: 68.73414729437327, Tokens per sec: 170330.7886868415, Loss: 3.5021793842315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4796 , TFLOPS: 67.13791682608732, Tokens per sec: 166375.15374712687, Loss: 3.497032642364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4797 , TFLOPS: 70.57353604684424, Tokens per sec: 174888.9966408628, Loss: 3.4906790256500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4798 , TFLOPS: 69.76031739352445, Tokens per sec: 172873.75123450637, Loss: 3.489971160888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4799 , TFLOPS: 67.4137354013639, Tokens per sec: 167058.66256058656, Loss: 3.499986171722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4800 , TFLOPS: 69.9851084064891, Tokens per sec: 173430.8081274071, Loss: 3.487715482711792 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/optimizer.pt +[rank0]:[2024-08-25 06:42:46,241] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006381344006513245, 'preprocessing_with_comm': 0.0012224919992149808, 'state_converting': 0.9809772910011816, : 0.9899566410022089}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800, took 4.39s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0002800 + eval ppl=24.605045318603516, eval loss=3.202951431274414 +------------------------------------------------------------------ +iteration: 4801 , TFLOPS: 66.0401513696292, Tokens per sec: 163654.77001717567, Loss: 3.486952304840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4802 , TFLOPS: 67.9841539824899, Tokens per sec: 168472.22264141083, Loss: 3.4849870204925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4803 , TFLOPS: 68.07073677465725, Tokens per sec: 168686.78433828344, Loss: 3.5143442153930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4804 , TFLOPS: 70.55736127852147, Tokens per sec: 174848.91378316117, Loss: 3.4952359199523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4805 , TFLOPS: 69.65056924142364, Tokens per sec: 172601.7832238436, Loss: 3.523225784301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4806 , TFLOPS: 66.28293567754763, Tokens per sec: 164256.41627709693, Loss: 3.488966464996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4807 , TFLOPS: 70.57803824544423, Tokens per sec: 174900.1535849517, Loss: 3.5171477794647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4808 , TFLOPS: 70.48401599778917, Tokens per sec: 174667.15609785687, Loss: 3.497915506362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4809 , TFLOPS: 66.48328776252275, Tokens per sec: 164752.91081426942, Loss: 3.509950637817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4810 , TFLOPS: 68.83136692676447, Tokens per sec: 170571.70964553347, Loss: 3.4978981018066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4811 , TFLOPS: 70.58889121442961, Tokens per sec: 174927.0484376507, Loss: 3.4949021339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4812 , TFLOPS: 68.29426647115136, Tokens per sec: 169240.71555002464, Loss: 3.5157527923583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4813 , TFLOPS: 68.14404191843336, Tokens per sec: 168868.44255978888, Loss: 3.4946258068084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4814 , TFLOPS: 69.55891393839735, Tokens per sec: 172374.65128627932, Loss: 3.49109148979187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4815 , TFLOPS: 69.73554251720073, Tokens per sec: 172812.35636179827, Loss: 3.494049549102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4816 , TFLOPS: 67.2131379741194, Tokens per sec: 166561.5600382439, Loss: 3.504855155944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4817 , TFLOPS: 68.78384677410786, Tokens per sec: 170453.9494724734, Loss: 3.4545135498046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4818 , TFLOPS: 70.53523227938967, Tokens per sec: 174794.07568560206, Loss: 3.500105142593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4819 , TFLOPS: 68.03280848048495, Tokens per sec: 168592.79384717796, Loss: 3.482208013534546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4820 , TFLOPS: 69.18548313502743, Tokens per sec: 171449.2486187297, Loss: 3.5007359981536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4821 , TFLOPS: 69.57274381165777, Tokens per sec: 172408.92323570495, Loss: 3.4989442825317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4822 , TFLOPS: 68.04964218888054, Tokens per sec: 168634.5096309683, Loss: 3.495694160461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4823 , TFLOPS: 68.62345139646011, Tokens per sec: 170056.47205765286, Loss: 3.482591390609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4824 , TFLOPS: 70.52974721564183, Tokens per sec: 174780.48309340526, Loss: 3.503382682800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4825 , TFLOPS: 68.59262790400852, Tokens per sec: 169980.08804786982, Loss: 3.492949962615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4826 , TFLOPS: 67.33886882036997, Tokens per sec: 166873.1349256489, Loss: 3.507026433944702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4827 , TFLOPS: 70.53886180056442, Tokens per sec: 174803.07003889786, Loss: 3.502718687057495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4828 , TFLOPS: 70.60157769271356, Tokens per sec: 174958.48692836927, Loss: 3.5029585361480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4829 , TFLOPS: 65.44060537500589, Tokens per sec: 162169.0289970556, Loss: 3.5233795642852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4830 , TFLOPS: 69.97192015246722, Tokens per sec: 173398.12618113478, Loss: 3.490361213684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4831 , TFLOPS: 70.5805165253548, Tokens per sec: 174906.29503557517, Loss: 3.4926624298095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4832 , TFLOPS: 68.12596870083125, Tokens per sec: 168823.65513564157, Loss: 3.4710581302642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4833 , TFLOPS: 68.06586148554443, Tokens per sec: 168674.70283480446, Loss: 3.4964466094970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4834 , TFLOPS: 70.5919853100257, Tokens per sec: 174934.7159473806, Loss: 3.481497287750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4835 , TFLOPS: 69.75671759427482, Tokens per sec: 172864.83053541518, Loss: 3.5054783821105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4836 , TFLOPS: 66.70926689822501, Tokens per sec: 165312.91200619645, Loss: 3.472219228744507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4837 , TFLOPS: 70.5813401105298, Tokens per sec: 174908.3359703642, Loss: 3.5025932788848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4838 , TFLOPS: 70.63428597533886, Tokens per sec: 175039.54165582953, Loss: 3.5030887126922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4839 , TFLOPS: 68.0545515369717, Tokens per sec: 168646.67553634752, Loss: 3.4711079597473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4840 , TFLOPS: 67.8448752444047, Tokens per sec: 168127.07458561612, Loss: 3.513079881668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4841 , TFLOPS: 70.60586464049649, Tokens per sec: 174969.1104572217, Loss: 3.4635732173919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4842 , TFLOPS: 68.18627074080044, Tokens per sec: 168973.0902335033, Loss: 3.5126571655273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4843 , TFLOPS: 68.61962950710712, Tokens per sec: 170047.00099482026, Loss: 3.5055055618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4844 , TFLOPS: 69.53064099787142, Tokens per sec: 172304.5878251345, Loss: 3.4787323474884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4845 , TFLOPS: 70.63230897350887, Tokens per sec: 175034.64242751012, Loss: 3.484279155731201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4846 , TFLOPS: 66.11269925044286, Tokens per sec: 163834.55165764075, Loss: 3.5171823501586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4847 , TFLOPS: 69.59116798640527, Tokens per sec: 172454.58037031963, Loss: 3.5154290199279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4848 , TFLOPS: 70.54681585854406, Tokens per sec: 174822.78107078327, Loss: 3.497647762298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4849 , TFLOPS: 67.43310974266004, Tokens per sec: 167106.67431257828, Loss: 3.500210762023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4850 , TFLOPS: 69.083003858635, Tokens per sec: 171195.29368280503, Loss: 3.504857301712036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4851 , TFLOPS: 69.53293753937895, Tokens per sec: 172310.27890797515, Loss: 3.5126309394836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4852 , TFLOPS: 68.78635830803918, Tokens per sec: 170460.17333022354, Loss: 3.49552845954895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4853 , TFLOPS: 67.54431261973507, Tokens per sec: 167382.24729197836, Loss: 3.5142619609832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4854 , TFLOPS: 69.67428965484324, Tokens per sec: 172660.56502131658, Loss: 3.5060768127441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4855 , TFLOPS: 69.80150966057616, Tokens per sec: 172975.83020996331, Loss: 3.498847007751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4856 , TFLOPS: 67.01760641406692, Tokens per sec: 166077.01129285435, Loss: 3.4869651794433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4857 , TFLOPS: 70.58837328382374, Tokens per sec: 174925.7649485546, Loss: 3.4882919788360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4858 , TFLOPS: 69.50733491464054, Tokens per sec: 172246.83278351143, Loss: 3.488832473754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4859 , TFLOPS: 67.9445802235388, Tokens per sec: 168374.15450732055, Loss: 3.5170178413391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4860 , TFLOPS: 68.81755198891652, Tokens per sec: 170537.4747077052, Loss: 3.4930472373962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4861 , TFLOPS: 70.5342330692994, Tokens per sec: 174791.5995329263, Loss: 3.5199127197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4862 , TFLOPS: 67.98647271126673, Tokens per sec: 168477.96870674886, Loss: 3.5211243629455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4863 , TFLOPS: 68.14328821984584, Tokens per sec: 168866.57481166226, Loss: 3.4713664054870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4864 , TFLOPS: 70.62681243863935, Tokens per sec: 175021.0214086101, Loss: 3.4783506393432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4865 , TFLOPS: 70.52943152929593, Tokens per sec: 174779.7007878644, Loss: 3.5210421085357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4866 , TFLOPS: 67.07954038177749, Tokens per sec: 166230.4904874309, Loss: 3.506347894668579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4867 , TFLOPS: 69.93991429572647, Tokens per sec: 173318.81214239498, Loss: 3.484976053237915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4868 , TFLOPS: 70.5506406804097, Tokens per sec: 174832.2594007052, Loss: 3.493058204650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4869 , TFLOPS: 67.25608944206334, Tokens per sec: 166667.99850730377, Loss: 3.503340482711792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4870 , TFLOPS: 68.35109577076767, Tokens per sec: 169381.54481473807, Loss: 3.5139970779418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4871 , TFLOPS: 70.6106268887859, Tokens per sec: 174980.91183308777, Loss: 3.48685359954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4872 , TFLOPS: 68.84695410178989, Tokens per sec: 170610.33638231555, Loss: 3.478952646255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4873 , TFLOPS: 67.07400297849867, Tokens per sec: 166216.76819211067, Loss: 3.5002639293670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4874 , TFLOPS: 70.53127717752814, Tokens per sec: 174784.27450749875, Loss: 3.5008835792541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4875 , TFLOPS: 69.79127853174089, Tokens per sec: 172950.47634565763, Loss: 3.484623908996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4876 , TFLOPS: 67.10108939229225, Tokens per sec: 166283.89130930594, Loss: 3.4797427654266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4877 , TFLOPS: 69.54551657323158, Tokens per sec: 172341.4511395569, Loss: 3.498185157775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4878 , TFLOPS: 70.57104181404915, Tokens per sec: 174882.81565723414, Loss: 3.499191999435425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4879 , TFLOPS: 68.76693634394441, Tokens per sec: 170412.04356369076, Loss: 3.4933571815490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4880 , TFLOPS: 67.26963948622839, Tokens per sec: 166701.57700940533, Loss: 3.510310173034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4881 , TFLOPS: 70.55511247465681, Tokens per sec: 174843.34099945833, Loss: 3.47648549079895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4882 , TFLOPS: 68.91818936318585, Tokens per sec: 170786.86520726656, Loss: 3.480483293533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4883 , TFLOPS: 67.95901088990892, Tokens per sec: 168409.91528825488, Loss: 3.485043525695801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4884 , TFLOPS: 69.55673514531831, Tokens per sec: 172369.25199701558, Loss: 3.4740755558013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4885 , TFLOPS: 70.5063194830014, Tokens per sec: 174722.4265911445, Loss: 3.481271505355835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4886 , TFLOPS: 67.88272275203202, Tokens per sec: 168220.86487876376, Loss: 3.4950790405273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4887 , TFLOPS: 68.97289367518974, Tokens per sec: 170922.42851858964, Loss: 3.49592924118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4888 , TFLOPS: 69.72589452604183, Tokens per sec: 172788.44757689256, Loss: 3.514749765396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4889 , TFLOPS: 68.16135209571738, Tokens per sec: 168911.3390859721, Loss: 3.489546775817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4890 , TFLOPS: 68.0686690356778, Tokens per sec: 168681.66025331215, Loss: 3.497742176055908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4891 , TFLOPS: 69.58010618767256, Tokens per sec: 172427.16801450224, Loss: 3.5127921104431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4892 , TFLOPS: 69.67394527020119, Tokens per sec: 172659.7115982365, Loss: 3.5196006298065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4893 , TFLOPS: 66.63373722222968, Tokens per sec: 165125.74114879934, Loss: 3.505693197250366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4894 , TFLOPS: 70.60042130809147, Tokens per sec: 174955.62127988946, Loss: 3.5053350925445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4895 , TFLOPS: 69.66037048675273, Tokens per sec: 172626.07178946538, Loss: 3.50600004196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4896 , TFLOPS: 67.31645111187126, Tokens per sec: 166817.58137447466, Loss: 3.475668430328369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4897 , TFLOPS: 69.8542118915893, Tokens per sec: 173106.43214404388, Loss: 3.5059547424316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4898 , TFLOPS: 70.59080682012879, Tokens per sec: 174931.79551959466, Loss: 3.47011661529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4899 , TFLOPS: 67.74045395137192, Tokens per sec: 167868.30711852587, Loss: 3.502659320831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4900 , TFLOPS: 67.86180691757532, Tokens per sec: 168169.03313691067, Loss: 3.503838539123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4901 , TFLOPS: 70.51125371450394, Tokens per sec: 174734.65416035824, Loss: 3.501288414001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4902 , TFLOPS: 69.67542667364626, Tokens per sec: 172663.38267916345, Loss: 3.5103259086608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4903 , TFLOPS: 67.42531372821423, Tokens per sec: 167087.3549299797, Loss: 3.4752814769744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4904 , TFLOPS: 70.56978022141237, Tokens per sec: 174879.6892917043, Loss: 3.5014688968658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4905 , TFLOPS: 70.55554142915467, Tokens per sec: 174844.40399595725, Loss: 3.504638195037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4906 , TFLOPS: 67.66540265035651, Tokens per sec: 167682.3217270264, Loss: 3.491694688796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4907 , TFLOPS: 68.74718777522551, Tokens per sec: 170363.10443492088, Loss: 3.4736788272857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4908 , TFLOPS: 70.52187910631527, Tokens per sec: 174760.98505175023, Loss: 3.5005245208740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4909 , TFLOPS: 68.13252997736616, Tokens per sec: 168839.91470167934, Loss: 3.4904544353485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4910 , TFLOPS: 67.18431071108449, Tokens per sec: 166490.1229048582, Loss: 3.5029122829437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4911 , TFLOPS: 70.59428570414775, Tokens per sec: 174940.41657742436, Loss: 3.4935359954833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4912 , TFLOPS: 70.61103869138813, Tokens per sec: 174981.9323252999, Loss: 3.5147573947906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4913 , TFLOPS: 65.39577375229906, Tokens per sec: 162057.93129737716, Loss: 3.5040712356567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4914 , TFLOPS: 69.61076548692074, Tokens per sec: 172503.14513543987, Loss: 3.4910237789154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4915 , TFLOPS: 70.54630046285311, Tokens per sec: 174821.5038634855, Loss: 3.4876370429992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4916 , TFLOPS: 67.27184177561848, Tokens per sec: 166707.0345251162, Loss: 3.510769844055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4917 , TFLOPS: 69.54466484046901, Tokens per sec: 172339.34045194517, Loss: 3.490992784500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4918 , TFLOPS: 69.55527333226779, Tokens per sec: 172365.6294632444, Loss: 3.496201276779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4919 , TFLOPS: 69.56578781322006, Tokens per sec: 172391.68550530978, Loss: 3.4905831813812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4920 , TFLOPS: 67.2623326617716, Tokens per sec: 166683.46989348825, Loss: 3.4878361225128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4921 , TFLOPS: 69.33897140646619, Tokens per sec: 171829.60946348385, Loss: 3.503660202026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4922 , TFLOPS: 69.68035404641721, Tokens per sec: 172675.59325168477, Loss: 3.505399465560913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4923 , TFLOPS: 67.20300386944491, Tokens per sec: 166536.4466104967, Loss: 3.493990182876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4924 , TFLOPS: 70.55844890589752, Tokens per sec: 174851.60904361104, Loss: 3.515615463256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4925 , TFLOPS: 69.55850811107105, Tokens per sec: 172373.64560145917, Loss: 3.5109126567840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4926 , TFLOPS: 67.92195900811825, Tokens per sec: 168318.09664357587, Loss: 3.495326042175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4927 , TFLOPS: 68.67852711074102, Tokens per sec: 170192.9557447317, Loss: 3.490760564804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4928 , TFLOPS: 69.6573525316023, Tokens per sec: 172618.5929641474, Loss: 3.50291109085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4929 , TFLOPS: 68.90247074200188, Tokens per sec: 170747.91273242366, Loss: 3.511991024017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4930 , TFLOPS: 67.54997512245333, Tokens per sec: 167396.2795974904, Loss: 3.486067295074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4931 , TFLOPS: 70.56540859586684, Tokens per sec: 174868.85592202816, Loss: 3.517259359359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4932 , TFLOPS: 69.53988371782742, Tokens per sec: 172327.4923033552, Loss: 3.5155160427093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4933 , TFLOPS: 66.49518900820168, Tokens per sec: 164782.40341209224, Loss: 3.475243091583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4934 , TFLOPS: 69.8342546317431, Tokens per sec: 173056.97585567136, Loss: 3.4752578735351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4935 , TFLOPS: 70.58842662754351, Tokens per sec: 174925.89714016847, Loss: 3.4912161827087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4936 , TFLOPS: 67.90753919359163, Tokens per sec: 168282.3627547052, Loss: 3.4960644245147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4937 , TFLOPS: 68.99935379710611, Tokens per sec: 170987.9996155215, Loss: 3.492091417312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4938 , TFLOPS: 70.63071613885236, Tokens per sec: 175030.69520776605, Loss: 3.482090711593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4939 , TFLOPS: 69.61649563209558, Tokens per sec: 172517.34506641072, Loss: 3.491525173187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4940 , TFLOPS: 66.93724952852709, Tokens per sec: 165877.87807844498, Loss: 3.498946189880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4941 , TFLOPS: 70.61190314582585, Tokens per sec: 174984.0745386241, Loss: 3.5023579597473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4942 , TFLOPS: 69.68097043089375, Tokens per sec: 172677.120720318, Loss: 3.514096260070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4943 , TFLOPS: 66.2190384631324, Tokens per sec: 164098.07194091674, Loss: 3.488511562347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4944 , TFLOPS: 70.57095244054912, Tokens per sec: 174882.59417985575, Loss: 3.5234053134918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4945 , TFLOPS: 70.58631980959609, Tokens per sec: 174920.6762132096, Loss: 3.485750436782837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4946 , TFLOPS: 68.78113917857918, Tokens per sec: 170447.2397524871, Loss: 3.5000174045562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4947 , TFLOPS: 66.89908523833145, Tokens per sec: 165783.30276319652, Loss: 3.5089221000671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4948 , TFLOPS: 70.62296957486662, Tokens per sec: 175011.4983688549, Loss: 3.4996042251586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4949 , TFLOPS: 68.93218701016079, Tokens per sec: 170821.5529184386, Loss: 3.500410318374634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4950 , TFLOPS: 67.19603767644797, Tokens per sec: 166519.18361686077, Loss: 3.4983718395233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4951 , TFLOPS: 69.52633400123352, Tokens per sec: 172293.914613011, Loss: 3.4870004653930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4952 , TFLOPS: 70.57106422097482, Tokens per sec: 174882.87118406428, Loss: 3.477471351623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4953 , TFLOPS: 67.16203707092974, Tokens per sec: 166434.92637091072, Loss: 3.5043020248413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4954 , TFLOPS: 68.03875279039104, Tokens per sec: 168607.52450194507, Loss: 3.486483573913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4955 , TFLOPS: 70.60744416016632, Tokens per sec: 174973.02468096194, Loss: 3.499680280685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4956 , TFLOPS: 68.08765977509373, Tokens per sec: 168728.72139758742, Loss: 3.4806833267211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4957 , TFLOPS: 68.86097624922165, Tokens per sec: 170645.08480831853, Loss: 3.515390634536743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4958 , TFLOPS: 69.56359252132438, Tokens per sec: 172386.24533016092, Loss: 3.484522819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4959 , TFLOPS: 69.66139627308935, Tokens per sec: 172628.61380100725, Loss: 3.4816696643829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4960 , TFLOPS: 66.97883817034446, Tokens per sec: 165980.9393142355, Loss: 3.4863533973693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4961 , TFLOPS: 70.57741675506618, Tokens per sec: 174898.61346333206, Loss: 3.4712889194488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4962 , TFLOPS: 69.6346108364079, Tokens per sec: 172562.23653825113, Loss: 3.5061593055725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4963 , TFLOPS: 67.42842527429679, Tokens per sec: 167095.06568393734, Loss: 3.4935264587402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4964 , TFLOPS: 70.01849004943796, Tokens per sec: 173513.5315159259, Loss: 3.5096070766448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4965 , TFLOPS: 69.37462400204922, Tokens per sec: 171917.96052279623, Loss: 3.5009422302246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4966 , TFLOPS: 68.77327971883402, Tokens per sec: 170427.76314544765, Loss: 3.5154941082000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4967 , TFLOPS: 67.79428572473243, Tokens per sec: 168001.7081829726, Loss: 3.509166717529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4968 , TFLOPS: 70.57563442482605, Tokens per sec: 174894.19665265715, Loss: 3.460887908935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4969 , TFLOPS: 68.69044476335613, Tokens per sec: 170222.48900074977, Loss: 3.493654727935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4970 , TFLOPS: 66.74140006952938, Tokens per sec: 165392.5415444497, Loss: 3.4914658069610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4971 , TFLOPS: 70.63835949585034, Tokens per sec: 175049.6362883923, Loss: 3.5029184818267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4972 , TFLOPS: 70.63752531516629, Tokens per sec: 175047.56909676464, Loss: 3.4851443767547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4973 , TFLOPS: 66.39574650720472, Tokens per sec: 164535.97394012596, Loss: 3.4985756874084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4974 , TFLOPS: 68.72090215723162, Tokens per sec: 170297.96577793217, Loss: 3.4895949363708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4975 , TFLOPS: 70.61017482312432, Tokens per sec: 174979.79156457927, Loss: 3.5042495727539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4976 , TFLOPS: 68.04666910998989, Tokens per sec: 168627.1420139656, Loss: 3.501722812652588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4977 , TFLOPS: 68.5181709964147, Tokens per sec: 169795.57562874685, Loss: 3.4890246391296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4978 , TFLOPS: 70.58161435146903, Tokens per sec: 174909.01556962173, Loss: 3.4941468238830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4979 , TFLOPS: 70.56649581281087, Tokens per sec: 174871.5501653823, Loss: 3.483729362487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4980 , TFLOPS: 65.69974055551498, Tokens per sec: 162811.1945204538, Loss: 3.503704309463501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4981 , TFLOPS: 70.52745627750379, Tokens per sec: 174774.80589633153, Loss: 3.508327007293701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4982 , TFLOPS: 70.58602019825462, Tokens per sec: 174919.93374330053, Loss: 3.500783681869507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4983 , TFLOPS: 68.05792477735511, Tokens per sec: 168655.03479761074, Loss: 3.5211758613586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4984 , TFLOPS: 68.10617172308599, Tokens per sec: 168774.5960439717, Loss: 3.487272024154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4985 , TFLOPS: 70.60648193323765, Tokens per sec: 174970.64017663465, Loss: 3.5007240772247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4986 , TFLOPS: 69.55492414703193, Tokens per sec: 172364.76414376398, Loss: 3.464002847671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4987 , TFLOPS: 67.39971166959924, Tokens per sec: 167023.91020843215, Loss: 3.475219249725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4988 , TFLOPS: 69.65980247509746, Tokens per sec: 172624.664194012, Loss: 3.485208749771118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4989 , TFLOPS: 69.73255555483516, Tokens per sec: 172804.9543400153, Loss: 3.482661247253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4990 , TFLOPS: 66.48615676162702, Tokens per sec: 164760.02051009613, Loss: 3.491021156311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4991 , TFLOPS: 70.60677271624877, Tokens per sec: 174971.36076896862, Loss: 3.4862630367279053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4992 , TFLOPS: 69.5117907375024, Tokens per sec: 172257.8748034125, Loss: 3.5079851150512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4993 , TFLOPS: 68.04984985093189, Tokens per sec: 168635.02424040742, Loss: 3.5218939781188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4994 , TFLOPS: 68.27223436116365, Tokens per sec: 169186.11755443786, Loss: 3.5389890670776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4995 , TFLOPS: 69.55590987579107, Tokens per sec: 172367.20688821617, Loss: 3.497875928878784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4996 , TFLOPS: 68.97870149391274, Tokens per sec: 170936.8209331693, Loss: 3.482433557510376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4997 , TFLOPS: 68.16557557871816, Tokens per sec: 168921.80534210312, Loss: 3.481812000274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4998 , TFLOPS: 70.57979878327815, Tokens per sec: 174904.51639164338, Loss: 3.497983932495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4999 , TFLOPS: 69.66570307958203, Tokens per sec: 172639.28654193, Loss: 3.5064287185668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5000 , TFLOPS: 67.15786015160208, Tokens per sec: 166424.5755046911, Loss: 3.5022470951080322 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/optimizer.pt +[rank0]:[2024-08-25 07:08:28,276] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006317735998891294, 'preprocessing_with_comm': 0.0014026850039954297, 'state_converting': 0.9806032129999949, : 0.989674748001562}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000, took 4.44s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003000 + eval ppl=22.628711700439453, eval loss=3.1192195415496826 +------------------------------------------------------------------ +iteration: 5001 , TFLOPS: 65.01847953543384, Tokens per sec: 161122.95466559607, Loss: 3.510768413543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5002 , TFLOPS: 69.71525367464143, Tokens per sec: 172762.0783749358, Loss: 3.4783055782318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5003 , TFLOPS: 67.68048607263844, Tokens per sec: 167719.7001090171, Loss: 3.4884417057037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5004 , TFLOPS: 69.09302222505822, Tokens per sec: 171220.12029841595, Loss: 3.479509115219116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5005 , TFLOPS: 70.53375627158356, Tokens per sec: 174790.41797566286, Loss: 3.4789628982543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5006 , TFLOPS: 68.7279462923568, Tokens per sec: 170315.42192074787, Loss: 3.506528615951538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5007 , TFLOPS: 68.16227163928906, Tokens per sec: 168913.6178162388, Loss: 3.5143797397613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5008 , TFLOPS: 70.59962140659091, Tokens per sec: 174953.63903585463, Loss: 3.4878807067871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5009 , TFLOPS: 69.64129347691353, Tokens per sec: 172578.79685757772, Loss: 3.497223377227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5010 , TFLOPS: 66.03546786113675, Tokens per sec: 163643.16376720034, Loss: 3.4970991611480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5011 , TFLOPS: 70.53589468558428, Tokens per sec: 174795.71720112237, Loss: 3.5092625617980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5012 , TFLOPS: 70.50815105746089, Tokens per sec: 174726.9654344197, Loss: 3.5058462619781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5013 , TFLOPS: 66.29600513088883, Tokens per sec: 164288.80382219554, Loss: 3.491107225418091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5014 , TFLOPS: 68.67046843607059, Tokens per sec: 170172.98546116214, Loss: 3.483856201171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5015 , TFLOPS: 70.52435648758814, Tokens per sec: 174767.12427545048, Loss: 3.51883864402771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5016 , TFLOPS: 68.87765155434178, Tokens per sec: 170686.40805134267, Loss: 3.505540370941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5017 , TFLOPS: 67.71576650822908, Tokens per sec: 167807.12891487154, Loss: 3.4915432929992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5018 , TFLOPS: 70.4780267681529, Tokens per sec: 174652.31412705008, Loss: 3.514538288116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5019 , TFLOPS: 69.57731703596059, Tokens per sec: 172420.25618930958, Loss: 3.4814233779907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5020 , TFLOPS: 68.81246806795424, Tokens per sec: 170524.87619152057, Loss: 3.503448486328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5021 , TFLOPS: 68.98814467227456, Tokens per sec: 170960.22216940185, Loss: 3.480501651763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5022 , TFLOPS: 70.57288039341883, Tokens per sec: 174887.37185944844, Loss: 3.514014959335327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5023 , TFLOPS: 69.16596798037594, Tokens per sec: 171400.88791573158, Loss: 3.4810850620269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5024 , TFLOPS: 69.03181129013788, Tokens per sec: 171068.43285874222, Loss: 3.5222582817077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5025 , TFLOPS: 69.63625931636594, Tokens per sec: 172566.3216531822, Loss: 3.525068759918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5026 , TFLOPS: 69.44399273184156, Tokens per sec: 172089.8638768181, Loss: 3.501627206802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5027 , TFLOPS: 67.97580363357615, Tokens per sec: 168451.52955693568, Loss: 3.4987235069274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5028 , TFLOPS: 69.49019038907302, Tokens per sec: 172204.3467605297, Loss: 3.4918272495269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5029 , TFLOPS: 69.80537035239814, Tokens per sec: 172985.3974295885, Loss: 3.491422653198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5030 , TFLOPS: 67.11930389720533, Tokens per sec: 166329.0288589739, Loss: 3.4932680130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5031 , TFLOPS: 70.52532622890872, Tokens per sec: 174769.52740126874, Loss: 3.4745774269104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5032 , TFLOPS: 69.68225592162203, Tokens per sec: 172680.30630795116, Loss: 3.4972825050354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5033 , TFLOPS: 67.30920911105926, Tokens per sec: 166799.63489869024, Loss: 3.4882256984710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5034 , TFLOPS: 68.69825690007029, Tokens per sec: 170241.84833610538, Loss: 3.5132646560668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5035 , TFLOPS: 70.59759568884597, Tokens per sec: 174948.61908413144, Loss: 3.4957222938537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5036 , TFLOPS: 67.88351593198742, Tokens per sec: 168222.83046606922, Loss: 3.4933323860168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5037 , TFLOPS: 68.59014675840851, Tokens per sec: 169973.93949575222, Loss: 3.5012669563293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5038 , TFLOPS: 70.57307789902585, Tokens per sec: 174887.86130009967, Loss: 3.477733850479126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5039 , TFLOPS: 70.62042243920382, Tokens per sec: 175005.18628609233, Loss: 3.5135436058044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5040 , TFLOPS: 67.58873826669347, Tokens per sec: 167492.3389390303, Loss: 3.5202908515930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5041 , TFLOPS: 68.9770495891386, Tokens per sec: 170932.72733116665, Loss: 3.5055439472198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5042 , TFLOPS: 70.56190628160401, Tokens per sec: 174860.1768014731, Loss: 3.5095744132995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5043 , TFLOPS: 67.94263521612453, Tokens per sec: 168369.33456468774, Loss: 3.5070767402648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5044 , TFLOPS: 68.74078431768288, Tokens per sec: 170347.23596173117, Loss: 3.4682092666625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5045 , TFLOPS: 70.58023602636972, Tokens per sec: 174905.5999281906, Loss: 3.4847264289855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5046 , TFLOPS: 69.47387230881931, Tokens per sec: 172163.9087600761, Loss: 3.4768354892730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5047 , TFLOPS: 67.0544472796876, Tokens per sec: 166168.30701621875, Loss: 3.5172343254089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5048 , TFLOPS: 70.57075972536384, Tokens per sec: 174882.11661039732, Loss: 3.5107696056365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5049 , TFLOPS: 69.7997003690296, Tokens per sec: 172971.34658620102, Loss: 3.5107834339141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5050 , TFLOPS: 67.08626362311642, Tokens per sec: 166247.15142009495, Loss: 3.514639377593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5051 , TFLOPS: 69.88381570045159, Tokens per sec: 173179.79364353343, Loss: 3.4913878440856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5052 , TFLOPS: 70.52474962297802, Tokens per sec: 174768.09850825404, Loss: 3.50396466255188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5053 , TFLOPS: 68.07534270764428, Tokens per sec: 168698.1983476107, Loss: 3.465454578399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5054 , TFLOPS: 67.29821987502909, Tokens per sec: 166772.40236124324, Loss: 3.512053966522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5055 , TFLOPS: 70.55719664930047, Tokens per sec: 174848.5058138161, Loss: 3.497281074523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5056 , TFLOPS: 69.757797313963, Tokens per sec: 172867.50619974342, Loss: 3.4779582023620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5057 , TFLOPS: 68.03556320723591, Tokens per sec: 168599.62036352506, Loss: 3.5003538131713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5058 , TFLOPS: 69.63824541775712, Tokens per sec: 172571.2434312167, Loss: 3.4838294982910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5059 , TFLOPS: 70.59258117256773, Tokens per sec: 174936.1925603994, Loss: 3.509402275085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5060 , TFLOPS: 68.74379849231347, Tokens per sec: 170354.70541850454, Loss: 3.485908031463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5061 , TFLOPS: 68.76944087266777, Tokens per sec: 170418.25006176386, Loss: 3.500527858734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5062 , TFLOPS: 69.55627275075909, Tokens per sec: 172368.10613236087, Loss: 3.4995388984680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5063 , TFLOPS: 68.90160591229007, Tokens per sec: 170745.7695891289, Loss: 3.4782559871673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5064 , TFLOPS: 68.79989523386767, Tokens per sec: 170493.71932364203, Loss: 3.5113584995269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5065 , TFLOPS: 69.69675546180537, Tokens per sec: 172716.2377657817, Loss: 3.504408121109009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5066 , TFLOPS: 69.55507062661198, Tokens per sec: 172365.12713629942, Loss: 3.499751091003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5067 , TFLOPS: 67.95209357130628, Tokens per sec: 168392.77341075195, Loss: 3.4777157306671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5068 , TFLOPS: 70.62666634726378, Tokens per sec: 175020.65937808817, Loss: 3.4905216693878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5069 , TFLOPS: 69.48120045993943, Tokens per sec: 172182.06872581423, Loss: 3.5010509490966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5070 , TFLOPS: 67.6424502225624, Tokens per sec: 167625.44308253395, Loss: 3.476559638977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5071 , TFLOPS: 69.56520445129557, Tokens per sec: 172390.23987019827, Loss: 3.509747266769409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5072 , TFLOPS: 70.56746756928642, Tokens per sec: 174873.9582849765, Loss: 3.4932453632354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5073 , TFLOPS: 67.36775892459195, Tokens per sec: 166944.72778641866, Loss: 3.4791154861450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5074 , TFLOPS: 67.45053054262371, Tokens per sec: 167149.84497394782, Loss: 3.5111398696899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5075 , TFLOPS: 70.53327885069518, Tokens per sec: 174789.23487410918, Loss: 3.513737916946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5076 , TFLOPS: 69.8049547323011, Tokens per sec: 172984.36747720372, Loss: 3.514467716217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5077 , TFLOPS: 66.9462610055062, Tokens per sec: 165900.20951109534, Loss: 3.4903500080108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5078 , TFLOPS: 70.60134678375147, Tokens per sec: 174957.91471052446, Loss: 3.4844820499420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5079 , TFLOPS: 70.5403754378284, Tokens per sec: 174806.82099877895, Loss: 3.5283985137939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5080 , TFLOPS: 67.94024612785742, Tokens per sec: 168363.4141407821, Loss: 3.4946701526641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5081 , TFLOPS: 68.71362235050825, Tokens per sec: 170279.9256149347, Loss: 3.5028793811798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5082 , TFLOPS: 70.61457983796977, Tokens per sec: 174990.70767661868, Loss: 3.5066030025482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5083 , TFLOPS: 68.8383792455416, Tokens per sec: 170589.08694393391, Loss: 3.5159754753112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5084 , TFLOPS: 67.66499684124116, Tokens per sec: 167681.31608733497, Loss: 3.4671390056610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5085 , TFLOPS: 70.60319374034769, Tokens per sec: 174962.49167242902, Loss: 3.5094943046569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5086 , TFLOPS: 69.66020766121238, Tokens per sec: 172625.66828983967, Loss: 3.4970171451568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5087 , TFLOPS: 68.64439316204401, Tokens per sec: 170108.36805969567, Loss: 3.5004801750183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5088 , TFLOPS: 69.9523113694993, Tokens per sec: 173349.53345685446, Loss: 3.504390239715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5089 , TFLOPS: 70.58482399010563, Tokens per sec: 174916.9694077341, Loss: 3.486150026321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5090 , TFLOPS: 67.91389469786407, Tokens per sec: 168298.1123944089, Loss: 3.5025713443756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5091 , TFLOPS: 67.97096996199362, Tokens per sec: 168439.5511862813, Loss: 3.497406244277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5092 , TFLOPS: 70.6138602728067, Tokens per sec: 174988.924514878, Loss: 3.4785876274108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5093 , TFLOPS: 68.99531221768471, Tokens per sec: 170977.98413649853, Loss: 3.4938931465148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5094 , TFLOPS: 67.19404291281097, Tokens per sec: 166514.24037282693, Loss: 3.465876340866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5095 , TFLOPS: 68.85675016600133, Tokens per sec: 170634.61210855705, Loss: 3.5017881393432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5096 , TFLOPS: 69.65798858955534, Tokens per sec: 172620.1691858226, Loss: 3.5006680488586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5097 , TFLOPS: 67.76845398858717, Tokens per sec: 167937.69429520395, Loss: 3.4821977615356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5098 , TFLOPS: 70.54462661598325, Tokens per sec: 174817.35588655257, Loss: 3.4746429920196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5099 , TFLOPS: 69.24274442941623, Tokens per sec: 171591.14841407747, Loss: 3.4863977432250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5100 , TFLOPS: 68.81512052820807, Tokens per sec: 170531.44928022218, Loss: 3.4959592819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5101 , TFLOPS: 68.7092962551589, Tokens per sec: 170269.20507410032, Loss: 3.4821224212646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5102 , TFLOPS: 69.32209267707195, Tokens per sec: 171787.7821126994, Loss: 3.4920918941497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5103 , TFLOPS: 68.72895097548532, Tokens per sec: 170317.91163621534, Loss: 3.4910433292388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5104 , TFLOPS: 68.71085760914784, Tokens per sec: 170273.0742812822, Loss: 3.503422737121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5105 , TFLOPS: 70.57061175865215, Tokens per sec: 174881.74993258587, Loss: 3.4867544174194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5106 , TFLOPS: 66.58456359468998, Tokens per sec: 165003.88348283374, Loss: 3.49330997467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5107 , TFLOPS: 68.95244872645435, Tokens per sec: 170871.76368342634, Loss: 3.483126163482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5108 , TFLOPS: 69.06885242442887, Tokens per sec: 171160.2248699331, Loss: 3.4967739582061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5109 , TFLOPS: 70.52916822610064, Tokens per sec: 174779.04829354121, Loss: 3.493762969970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5110 , TFLOPS: 66.87081305369013, Tokens per sec: 165713.2411752171, Loss: 3.503018856048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5111 , TFLOPS: 68.69117113281813, Tokens per sec: 170224.2890242928, Loss: 3.4950613975524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5112 , TFLOPS: 70.59083176205638, Tokens per sec: 174931.8573284388, Loss: 3.4917521476745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5113 , TFLOPS: 69.02387318670775, Tokens per sec: 171048.76136398807, Loss: 3.4897234439849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5114 , TFLOPS: 66.24410669291181, Tokens per sec: 164160.1937758036, Loss: 3.485041856765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5115 , TFLOPS: 70.53356468037704, Tokens per sec: 174789.94319154756, Loss: 3.513230323791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5116 , TFLOPS: 69.75298195315244, Tokens per sec: 172855.57320519912, Loss: 3.5140860080718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5117 , TFLOPS: 66.94936378781657, Tokens per sec: 165907.89854746012, Loss: 3.479897975921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5118 , TFLOPS: 70.60942833033047, Tokens per sec: 174977.9416732031, Loss: 3.4845094680786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5119 , TFLOPS: 70.5696806922076, Tokens per sec: 174879.44264737083, Loss: 3.4963269233703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5120 , TFLOPS: 68.82586206011531, Tokens per sec: 170558.06797956576, Loss: 3.517683982849121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5121 , TFLOPS: 68.6158497771793, Tokens per sec: 170037.63440768752, Loss: 3.4876902103424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5122 , TFLOPS: 70.59513589231074, Tokens per sec: 174942.52343734817, Loss: 3.500394344329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5123 , TFLOPS: 69.6687148092114, Tokens per sec: 172646.74993972096, Loss: 3.512523889541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5124 , TFLOPS: 67.37584613837032, Tokens per sec: 166964.76879304159, Loss: 3.5015511512756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5125 , TFLOPS: 70.44096559877289, Tokens per sec: 174560.47246670155, Loss: 3.501218318939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5126 , TFLOPS: 70.46891191418557, Tokens per sec: 174629.72651483497, Loss: 3.482508659362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5127 , TFLOPS: 68.72658842696643, Tokens per sec: 170312.05698072934, Loss: 3.5082836151123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5128 , TFLOPS: 67.09705718450552, Tokens per sec: 166273.89905422606, Loss: 3.5064778327941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5129 , TFLOPS: 70.54215197418146, Tokens per sec: 174811.22345156345, Loss: 3.5193872451782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5130 , TFLOPS: 67.90853508536556, Tokens per sec: 168284.83068422755, Loss: 3.511467695236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5131 , TFLOPS: 68.78971928201246, Tokens per sec: 170468.50219397116, Loss: 3.4888546466827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5132 , TFLOPS: 69.62545306180672, Tokens per sec: 172539.54256397724, Loss: 3.4936864376068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5133 , TFLOPS: 68.89175181154972, Tokens per sec: 170721.35004197553, Loss: 3.5093443393707275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5134 , TFLOPS: 67.19973196896858, Tokens per sec: 166528.33848068674, Loss: 3.493147850036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5135 , TFLOPS: 70.60406817158024, Tokens per sec: 174964.6586093498, Loss: 3.507246971130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5136 , TFLOPS: 68.72507851944219, Tokens per sec: 170308.31526355425, Loss: 3.4941253662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5137 , TFLOPS: 68.78649581290237, Tokens per sec: 170460.51408242196, Loss: 3.4802463054656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5138 , TFLOPS: 69.74666840669198, Tokens per sec: 172839.9275415755, Loss: 3.5230212211608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5139 , TFLOPS: 69.66920662535101, Tokens per sec: 172647.96871429292, Loss: 3.491347074508667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5140 , TFLOPS: 69.5655988577372, Tokens per sec: 172391.21725280813, Loss: 3.500131607055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5141 , TFLOPS: 68.17584745004129, Tokens per sec: 168947.26017078187, Loss: 3.4940009117126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5142 , TFLOPS: 70.5641913419319, Tokens per sec: 174865.8394326858, Loss: 3.480405330657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5143 , TFLOPS: 68.69678518458676, Tokens per sec: 170238.20126301434, Loss: 3.5011355876922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5144 , TFLOPS: 68.09531642631684, Tokens per sec: 168747.69542276807, Loss: 3.4905316829681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5145 , TFLOPS: 70.56539443149423, Tokens per sec: 174868.8208211525, Loss: 3.4964499473571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5146 , TFLOPS: 70.54850221826257, Tokens per sec: 174826.96005593354, Loss: 3.4884445667266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5147 , TFLOPS: 67.57229622868051, Tokens per sec: 167451.59375759424, Loss: 3.5071024894714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5148 , TFLOPS: 68.65537403509865, Tokens per sec: 170135.57987276808, Loss: 3.518871784210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5149 , TFLOPS: 70.57854427287754, Tokens per sec: 174901.40757667497, Loss: 3.496527671813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5150 , TFLOPS: 68.02233258620801, Tokens per sec: 168566.8334271447, Loss: 3.501169204711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5151 , TFLOPS: 67.97573960787872, Tokens per sec: 168451.37089420445, Loss: 3.4926693439483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5152 , TFLOPS: 70.5404990388109, Tokens per sec: 174807.12729562935, Loss: 3.491584539413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5153 , TFLOPS: 69.75006439103471, Tokens per sec: 172848.34316487535, Loss: 3.5232131481170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5154 , TFLOPS: 67.27150452458686, Tokens per sec: 166706.19877990856, Loss: 3.5052719116210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5155 , TFLOPS: 69.86454414224312, Tokens per sec: 173132.03659935543, Loss: 3.489886999130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5156 , TFLOPS: 70.55510541334972, Tokens per sec: 174843.32350076147, Loss: 3.4965555667877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5157 , TFLOPS: 69.08873783389295, Tokens per sec: 171209.50310514378, Loss: 3.525156021118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5158 , TFLOPS: 68.06688708059566, Tokens per sec: 168677.24437232, Loss: 3.476393938064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5159 , TFLOPS: 70.61716941830126, Tokens per sec: 174997.12494194595, Loss: 3.5077741146087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5160 , TFLOPS: 69.68307693738011, Tokens per sec: 172682.34087544205, Loss: 3.50272274017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5161 , TFLOPS: 66.95069119413651, Tokens per sec: 165911.18800654678, Loss: 3.4852898120880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5162 , TFLOPS: 70.54939360832527, Tokens per sec: 174829.16901870366, Loss: 3.4825029373168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5163 , TFLOPS: 69.72379753222343, Tokens per sec: 172783.25099520764, Loss: 3.473296642303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5164 , TFLOPS: 68.00071730139223, Tokens per sec: 168513.26837025103, Loss: 3.4873979091644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5165 , TFLOPS: 69.57559200941985, Tokens per sec: 172415.98138351424, Loss: 3.4804205894470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5166 , TFLOPS: 70.5980571768695, Tokens per sec: 174949.76270229087, Loss: 3.4751224517822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5167 , TFLOPS: 68.8029782746966, Tokens per sec: 170501.35943844108, Loss: 3.5194458961486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5168 , TFLOPS: 68.69236719890925, Tokens per sec: 170227.25300782418, Loss: 3.5021567344665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5169 , TFLOPS: 69.58719996667753, Tokens per sec: 172444.74718032085, Loss: 3.480849504470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5170 , TFLOPS: 68.8207912784885, Tokens per sec: 170545.5020241018, Loss: 3.498694896697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5171 , TFLOPS: 68.25809126746312, Tokens per sec: 169151.06941025754, Loss: 3.49676513671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5172 , TFLOPS: 70.54064471764893, Tokens per sec: 174807.4883038378, Loss: 3.4880387783050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5173 , TFLOPS: 69.47608818967065, Tokens per sec: 172169.3999569251, Loss: 3.480154514312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5174 , TFLOPS: 67.52798916009453, Tokens per sec: 167341.79596081164, Loss: 3.5053422451019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5175 , TFLOPS: 68.97599623953616, Tokens per sec: 170930.11701481574, Loss: 3.4964311122894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5176 , TFLOPS: 69.70798073867587, Tokens per sec: 172744.05523843312, Loss: 3.5000293254852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5177 , TFLOPS: 68.76246820470878, Tokens per sec: 170400.9710224587, Loss: 3.4899356365203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5178 , TFLOPS: 68.78605345963787, Tokens per sec: 170459.4178823033, Loss: 3.503031015396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5179 , TFLOPS: 70.5664902729152, Tokens per sec: 174871.53643691057, Loss: 3.5196094512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5180 , TFLOPS: 68.6680401345901, Tokens per sec: 170166.96786259394, Loss: 3.502779483795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5181 , TFLOPS: 67.88497128656938, Tokens per sec: 168226.43699504386, Loss: 3.4978537559509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5182 , TFLOPS: 70.60001225599662, Tokens per sec: 174954.60760373762, Loss: 3.516392469406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5183 , TFLOPS: 69.72486591184547, Tokens per sec: 172785.89855760275, Loss: 3.4870455265045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5184 , TFLOPS: 66.96858596037866, Tokens per sec: 165955.5332085661, Loss: 3.49570369720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5185 , TFLOPS: 70.52064471737222, Tokens per sec: 174757.9260999702, Loss: 3.5137603282928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5186 , TFLOPS: 70.53056905267401, Tokens per sec: 174782.51969610382, Loss: 3.5020415782928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5187 , TFLOPS: 68.81076525932549, Tokens per sec: 170520.65644415867, Loss: 3.500624656677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5188 , TFLOPS: 67.81062597475815, Tokens per sec: 168042.20112256348, Loss: 3.500576972961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5189 , TFLOPS: 70.57280758138262, Tokens per sec: 174887.1914232018, Loss: 3.4666428565979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5190 , TFLOPS: 69.66949356260768, Tokens per sec: 172648.67977642373, Loss: 3.4982056617736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5191 , TFLOPS: 67.11553791284163, Tokens per sec: 166319.69633486326, Loss: 3.532358169555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5192 , TFLOPS: 70.54660195544557, Tokens per sec: 174822.25099534178, Loss: 3.506743907928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5193 , TFLOPS: 70.59458235462014, Tokens per sec: 174941.1517099737, Loss: 3.4730823040008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5194 , TFLOPS: 67.32954659009378, Tokens per sec: 166850.03341209606, Loss: 3.4792640209198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5195 , TFLOPS: 67.66018854897743, Tokens per sec: 167669.40060942832, Loss: 3.4902758598327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5196 , TFLOPS: 70.57243198108792, Tokens per sec: 174886.26064429575, Loss: 3.501072406768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5197 , TFLOPS: 68.9114983697341, Tokens per sec: 170770.2841884194, Loss: 3.513070583343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5198 , TFLOPS: 67.9397328396696, Tokens per sec: 168362.1421561093, Loss: 3.4897232055664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5199 , TFLOPS: 70.60859923050926, Tokens per sec: 174975.88707251326, Loss: 3.4785373210906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5200 , TFLOPS: 69.57556513742412, Tokens per sec: 172415.91479174866, Loss: 3.514570951461792 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/optimizer.pt +[rank0]:[2024-08-25 07:34:07,093] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006390043999999762, 'preprocessing_with_comm': 0.0012541370015242137, 'state_converting': 1.0022386150012608, : 1.011334689996147}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003200 + eval ppl=24.8361759185791, eval loss=3.212301254272461 +------------------------------------------------------------------ +iteration: 5201 , TFLOPS: 63.23108473241932, Tokens per sec: 156693.5934459342, Loss: 3.4968597888946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5202 , TFLOPS: 69.01807949673082, Tokens per sec: 171034.40396199762, Loss: 3.4881396293640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5203 , TFLOPS: 69.71934476791996, Tokens per sec: 172772.21655475115, Loss: 3.503849983215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5204 , TFLOPS: 68.10776716948415, Tokens per sec: 168778.54973589926, Loss: 3.5069425106048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5205 , TFLOPS: 68.8783535092552, Tokens per sec: 170688.14757294487, Loss: 3.5032944679260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5206 , TFLOPS: 69.3934079812343, Tokens per sec: 171964.50929244302, Loss: 3.5126707553863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5207 , TFLOPS: 68.09124529919644, Tokens per sec: 168737.60672129205, Loss: 3.5123963356018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5208 , TFLOPS: 68.70362255535422, Tokens per sec: 170255.14502097468, Loss: 3.513516902923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5209 , TFLOPS: 70.46569565453873, Tokens per sec: 174621.75626912984, Loss: 3.5042166709899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5210 , TFLOPS: 67.69797440463911, Tokens per sec: 167763.0381222136, Loss: 3.502873420715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5211 , TFLOPS: 67.47298474759035, Tokens per sec: 167205.48896739, Loss: 3.497910499572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5212 , TFLOPS: 70.55199333023775, Tokens per sec: 174835.61141598577, Loss: 3.51632022857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5213 , TFLOPS: 69.5334376958801, Tokens per sec: 172311.51835088176, Loss: 3.4866111278533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5214 , TFLOPS: 67.35970885799951, Tokens per sec: 166924.77883461452, Loss: 3.502399444580078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5215 , TFLOPS: 68.87768637570925, Tokens per sec: 170686.49434252715, Loss: 3.4983513355255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5216 , TFLOPS: 70.58323345638706, Tokens per sec: 174913.02788996778, Loss: 3.490466594696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5217 , TFLOPS: 67.54303471857895, Tokens per sec: 167379.08051214108, Loss: 3.479888439178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5218 , TFLOPS: 67.92015792384001, Tokens per sec: 168313.63335832857, Loss: 3.4664530754089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5219 , TFLOPS: 70.61660145377853, Tokens per sec: 174995.71746329218, Loss: 3.4885246753692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5220 , TFLOPS: 69.53184025050248, Tokens per sec: 172307.55970526292, Loss: 3.5047006607055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5221 , TFLOPS: 65.4823877862454, Tokens per sec: 162272.57041481967, Loss: 3.500997304916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5222 , TFLOPS: 70.55158565704652, Tokens per sec: 174834.601156909, Loss: 3.5080931186676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5223 , TFLOPS: 69.80805149628516, Tokens per sec: 172992.04159949292, Loss: 3.484820604324341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5224 , TFLOPS: 67.99668242752509, Tokens per sec: 168503.26950833018, Loss: 3.498289108276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5225 , TFLOPS: 68.74531028467693, Tokens per sec: 170358.45180651886, Loss: 3.483513116836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5226 , TFLOPS: 70.57534365358862, Tokens per sec: 174893.4760894997, Loss: 3.4834907054901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5227 , TFLOPS: 69.63067902512284, Tokens per sec: 172552.493076762, Loss: 3.486098051071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5228 , TFLOPS: 67.11965794410357, Tokens per sec: 166329.90622618925, Loss: 3.5034806728363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5229 , TFLOPS: 70.54299705336928, Tokens per sec: 174813.31765088398, Loss: 3.512556314468384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5230 , TFLOPS: 69.01907440621967, Tokens per sec: 171036.86945730934, Loss: 3.488301992416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5231 , TFLOPS: 68.03375265735173, Tokens per sec: 168595.13362146352, Loss: 3.5051608085632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5232 , TFLOPS: 69.01398201571308, Tokens per sec: 171024.24995266084, Loss: 3.492915630340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5233 , TFLOPS: 70.55817963113962, Tokens per sec: 174850.9417510979, Loss: 3.507798910140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5234 , TFLOPS: 67.40824116265978, Tokens per sec: 167045.04723183738, Loss: 3.4970896244049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5235 , TFLOPS: 68.70050325019318, Tokens per sec: 170247.4150391658, Loss: 3.5067005157470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5236 , TFLOPS: 69.52527407999612, Tokens per sec: 172291.28801142474, Loss: 3.486788749694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5237 , TFLOPS: 68.19983869277752, Tokens per sec: 169006.7131131358, Loss: 3.5232279300689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5238 , TFLOPS: 67.9693643912379, Tokens per sec: 168435.5724050804, Loss: 3.514350652694702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5239 , TFLOPS: 69.62036681011467, Tokens per sec: 172526.9382719875, Loss: 3.502415895462036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5240 , TFLOPS: 69.489247482641, Tokens per sec: 172202.01013452144, Loss: 3.492262125015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5241 , TFLOPS: 67.16984639895692, Tokens per sec: 166454.27874603058, Loss: 3.495856761932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5242 , TFLOPS: 70.57585270946518, Tokens per sec: 174894.73758603894, Loss: 3.508899211883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5243 , TFLOPS: 68.74618240713588, Tokens per sec: 170360.61302204433, Loss: 3.5057613849639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5244 , TFLOPS: 68.10471270342424, Tokens per sec: 168770.98043252557, Loss: 3.5074567794799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5245 , TFLOPS: 68.80964119876704, Tokens per sec: 170517.87089826932, Loss: 3.514512777328491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5246 , TFLOPS: 70.58682271841917, Tokens per sec: 174921.9224766732, Loss: 3.4908158779144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5247 , TFLOPS: 68.57690423962043, Tokens per sec: 169941.1230754115, Loss: 3.5226800441741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5248 , TFLOPS: 67.93967637970542, Tokens per sec: 168362.00224209845, Loss: 3.4997034072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5249 , TFLOPS: 70.53649011850726, Tokens per sec: 174797.19274949783, Loss: 3.4990386962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5250 , TFLOPS: 69.81025727419004, Tokens per sec: 172997.50776012772, Loss: 3.5188019275665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5251 , TFLOPS: 67.94581661512747, Tokens per sec: 168377.21842187707, Loss: 3.491346836090088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5252 , TFLOPS: 69.14245937809554, Tokens per sec: 171342.63100959442, Loss: 3.5143089294433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5253 , TFLOPS: 70.57674998416837, Tokens per sec: 174896.9611315964, Loss: 3.498513698577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5254 , TFLOPS: 66.21415280403869, Tokens per sec: 164085.96473948273, Loss: 3.4896304607391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5255 , TFLOPS: 68.68874765867292, Tokens per sec: 170218.28338839326, Loss: 3.509693145751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5256 , TFLOPS: 70.54592179818391, Tokens per sec: 174820.5654907219, Loss: 3.5089926719665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5257 , TFLOPS: 67.33631474136693, Tokens per sec: 166866.80563652434, Loss: 3.4932055473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5258 , TFLOPS: 67.95690468790185, Tokens per sec: 168404.69588766407, Loss: 3.470594644546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5259 , TFLOPS: 70.57000546828789, Tokens per sec: 174880.24747827483, Loss: 3.4968504905700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5260 , TFLOPS: 69.53437875175604, Tokens per sec: 172313.85039100822, Loss: 3.5133771896362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5261 , TFLOPS: 67.05554219224469, Tokens per sec: 166171.02033014983, Loss: 3.494103193283081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5262 , TFLOPS: 70.58323566765938, Tokens per sec: 174913.03336974417, Loss: 3.4985475540161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5263 , TFLOPS: 69.75052624141797, Tokens per sec: 172849.48768100207, Loss: 3.5019752979278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5264 , TFLOPS: 68.92321844744558, Tokens per sec: 170799.32783206154, Loss: 3.512115240097046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5265 , TFLOPS: 68.17702599732057, Tokens per sec: 168950.18074076762, Loss: 3.4924418926239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5266 , TFLOPS: 70.5803362816692, Tokens per sec: 174905.84837186674, Loss: 3.4851107597351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5267 , TFLOPS: 69.60086770015269, Tokens per sec: 172478.61732949744, Loss: 3.5308544635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5268 , TFLOPS: 68.00578263674649, Tokens per sec: 168525.82082925164, Loss: 3.518634557723999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5269 , TFLOPS: 69.53415756518326, Tokens per sec: 172313.30226631515, Loss: 3.5060696601867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5270 , TFLOPS: 69.78966583196376, Tokens per sec: 172946.4798979564, Loss: 3.4868438243865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5271 , TFLOPS: 68.00868149524646, Tokens per sec: 168533.00451995016, Loss: 3.4830703735351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5272 , TFLOPS: 67.91134103946273, Tokens per sec: 168291.78414758167, Loss: 3.5138754844665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5273 , TFLOPS: 70.59780194654714, Tokens per sec: 174949.13021343612, Loss: 3.488025665283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5274 , TFLOPS: 67.26790077831781, Tokens per sec: 166697.26829966842, Loss: 3.504668951034546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5275 , TFLOPS: 68.73950085979536, Tokens per sec: 170344.05541170132, Loss: 3.513242721557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5276 , TFLOPS: 69.69848146440539, Tokens per sec: 172720.51499035946, Loss: 3.4779555797576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5277 , TFLOPS: 68.9433435180767, Tokens per sec: 170849.199974045, Loss: 3.4949722290039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5278 , TFLOPS: 67.39497754291025, Tokens per sec: 167012.17852098975, Loss: 3.516097068786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5279 , TFLOPS: 70.61486892644204, Tokens per sec: 174991.42406969873, Loss: 3.4854824542999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5280 , TFLOPS: 68.5654217406464, Tokens per sec: 169912.66817805218, Loss: 3.506767988204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5281 , TFLOPS: 68.10578203939873, Tokens per sec: 168773.63036486745, Loss: 3.5144236087799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5282 , TFLOPS: 69.93067531879085, Tokens per sec: 173295.9169397912, Loss: 3.473578929901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5283 , TFLOPS: 70.59948726145343, Tokens per sec: 174953.30660942665, Loss: 3.5170845985412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5284 , TFLOPS: 67.50376782192441, Tokens per sec: 167281.77281662542, Loss: 3.5034255981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5285 , TFLOPS: 68.00923616953027, Tokens per sec: 168534.37906392786, Loss: 3.5181918144226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5286 , TFLOPS: 70.55588097828303, Tokens per sec: 174845.24543610177, Loss: 3.5080628395080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5287 , TFLOPS: 69.5258211738789, Tokens per sec: 172292.6437703336, Loss: 3.486570358276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5288 , TFLOPS: 67.9947071383205, Tokens per sec: 168498.37452408517, Loss: 3.517315149307251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5289 , TFLOPS: 70.53458188561416, Tokens per sec: 174792.46393817954, Loss: 3.486994743347168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5290 , TFLOPS: 69.78943335883444, Tokens per sec: 172945.9038039329, Loss: 3.5010874271392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5291 , TFLOPS: 67.16052921030747, Tokens per sec: 166431.18972618415, Loss: 3.4969356060028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5292 , TFLOPS: 68.8413908766873, Tokens per sec: 170596.55009767148, Loss: 3.5184872150421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5293 , TFLOPS: 70.56139897855611, Tokens per sec: 174858.91964863637, Loss: 3.488647699356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5294 , TFLOPS: 67.97251122255489, Tokens per sec: 168443.37059973596, Loss: 3.4770336151123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5295 , TFLOPS: 67.71569476950823, Tokens per sec: 167806.9511384186, Loss: 3.5034632682800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5296 , TFLOPS: 70.53896849821119, Tokens per sec: 174803.3344474203, Loss: 3.491957664489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5297 , TFLOPS: 68.95883512234315, Tokens per sec: 170887.58987595837, Loss: 3.491364002227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5298 , TFLOPS: 67.96066759861135, Tokens per sec: 168414.020794921, Loss: 3.5061895847320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5299 , TFLOPS: 69.62080405674527, Tokens per sec: 172528.02181730483, Loss: 3.4848337173461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5300 , TFLOPS: 70.59748362668098, Tokens per sec: 174948.34138174192, Loss: 3.4850339889526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5301 , TFLOPS: 68.32310093819423, Tokens per sec: 169312.17053573567, Loss: 3.5017664432525635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5302 , TFLOPS: 68.07295848302581, Tokens per sec: 168692.28997636214, Loss: 3.4800074100494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5303 , TFLOPS: 70.59819439498033, Tokens per sec: 174950.10274388533, Loss: 3.5212864875793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5304 , TFLOPS: 68.08394641432618, Tokens per sec: 168719.519280545, Loss: 3.5011518001556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5305 , TFLOPS: 68.0839172588235, Tokens per sec: 168719.44702999765, Loss: 3.5163958072662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5306 , TFLOPS: 69.63042455091107, Tokens per sec: 172551.86246163258, Loss: 3.5039772987365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5307 , TFLOPS: 69.55562425720348, Tokens per sec: 172366.4990938926, Loss: 3.4877636432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5308 , TFLOPS: 67.9609553519589, Tokens per sec: 168414.73387941488, Loss: 3.522197723388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5309 , TFLOPS: 69.55330189309663, Tokens per sec: 172360.7440198063, Loss: 3.4941413402557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5310 , TFLOPS: 69.64439473721875, Tokens per sec: 172586.4821222461, Loss: 3.490658760070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5311 , TFLOPS: 68.19344225238748, Tokens per sec: 168990.86202922324, Loss: 3.4914073944091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5312 , TFLOPS: 68.78091529692963, Tokens per sec: 170446.6849490969, Loss: 3.502202272415161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5313 , TFLOPS: 69.53219482912586, Tokens per sec: 172308.43839015168, Loss: 3.4918770790100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5314 , TFLOPS: 68.89875444588633, Tokens per sec: 170738.7033412655, Loss: 3.4743523597717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5315 , TFLOPS: 67.48833646174646, Tokens per sec: 167243.53220619977, Loss: 3.504232883453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5316 , TFLOPS: 70.58749533569517, Tokens per sec: 174923.58929637802, Loss: 3.5053277015686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5317 , TFLOPS: 68.80145695307428, Tokens per sec: 170497.58943587358, Loss: 3.4737603664398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5318 , TFLOPS: 68.3223722543714, Tokens per sec: 169310.36477695155, Loss: 3.4855120182037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5319 , TFLOPS: 69.02877524451087, Tokens per sec: 171060.90920323046, Loss: 3.481733798980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5320 , TFLOPS: 70.62446567373009, Tokens per sec: 175015.20586664748, Loss: 3.4997339248657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5321 , TFLOPS: 67.92667495216179, Tokens per sec: 168329.78327241988, Loss: 3.514974594116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5322 , TFLOPS: 68.83975076928081, Tokens per sec: 170592.48573084548, Loss: 3.4925060272216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5323 , TFLOPS: 70.54349304745334, Tokens per sec: 174814.54677886254, Loss: 3.501098394393921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5324 , TFLOPS: 68.05579002708916, Tokens per sec: 168649.74465128954, Loss: 3.490182876586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5325 , TFLOPS: 67.4702634628065, Tokens per sec: 167198.74532392135, Loss: 3.486692428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5326 , TFLOPS: 70.5666644263763, Tokens per sec: 174871.96800837177, Loss: 3.4953019618988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5327 , TFLOPS: 69.44295762612015, Tokens per sec: 172087.2987708141, Loss: 3.4939095973968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5328 , TFLOPS: 66.93510052090389, Tokens per sec: 165872.55260082195, Loss: 3.5088109970092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5329 , TFLOPS: 70.61555134625783, Tokens per sec: 174993.115181177, Loss: 3.479001522064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5330 , TFLOPS: 70.58625525138778, Tokens per sec: 174920.51623085776, Loss: 3.5063416957855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5331 , TFLOPS: 68.2464871264955, Tokens per sec: 169122.31307063813, Loss: 3.491702079772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5332 , TFLOPS: 67.832339777569, Tokens per sec: 168096.01031790162, Loss: 3.501312494277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5333 , TFLOPS: 70.58880821439402, Tokens per sec: 174926.8427544193, Loss: 3.4889321327209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5334 , TFLOPS: 68.90678693881506, Tokens per sec: 170758.60874359502, Loss: 3.499314785003662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5335 , TFLOPS: 66.2125474012016, Tokens per sec: 164081.9863744027, Loss: 3.5083069801330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5336 , TFLOPS: 70.55503032504211, Tokens per sec: 174843.1374236636, Loss: 3.479724407196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5337 , TFLOPS: 69.69505365568757, Tokens per sec: 172712.02050275236, Loss: 3.4975907802581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5338 , TFLOPS: 68.10518660175119, Tokens per sec: 168772.15480478393, Loss: 3.477296829223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5339 , TFLOPS: 67.83818883350075, Tokens per sec: 168110.50492282715, Loss: 3.503858804702759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5340 , TFLOPS: 70.60817983829074, Tokens per sec: 174974.8477723959, Loss: 3.5023105144500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5341 , TFLOPS: 67.87584475404037, Tokens per sec: 168203.82044206644, Loss: 3.514862060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5342 , TFLOPS: 68.93632918757659, Tokens per sec: 170831.81769037346, Loss: 3.5154178142547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5343 , TFLOPS: 69.60050529302775, Tokens per sec: 172477.7192447194, Loss: 3.4942095279693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5344 , TFLOPS: 68.94253571120416, Tokens per sec: 170847.19813962758, Loss: 3.525500774383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5345 , TFLOPS: 67.07907082477014, Tokens per sec: 166229.32687344073, Loss: 3.476388692855835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5346 , TFLOPS: 69.54224897663576, Tokens per sec: 172333.35367523698, Loss: 3.4965789318084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5347 , TFLOPS: 69.54168960336692, Tokens per sec: 172331.96748665677, Loss: 3.4873523712158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5348 , TFLOPS: 68.81530295115533, Tokens per sec: 170531.90134438104, Loss: 3.5187394618988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5349 , TFLOPS: 69.94683827850068, Tokens per sec: 173335.97053444918, Loss: 3.4965574741363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5350 , TFLOPS: 69.64857936767412, Tokens per sec: 172596.85209748958, Loss: 3.4736218452453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5351 , TFLOPS: 68.04710377134779, Tokens per sec: 168628.21915269239, Loss: 3.514723062515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5352 , TFLOPS: 67.99257616864736, Tokens per sec: 168493.0937464894, Loss: 3.490485191345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5353 , TFLOPS: 70.55969100371091, Tokens per sec: 174854.68709882098, Loss: 3.517422914505005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5354 , TFLOPS: 67.72296885942158, Tokens per sec: 167824.97713453107, Loss: 3.5087687969207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5355 , TFLOPS: 67.24156136472288, Tokens per sec: 166631.99633125425, Loss: 3.4955391883850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5356 , TFLOPS: 70.64462596626359, Tokens per sec: 175065.16529238506, Loss: 3.498842239379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5357 , TFLOPS: 69.67976755087244, Tokens per sec: 172674.139851117, Loss: 3.4757204055786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5358 , TFLOPS: 68.01197768214115, Tokens per sec: 168541.17283418015, Loss: 3.5084214210510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5359 , TFLOPS: 68.62349502504551, Tokens per sec: 170056.58017409357, Loss: 3.514896869659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5360 , TFLOPS: 70.52118565496139, Tokens per sec: 174759.26660290535, Loss: 3.500232458114624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5361 , TFLOPS: 67.68876946270976, Tokens per sec: 167740.2272621046, Loss: 3.48569393157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5362 , TFLOPS: 68.38044105790867, Tokens per sec: 169454.26566892368, Loss: 3.488550901412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5363 , TFLOPS: 70.59720849993549, Tokens per sec: 174947.65958735888, Loss: 3.498896598815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5364 , TFLOPS: 68.95115054575169, Tokens per sec: 170868.54664863873, Loss: 3.490830659866333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5365 , TFLOPS: 67.77735307783747, Tokens per sec: 167959.74721868185, Loss: 3.497718334197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5366 , TFLOPS: 69.8736423664623, Tokens per sec: 173154.5830012227, Loss: 3.4907054901123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5367 , TFLOPS: 70.58351849591995, Tokens per sec: 174913.73424932992, Loss: 3.4973771572113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5368 , TFLOPS: 68.9398300701026, Tokens per sec: 170840.49326292696, Loss: 3.4512922763824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5369 , TFLOPS: 68.62671371385595, Tokens per sec: 170064.55643953368, Loss: 3.4821348190307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5370 , TFLOPS: 70.64201596352693, Tokens per sec: 175058.6974180881, Loss: 3.4886369705200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5371 , TFLOPS: 68.06749012732223, Tokens per sec: 168678.73878853951, Loss: 3.490957260131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5372 , TFLOPS: 67.93422144823036, Tokens per sec: 168348.48432099295, Loss: 3.4904439449310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5373 , TFLOPS: 69.60118993349991, Tokens per sec: 172479.41585922876, Loss: 3.49114727973938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5374 , TFLOPS: 68.86413956847271, Tokens per sec: 170652.92386189103, Loss: 3.5019328594207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5375 , TFLOPS: 67.30122793790042, Tokens per sec: 166779.85667240384, Loss: 3.4873878955841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5376 , TFLOPS: 69.64897657094548, Tokens per sec: 172597.83641095157, Loss: 3.4921486377716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5377 , TFLOPS: 69.76381834639876, Tokens per sec: 172882.42698139072, Loss: 3.500385046005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5378 , TFLOPS: 67.98217188781486, Tokens per sec: 168467.31079246034, Loss: 3.4985744953155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5379 , TFLOPS: 68.76447602902117, Tokens per sec: 170405.9466322849, Loss: 3.5094590187072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5380 , TFLOPS: 69.60509986604977, Tokens per sec: 172489.10510280196, Loss: 3.525649309158325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5381 , TFLOPS: 69.67250527155363, Tokens per sec: 172656.1431229589, Loss: 3.5224528312683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5382 , TFLOPS: 67.4580951362648, Tokens per sec: 167168.5908702992, Loss: 3.4943246841430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5383 , TFLOPS: 69.69396668460209, Tokens per sec: 172709.3268686628, Loss: 3.490906000137329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5384 , TFLOPS: 69.77107016930353, Tokens per sec: 172900.39779740298, Loss: 3.5134665966033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5385 , TFLOPS: 68.45062595697668, Tokens per sec: 169628.19158031908, Loss: 3.523507595062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5386 , TFLOPS: 69.04462433347187, Tokens per sec: 171100.18499159714, Loss: 3.505345106124878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5387 , TFLOPS: 69.58426199692344, Tokens per sec: 172437.46656762614, Loss: 3.4976654052734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5388 , TFLOPS: 68.1381843157764, Tokens per sec: 168853.92677513673, Loss: 3.4894204139709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5389 , TFLOPS: 68.8329642248675, Tokens per sec: 170575.66792618972, Loss: 3.483369827270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5390 , TFLOPS: 70.55877423666122, Tokens per sec: 174852.41524908165, Loss: 3.51605486869812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5391 , TFLOPS: 67.27861689095602, Tokens per sec: 166723.82400726315, Loss: 3.482574701309204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5392 , TFLOPS: 68.10248009170958, Tokens per sec: 168765.44777475417, Loss: 3.4922118186950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5393 , TFLOPS: 70.61977657740617, Tokens per sec: 175003.58576941022, Loss: 3.5065088272094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5394 , TFLOPS: 68.88566086096935, Tokens per sec: 170706.25599546378, Loss: 3.475517988204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5395 , TFLOPS: 66.99783413280028, Tokens per sec: 166028.01340177906, Loss: 3.5061891078948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5396 , TFLOPS: 69.85258632851655, Tokens per sec: 173102.40381967908, Loss: 3.479200601577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5397 , TFLOPS: 69.68486945659964, Tokens per sec: 172686.78293553035, Loss: 3.507343292236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5398 , TFLOPS: 68.79811361507656, Tokens per sec: 170489.30427601616, Loss: 3.503638505935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5399 , TFLOPS: 68.2881560052801, Tokens per sec: 169225.57314247228, Loss: 3.491286277770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5400 , TFLOPS: 70.5262832105656, Tokens per sec: 174771.8989072347, Loss: 3.507065773010254 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/optimizer.pt +[rank0]:[2024-08-25 07:59:49,392] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006368727001245134, 'preprocessing_with_comm': 0.0014306649973150343, 'state_converting': 0.9769069350004429, : 0.9860924480017275}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400, took 4.30s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003400 + eval ppl=23.73668098449707, eval loss=3.1670215129852295 +------------------------------------------------------------------ +iteration: 5401 , TFLOPS: 65.92089537758923, Tokens per sec: 163359.24053176862, Loss: 3.484354257583618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5402 , TFLOPS: 66.81455779747539, Tokens per sec: 165573.83445328893, Loss: 3.4941983222961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5403 , TFLOPS: 70.56099111201989, Tokens per sec: 174857.9089104294, Loss: 3.5061402320861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5404 , TFLOPS: 68.90164998498538, Tokens per sec: 170745.87880612293, Loss: 3.4764792919158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5405 , TFLOPS: 69.62840809094885, Tokens per sec: 172546.8654517148, Loss: 3.487165689468384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5406 , TFLOPS: 67.90361752684211, Tokens per sec: 168272.64443249264, Loss: 3.4938042163848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5407 , TFLOPS: 70.61839069541122, Tokens per sec: 175000.15140115924, Loss: 3.4903178215026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5408 , TFLOPS: 68.14444198636475, Tokens per sec: 168869.43397219398, Loss: 3.502464532852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5409 , TFLOPS: 67.13401416534093, Tokens per sec: 166365.48252388378, Loss: 3.4954943656921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5410 , TFLOPS: 70.53614881070664, Tokens per sec: 174796.34695116975, Loss: 3.4978342056274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5411 , TFLOPS: 69.0486130589427, Tokens per sec: 171110.0694927082, Loss: 3.4974493980407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5412 , TFLOPS: 66.65577718644693, Tokens per sec: 165180.35860803214, Loss: 3.4942877292633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5413 , TFLOPS: 69.52791927807172, Tokens per sec: 172297.8431036487, Loss: 3.491824150085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5414 , TFLOPS: 69.6993349962548, Tokens per sec: 172722.63013630657, Loss: 3.49936580657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5415 , TFLOPS: 66.73244780288803, Tokens per sec: 165370.35684153886, Loss: 3.4954049587249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5416 , TFLOPS: 70.59161114334188, Tokens per sec: 174933.7887211202, Loss: 3.5144224166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5417 , TFLOPS: 68.90323737358939, Tokens per sec: 170749.81252994895, Loss: 3.492873191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5418 , TFLOPS: 68.75949085412995, Tokens per sec: 170393.5928197421, Loss: 3.5161871910095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5419 , TFLOPS: 68.71432688780978, Tokens per sec: 170281.67153597908, Loss: 3.5085537433624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5420 , TFLOPS: 70.56987635520298, Tokens per sec: 174879.92752182746, Loss: 3.469423532485962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5421 , TFLOPS: 67.20733259999683, Tokens per sec: 166547.17368165342, Loss: 3.504540205001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5422 , TFLOPS: 68.11483225559091, Tokens per sec: 168796.0577975553, Loss: 3.487806797027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5423 , TFLOPS: 70.5990627996075, Tokens per sec: 174952.2547462141, Loss: 3.4955761432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5424 , TFLOPS: 68.09484483731995, Tokens per sec: 168746.526773284, Loss: 3.500706195831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5425 , TFLOPS: 69.67761056138282, Tokens per sec: 172668.79459354974, Loss: 3.5112385749816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5426 , TFLOPS: 68.72333784742766, Tokens per sec: 170304.00168655135, Loss: 3.5085647106170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5427 , TFLOPS: 70.57592212199336, Tokens per sec: 174894.90959793018, Loss: 3.498821496963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5428 , TFLOPS: 67.41393579251557, Tokens per sec: 167059.1591519353, Loss: 3.4990692138671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5429 , TFLOPS: 68.028056801142, Tokens per sec: 168581.01866232546, Loss: 3.493408679962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5430 , TFLOPS: 70.63899160735053, Tokens per sec: 175051.2027303226, Loss: 3.5267765522003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5431 , TFLOPS: 67.24987674941488, Tokens per sec: 166652.60277053626, Loss: 3.502864360809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5432 , TFLOPS: 66.92553523135696, Tokens per sec: 165848.848759621, Loss: 3.4984073638916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5433 , TFLOPS: 70.56973766685665, Tokens per sec: 174879.58383682749, Loss: 3.4999873638153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5434 , TFLOPS: 69.69098876862563, Tokens per sec: 172701.9472648286, Loss: 3.4919748306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5435 , TFLOPS: 67.14853707284065, Tokens per sec: 166401.4718884984, Loss: 3.5051395893096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5436 , TFLOPS: 69.98761873550413, Tokens per sec: 173437.0289992419, Loss: 3.4802112579345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5437 , TFLOPS: 70.50854553338677, Tokens per sec: 174727.9429892191, Loss: 3.5199949741363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5438 , TFLOPS: 68.28220343153978, Tokens per sec: 169210.8220090137, Loss: 3.5021626949310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5439 , TFLOPS: 67.84143122053304, Tokens per sec: 168118.5399150716, Loss: 3.491230010986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5440 , TFLOPS: 70.57960966108126, Tokens per sec: 174904.04772600599, Loss: 3.4874939918518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5441 , TFLOPS: 68.91282065773493, Tokens per sec: 170773.56096374744, Loss: 3.4594502449035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5442 , TFLOPS: 67.50864339629032, Tokens per sec: 167293.85502699297, Loss: 3.476449728012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5443 , TFLOPS: 69.67812918843262, Tokens per sec: 172670.0798085106, Loss: 3.507586717605591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5444 , TFLOPS: 68.89237177709863, Tokens per sec: 170722.88638490057, Loss: 3.4996824264526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5445 , TFLOPS: 69.73825157145806, Tokens per sec: 172819.06969667494, Loss: 3.520700693130493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5446 , TFLOPS: 67.77261230642162, Tokens per sec: 167947.99906487425, Loss: 3.4929287433624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5447 , TFLOPS: 70.6392838740421, Tokens per sec: 175051.92699938017, Loss: 3.489786386489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5448 , TFLOPS: 68.11911383301914, Tokens per sec: 168806.66801807712, Loss: 3.4873123168945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5449 , TFLOPS: 68.0432433700585, Tokens per sec: 168618.6526530392, Loss: 3.5108492374420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5450 , TFLOPS: 69.62187590167379, Tokens per sec: 172530.67796711114, Loss: 3.511043071746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5451 , TFLOPS: 68.89635700616343, Tokens per sec: 170732.76222153235, Loss: 3.4854981899261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5452 , TFLOPS: 67.89945738148377, Tokens per sec: 168262.33513401682, Loss: 3.4890599250793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5453 , TFLOPS: 69.86883487122168, Tokens per sec: 173142.6694984271, Loss: 3.5038883686065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5454 , TFLOPS: 69.14917375924435, Tokens per sec: 171359.2699857313, Loss: 3.485440492630005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5455 , TFLOPS: 67.58810069531513, Tokens per sec: 167490.75896692052, Loss: 3.4863836765289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5456 , TFLOPS: 69.11073331811144, Tokens per sec: 171264.0103380398, Loss: 3.4943933486938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5457 , TFLOPS: 70.62548857387591, Tokens per sec: 175017.7407258906, Loss: 3.4776668548583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5458 , TFLOPS: 68.0069798848997, Tokens per sec: 168528.78774206305, Loss: 3.505128860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5459 , TFLOPS: 68.10814178835317, Tokens per sec: 168779.47808272435, Loss: 3.4874444007873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5460 , TFLOPS: 70.61229093859828, Tokens per sec: 174985.03553183307, Loss: 3.4850239753723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5461 , TFLOPS: 67.83876105314845, Tokens per sec: 168111.92294614934, Loss: 3.475515604019165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5462 , TFLOPS: 67.37231025916405, Tokens per sec: 166956.0064948592, Loss: 3.5059125423431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5463 , TFLOPS: 70.55332696462035, Tokens per sec: 174838.91630889784, Loss: 3.5236732959747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5464 , TFLOPS: 69.79037775041957, Tokens per sec: 172948.24411031415, Loss: 3.5109493732452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5465 , TFLOPS: 68.30548465475147, Tokens per sec: 169268.5154447707, Loss: 3.4885175228118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5466 , TFLOPS: 68.68098676746142, Tokens per sec: 170199.05104503845, Loss: 3.4672181606292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5467 , TFLOPS: 70.49557628305729, Tokens per sec: 174695.8037582218, Loss: 3.492004632949829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5468 , TFLOPS: 67.86467249740552, Tokens per sec: 168176.1343593996, Loss: 3.461808919906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5469 , TFLOPS: 68.01009971893244, Tokens per sec: 168536.51903447424, Loss: 3.504148483276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5470 , TFLOPS: 70.59741757007974, Tokens per sec: 174948.17768620735, Loss: 3.4946837425231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5471 , TFLOPS: 68.96281316018786, Tokens per sec: 170897.44789195462, Loss: 3.4913454055786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5472 , TFLOPS: 67.96215337771761, Tokens per sec: 168417.70271921842, Loss: 3.490553855895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5473 , TFLOPS: 68.85673011509346, Tokens per sec: 170634.56242019855, Loss: 3.5116348266601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5474 , TFLOPS: 70.53059837743707, Tokens per sec: 174782.5923660971, Loss: 3.490445137023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5475 , TFLOPS: 68.07073404409176, Tokens per sec: 168686.77757164137, Loss: 3.4872031211853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5476 , TFLOPS: 66.60167799533593, Tokens per sec: 165046.29485294124, Loss: 3.4930694103240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5477 , TFLOPS: 70.53914168885146, Tokens per sec: 174803.76363290538, Loss: 3.4973156452178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5478 , TFLOPS: 68.81541392593299, Tokens per sec: 170532.17635210496, Loss: 3.4897751808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5479 , TFLOPS: 67.93561993543112, Tokens per sec: 168351.9499263325, Loss: 3.4928674697875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5480 , TFLOPS: 69.68655475034242, Tokens per sec: 172690.95927907538, Loss: 3.5173120498657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5481 , TFLOPS: 69.71611769721352, Tokens per sec: 172764.21951804942, Loss: 3.5172886848449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5482 , TFLOPS: 66.75763278398152, Tokens per sec: 165432.76799904322, Loss: 3.493178367614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5483 , TFLOPS: 69.51783493348331, Tokens per sec: 172272.85298687636, Loss: 3.509035110473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5484 , TFLOPS: 70.55261082848602, Tokens per sec: 174837.14164466804, Loss: 3.5203285217285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5485 , TFLOPS: 68.30083755330419, Tokens per sec: 169256.99941546386, Loss: 3.5065832138061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5486 , TFLOPS: 68.81408210007235, Tokens per sec: 170528.87594090955, Loss: 3.4956064224243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5487 , TFLOPS: 69.53455652387606, Tokens per sec: 172314.29092990485, Loss: 3.4850831031799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5488 , TFLOPS: 68.88144414156261, Tokens per sec: 170695.80650026386, Loss: 3.495950222015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5489 , TFLOPS: 67.49299067786974, Tokens per sec: 167255.0658664575, Loss: 3.49985671043396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5490 , TFLOPS: 70.54834925905283, Tokens per sec: 174826.58100616198, Loss: 3.532681703567505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5491 , TFLOPS: 68.04243911627331, Tokens per sec: 168616.65962356367, Loss: 3.4693360328674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5492 , TFLOPS: 68.9544011047857, Tokens per sec: 170876.60189200364, Loss: 3.4668726921081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5493 , TFLOPS: 68.81799520364605, Tokens per sec: 170538.5730426291, Loss: 3.4984445571899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5494 , TFLOPS: 70.50708769401575, Tokens per sec: 174724.33030266344, Loss: 3.4845900535583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5495 , TFLOPS: 67.16344477535313, Tokens per sec: 166438.41481754332, Loss: 3.4957566261291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5496 , TFLOPS: 67.96039810632774, Tokens per sec: 168413.3529633552, Loss: 3.4994029998779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5497 , TFLOPS: 70.62591656618787, Tokens per sec: 175018.80133798686, Loss: 3.4886748790740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5498 , TFLOPS: 67.99298147184922, Tokens per sec: 168494.09813247144, Loss: 3.4917099475860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5499 , TFLOPS: 68.01436650555732, Tokens per sec: 168547.09260175607, Loss: 3.4982757568359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5500 , TFLOPS: 70.54862563151488, Tokens per sec: 174827.26588756786, Loss: 3.507153272628784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5501 , TFLOPS: 69.84208878695128, Tokens per sec: 173076.3897552807, Loss: 3.5110089778900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5502 , TFLOPS: 66.15712150286868, Tokens per sec: 163944.63489266697, Loss: 3.496647596359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5503 , TFLOPS: 69.96546134121299, Tokens per sec: 173382.12053534997, Loss: 3.48655366897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5504 , TFLOPS: 70.5982654236502, Tokens per sec: 174950.27876075372, Loss: 3.484302282333374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5505 , TFLOPS: 67.87002632797628, Tokens per sec: 168189.40174132708, Loss: 3.5049450397491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5506 , TFLOPS: 68.77463841140812, Tokens per sec: 170431.13013531858, Loss: 3.5190165042877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5507 , TFLOPS: 70.46211568030354, Tokens per sec: 174612.88469860834, Loss: 3.481149196624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5508 , TFLOPS: 69.00734653368525, Tokens per sec: 171007.80649723724, Loss: 3.509066104888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5509 , TFLOPS: 66.24530338134079, Tokens per sec: 164163.15930155673, Loss: 3.480851888656616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5510 , TFLOPS: 70.52776237245658, Tokens per sec: 174775.56443334368, Loss: 3.4876718521118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5511 , TFLOPS: 68.8134687124537, Tokens per sec: 170527.3558988203, Loss: 3.477625846862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5512 , TFLOPS: 68.79813309541473, Tokens per sec: 170489.35255044006, Loss: 3.465172529220581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5513 , TFLOPS: 68.63374855262597, Tokens per sec: 170081.98954495727, Loss: 3.4990122318267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5514 , TFLOPS: 70.64178537428069, Tokens per sec: 175058.1259925344, Loss: 3.4710631370544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5515 , TFLOPS: 68.27921168797903, Tokens per sec: 169203.40813890236, Loss: 3.514174222946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5516 , TFLOPS: 68.00882606855386, Tokens per sec: 168533.362788532, Loss: 3.5192456245422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5517 , TFLOPS: 69.62594630923842, Tokens per sec: 172540.76488544873, Loss: 3.5020384788513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5518 , TFLOPS: 68.83296897677324, Tokens per sec: 170575.6797019356, Loss: 3.488722562789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5519 , TFLOPS: 68.86087726326788, Tokens per sec: 170644.83951022083, Loss: 3.4930262565612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5520 , TFLOPS: 68.89654532012517, Tokens per sec: 170733.22888427405, Loss: 3.513545513153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5521 , TFLOPS: 70.62395809259608, Tokens per sec: 175013.94802468276, Loss: 3.4960010051727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5522 , TFLOPS: 67.69588899980893, Tokens per sec: 167757.8702593181, Loss: 3.4861795902252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5523 , TFLOPS: 68.9892086143947, Tokens per sec: 170962.85873517845, Loss: 3.4866113662719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5524 , TFLOPS: 69.6279369228779, Tokens per sec: 172545.69784533154, Loss: 3.490044593811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5525 , TFLOPS: 68.88312972058678, Tokens per sec: 170699.98355076762, Loss: 3.464109420776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5526 , TFLOPS: 68.15462126199276, Tokens per sec: 168894.65933854, Loss: 3.50691556930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5527 , TFLOPS: 70.56636925208844, Tokens per sec: 174871.2365339703, Loss: 3.5172488689422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5528 , TFLOPS: 67.86686879523303, Tokens per sec: 168181.57702735838, Loss: 3.5018560886383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5529 , TFLOPS: 67.26805006530437, Tokens per sec: 166697.63824926643, Loss: 3.494450807571411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5530 , TFLOPS: 70.55912636137619, Tokens per sec: 174853.28785291506, Loss: 3.4941906929016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5531 , TFLOPS: 69.70468288974374, Tokens per sec: 172735.88280549124, Loss: 3.4883320331573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5532 , TFLOPS: 67.97770629774742, Tokens per sec: 168456.24456834784, Loss: 3.5145516395568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5533 , TFLOPS: 68.52036867591808, Tokens per sec: 169801.0217206506, Loss: 3.5158371925354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5534 , TFLOPS: 70.54996933879632, Tokens per sec: 174830.59574223284, Loss: 3.516462564468384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5535 , TFLOPS: 67.2523785967762, Tokens per sec: 166658.80262389887, Loss: 3.501634120941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5536 , TFLOPS: 67.77940850948076, Tokens per sec: 167964.8408047184, Loss: 3.4923276901245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5537 , TFLOPS: 70.60398474035871, Tokens per sec: 174964.45185759215, Loss: 3.495795965194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5538 , TFLOPS: 68.95919741684688, Tokens per sec: 170888.4876816486, Loss: 3.4882678985595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5539 , TFLOPS: 68.72745866479583, Tokens per sec: 170314.21352593522, Loss: 3.5064761638641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5540 , TFLOPS: 69.16821184027827, Tokens per sec: 171406.44844775114, Loss: 3.514829397201538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5541 , TFLOPS: 70.58842732402182, Tokens per sec: 174925.89886611846, Loss: 3.5034375190734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5542 , TFLOPS: 67.16937885775882, Tokens per sec: 166453.12012743775, Loss: 3.476257801055908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5543 , TFLOPS: 68.24118366240208, Tokens per sec: 169109.17050238984, Loss: 3.477280616760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5544 , TFLOPS: 70.54149554096895, Tokens per sec: 174809.59673775075, Loss: 3.481505870819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5545 , TFLOPS: 68.91547939803661, Tokens per sec: 170780.14961509945, Loss: 3.4847259521484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5546 , TFLOPS: 67.90980600109835, Tokens per sec: 168287.98015341628, Loss: 3.492995262145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5547 , TFLOPS: 70.567072781776, Tokens per sec: 174872.97995803627, Loss: 3.510178327560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5548 , TFLOPS: 69.79376073198881, Tokens per sec: 172956.6275113089, Loss: 3.46341872215271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5549 , TFLOPS: 67.35275609312326, Tokens per sec: 166907.5491173409, Loss: 3.493997573852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5550 , TFLOPS: 68.96448032848414, Tokens per sec: 170901.57931864657, Loss: 3.5057458877563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5551 , TFLOPS: 69.64137146983931, Tokens per sec: 172578.99013263945, Loss: 3.4724514484405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5552 , TFLOPS: 68.05619481582052, Tokens per sec: 168650.74776235703, Loss: 3.4977712631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5553 , TFLOPS: 68.80256594852227, Tokens per sec: 170500.33764875965, Loss: 3.4944357872009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5554 , TFLOPS: 69.69403324494918, Tokens per sec: 172709.49181253513, Loss: 3.5054798126220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5555 , TFLOPS: 68.22671126293044, Tokens per sec: 169073.3063022327, Loss: 3.4763145446777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5556 , TFLOPS: 67.91201442909193, Tokens per sec: 168293.45288126278, Loss: 3.479994297027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5557 , TFLOPS: 69.60675143317778, Tokens per sec: 172493.19786808026, Loss: 3.5109381675720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5558 , TFLOPS: 68.82172318987388, Tokens per sec: 170547.81140317325, Loss: 3.511486053466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5559 , TFLOPS: 69.68047001147134, Tokens per sec: 172675.88062586356, Loss: 3.501462936401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5560 , TFLOPS: 68.7990064066276, Tokens per sec: 170491.51671182876, Loss: 3.4838685989379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5561 , TFLOPS: 69.58915647951012, Tokens per sec: 172449.5956346477, Loss: 3.4943389892578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5562 , TFLOPS: 68.12174716174516, Tokens per sec: 168813.19369674532, Loss: 3.5035042762756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5563 , TFLOPS: 67.9565496734271, Tokens per sec: 168403.81612268754, Loss: 3.5182387828826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5564 , TFLOPS: 70.5771127127063, Tokens per sec: 174897.86001287124, Loss: 3.4984004497528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5565 , TFLOPS: 68.0724506796522, Tokens per sec: 168691.0315836632, Loss: 3.4845733642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5566 , TFLOPS: 67.97603848573523, Tokens per sec: 168452.111546457, Loss: 3.490417242050171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5567 , TFLOPS: 70.45943012957312, Tokens per sec: 174606.22960806053, Loss: 3.4781265258789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5568 , TFLOPS: 69.77752705492762, Tokens per sec: 172916.39867126412, Loss: 3.502490282058716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5569 , TFLOPS: 67.96312020583152, Tokens per sec: 168420.09862578957, Loss: 3.5021896362304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5570 , TFLOPS: 69.01651362300204, Tokens per sec: 171030.5235544025, Loss: 3.5076961517333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5571 , TFLOPS: 70.53723824627625, Tokens per sec: 174799.04669252248, Loss: 3.518078565597534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5572 , TFLOPS: 67.41435895719842, Tokens per sec: 167060.20780063485, Loss: 3.4779791831970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5573 , TFLOPS: 68.19205735446859, Tokens per sec: 168987.43009962107, Loss: 3.5086302757263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5574 , TFLOPS: 70.49836128927454, Tokens per sec: 174702.7053104219, Loss: 3.4708971977233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5575 , TFLOPS: 68.89843698024345, Tokens per sec: 170737.91662643073, Loss: 3.494708776473999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5576 , TFLOPS: 66.58939616405608, Tokens per sec: 165015.85912207435, Loss: 3.5094079971313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5577 , TFLOPS: 70.59390055651461, Tokens per sec: 174939.4621391619, Loss: 3.4827558994293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5578 , TFLOPS: 69.76254004823114, Tokens per sec: 172879.2592177152, Loss: 3.4906208515167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5579 , TFLOPS: 68.17694919555994, Tokens per sec: 168949.99041754438, Loss: 3.5256991386413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5580 , TFLOPS: 68.79575589237069, Tokens per sec: 170483.46157942642, Loss: 3.4739601612091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5581 , TFLOPS: 70.52416902458731, Tokens per sec: 174766.65972148368, Loss: 3.512627601623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5582 , TFLOPS: 67.34943139224329, Tokens per sec: 166899.31014231432, Loss: 3.49440860748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5583 , TFLOPS: 66.77195208255496, Tokens per sec: 165468.25279830914, Loss: 3.5006184577941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5584 , TFLOPS: 70.55158177421848, Tokens per sec: 174834.5915348334, Loss: 3.4744925498962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5585 , TFLOPS: 68.86420303933276, Tokens per sec: 170653.0811496741, Loss: 3.513035297393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5586 , TFLOPS: 68.89616803874883, Tokens per sec: 170732.2939394625, Loss: 3.4963252544403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5587 , TFLOPS: 69.03810893017031, Tokens per sec: 171084.03910448492, Loss: 3.4994752407073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5588 , TFLOPS: 70.59711676320893, Tokens per sec: 174947.43225364486, Loss: 3.498401403427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5589 , TFLOPS: 67.58990989105402, Tokens per sec: 167495.24235326087, Loss: 3.5118722915649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5590 , TFLOPS: 68.80162716170116, Tokens per sec: 170498.011231601, Loss: 3.5119926929473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5591 , TFLOPS: 69.44011105092405, Tokens per sec: 172080.24464390203, Loss: 3.5048394203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5592 , TFLOPS: 68.67627997044906, Tokens per sec: 170187.3870835453, Loss: 3.482614040374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5593 , TFLOPS: 67.02896463807734, Tokens per sec: 166105.15822316383, Loss: 3.5024771690368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5594 , TFLOPS: 69.70345757938112, Tokens per sec: 172732.84635143363, Loss: 3.480443000793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5595 , TFLOPS: 68.91300628103333, Tokens per sec: 170774.02095872903, Loss: 3.5018069744110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5596 , TFLOPS: 67.11837636276083, Tokens per sec: 166326.7303264441, Loss: 3.5078659057617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5597 , TFLOPS: 70.53040914301909, Tokens per sec: 174782.12342236345, Loss: 3.498807907104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5598 , TFLOPS: 68.54377416659415, Tokens per sec: 169859.02310487357, Loss: 3.5066866874694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5599 , TFLOPS: 68.88607161134534, Tokens per sec: 170707.27388014336, Loss: 3.5025293827056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5600 , TFLOPS: 68.64817115964416, Tokens per sec: 170117.73035392963, Loss: 3.508178472518921 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/optimizer.pt +[rank0]:[2024-08-25 08:25:33,041] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006336817998089828, 'preprocessing_with_comm': 0.0012847300022258423, 'state_converting': 0.9606917589990189, : 0.9697137809998821}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600, took 4.22s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003600 + eval ppl=24.226396560668945, eval loss=3.1874427795410156 +------------------------------------------------------------------ +iteration: 5601 , TFLOPS: 65.58900993664179, Tokens per sec: 162536.7918489014, Loss: 3.4806041717529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5602 , TFLOPS: 66.17740936867409, Tokens per sec: 163994.91045902573, Loss: 3.487321376800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5603 , TFLOPS: 67.88681516933461, Tokens per sec: 168231.00633965657, Loss: 3.4770352840423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5604 , TFLOPS: 70.59311687012055, Tokens per sec: 174937.52007794727, Loss: 3.501044273376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5605 , TFLOPS: 68.91501120896407, Tokens per sec: 170778.98939100245, Loss: 3.5040345191955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5606 , TFLOPS: 66.64327608637065, Tokens per sec: 165149.37950493328, Loss: 3.4782888889312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5607 , TFLOPS: 69.51503408702786, Tokens per sec: 172265.91218082103, Loss: 3.490285873413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5608 , TFLOPS: 70.48530472325452, Tokens per sec: 174670.34970152483, Loss: 3.494999647140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5609 , TFLOPS: 67.38871579203897, Tokens per sec: 166996.6612125419, Loss: 3.5160059928894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5610 , TFLOPS: 68.9849444766187, Tokens per sec: 170952.29173203598, Loss: 3.4787564277648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5611 , TFLOPS: 70.58135766865625, Tokens per sec: 174908.37948133575, Loss: 3.5055506229400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5612 , TFLOPS: 68.97816959979485, Tokens per sec: 170935.50284095222, Loss: 3.5247623920440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5613 , TFLOPS: 64.37846035312894, Tokens per sec: 159536.91662791136, Loss: 3.4911909103393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5614 , TFLOPS: 70.55083909582382, Tokens per sec: 174832.75109595864, Loss: 3.483757734298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5615 , TFLOPS: 69.61419976467108, Tokens per sec: 172511.65565402215, Loss: 3.523937702178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5616 , TFLOPS: 67.10733353621464, Tokens per sec: 166299.36498579485, Loss: 3.495284080505371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5617 , TFLOPS: 69.67642107520922, Tokens per sec: 172665.84691577885, Loss: 3.5101606845855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5618 , TFLOPS: 70.5798102247489, Tokens per sec: 174904.54474486827, Loss: 3.4794094562530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5619 , TFLOPS: 68.33313774376994, Tokens per sec: 169337.0428455977, Loss: 3.4880826473236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5620 , TFLOPS: 67.68880446843664, Tokens per sec: 167740.31401015198, Loss: 3.515836715698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5621 , TFLOPS: 70.49992430543836, Tokens per sec: 174706.57863665628, Loss: 3.501098155975342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5622 , TFLOPS: 68.15098204790314, Tokens per sec: 168885.64096513335, Loss: 3.503591537475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5623 , TFLOPS: 68.02487909360835, Tokens per sec: 168573.1439530037, Loss: 3.4912288188934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5624 , TFLOPS: 68.8800807955642, Tokens per sec: 170692.42797869485, Loss: 3.459697961807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5625 , TFLOPS: 69.8152696997611, Tokens per sec: 173009.9291028571, Loss: 3.50380802154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5626 , TFLOPS: 68.30465151581609, Tokens per sec: 169266.45083471097, Loss: 3.502424716949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5627 , TFLOPS: 68.92724348176637, Tokens per sec: 170809.30231050262, Loss: 3.484452724456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5628 , TFLOPS: 69.52054022625396, Tokens per sec: 172279.55700037463, Loss: 3.4912221431732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5629 , TFLOPS: 68.88428575956013, Tokens per sec: 170702.84834269178, Loss: 3.498300552368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5630 , TFLOPS: 68.25216151351088, Tokens per sec: 169136.37482674757, Loss: 3.517575979232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5631 , TFLOPS: 69.53831940472311, Tokens per sec: 172323.61576315938, Loss: 3.5271413326263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5632 , TFLOPS: 68.87213639723251, Tokens per sec: 170672.7408844812, Loss: 3.5108675956726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5633 , TFLOPS: 68.91820814904372, Tokens per sec: 170786.91176069182, Loss: 3.4873127937316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5634 , TFLOPS: 69.91733061353658, Tokens per sec: 173262.84729013048, Loss: 3.5096209049224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5635 , TFLOPS: 69.54736121317686, Tokens per sec: 172346.02236054518, Loss: 3.4974417686462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5636 , TFLOPS: 67.351593849878, Tokens per sec: 166904.66895054752, Loss: 3.5089938640594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5637 , TFLOPS: 69.70392520920012, Tokens per sec: 172734.00518963876, Loss: 3.4832968711853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5638 , TFLOPS: 70.48697862767881, Tokens per sec: 174674.49782108402, Loss: 3.4989655017852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5639 , TFLOPS: 67.18702189487914, Tokens per sec: 166496.8415169629, Loss: 3.4680919647216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5640 , TFLOPS: 68.6854180990193, Tokens per sec: 170210.03237279033, Loss: 3.493318557739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5641 , TFLOPS: 70.60816580747564, Tokens per sec: 174974.81300249047, Loss: 3.503844738006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5642 , TFLOPS: 68.68517571766573, Tokens per sec: 170209.43172509567, Loss: 3.495124101638794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5643 , TFLOPS: 67.24979388164826, Tokens per sec: 166652.39741508197, Loss: 3.4937589168548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5644 , TFLOPS: 70.56698061549355, Tokens per sec: 174872.7515598354, Loss: 3.504866123199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5645 , TFLOPS: 69.81690936199495, Tokens per sec: 173013.9923664962, Loss: 3.503976345062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5646 , TFLOPS: 67.12420557662617, Tokens per sec: 166341.17576054332, Loss: 3.4852771759033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5647 , TFLOPS: 68.637795017825, Tokens per sec: 170092.01713147538, Loss: 3.5267624855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5648 , TFLOPS: 70.54014849097338, Tokens per sec: 174806.2585994719, Loss: 3.521482467651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5649 , TFLOPS: 68.72557615855297, Tokens per sec: 170309.54846809022, Loss: 3.4953341484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5650 , TFLOPS: 67.01299675565971, Tokens per sec: 166065.5880515255, Loss: 3.492424964904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5651 , TFLOPS: 70.53601983084586, Tokens per sec: 174796.02732486645, Loss: 3.4853858947753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5652 , TFLOPS: 68.85982705568382, Tokens per sec: 170642.23698013765, Loss: 3.4871487617492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5653 , TFLOPS: 68.40426787703052, Tokens per sec: 169513.3111514484, Loss: 3.4866933822631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5654 , TFLOPS: 69.63722430660665, Tokens per sec: 172568.71300530116, Loss: 3.4905283451080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5655 , TFLOPS: 70.5841860534518, Tokens per sec: 174915.38853043143, Loss: 3.4795799255371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5656 , TFLOPS: 67.13674657613261, Tokens per sec: 166372.25373882544, Loss: 3.5016214847564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5657 , TFLOPS: 68.72606582916082, Tokens per sec: 170310.76192579814, Loss: 3.46697735786438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5658 , TFLOPS: 70.51311193503162, Tokens per sec: 174739.2590355257, Loss: 3.5077991485595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5659 , TFLOPS: 67.95649560164692, Tokens per sec: 168403.68212685976, Loss: 3.470749855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5660 , TFLOPS: 68.70692072904295, Tokens per sec: 170263.31825869915, Loss: 3.5074751377105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5661 , TFLOPS: 69.5744118004453, Tokens per sec: 172413.0566956646, Loss: 3.4771225452423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5662 , TFLOPS: 69.68493584354465, Tokens per sec: 172686.9474496932, Loss: 3.4705312252044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5663 , TFLOPS: 66.641697646728, Tokens per sec: 165145.46795761932, Loss: 3.515510082244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5664 , TFLOPS: 70.63482823365246, Tokens per sec: 175040.88543166532, Loss: 3.4937803745269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5665 , TFLOPS: 68.80434321983677, Tokens per sec: 170504.74192285957, Loss: 3.5041117668151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5666 , TFLOPS: 68.06872010062175, Tokens per sec: 168681.7867978685, Loss: 3.473212242126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5667 , TFLOPS: 68.68074943346188, Tokens per sec: 170198.46290524307, Loss: 3.4906439781188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5668 , TFLOPS: 69.64988260293947, Tokens per sec: 172600.08165804032, Loss: 3.4893646240234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5669 , TFLOPS: 68.84783960233354, Tokens per sec: 170612.53075020862, Loss: 3.48718523979187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5670 , TFLOPS: 68.07548968283933, Tokens per sec: 168698.56256833472, Loss: 3.503891944885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5671 , TFLOPS: 70.44902834491988, Tokens per sec: 174580.4528398386, Loss: 3.4886674880981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5672 , TFLOPS: 67.86092252284695, Tokens per sec: 168166.84150934964, Loss: 3.4809350967407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5673 , TFLOPS: 68.66945103324944, Tokens per sec: 170170.46422489514, Loss: 3.4856414794921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5674 , TFLOPS: 69.64206700040582, Tokens per sec: 172580.71373400223, Loss: 3.497715473175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5675 , TFLOPS: 70.5643876986175, Tokens per sec: 174866.32602618317, Loss: 3.4986581802368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5676 , TFLOPS: 67.13218064517189, Tokens per sec: 166360.93885892592, Loss: 3.527099370956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5677 , TFLOPS: 69.06557520896922, Tokens per sec: 171152.10356900896, Loss: 3.494910478591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5678 , TFLOPS: 70.6142786116801, Tokens per sec: 174989.96120469007, Loss: 3.5068068504333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5679 , TFLOPS: 68.03047320657696, Tokens per sec: 168587.0067811823, Loss: 3.5211832523345947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5680 , TFLOPS: 68.20499508531542, Tokens per sec: 169019.49122186826, Loss: 3.501786947250366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5681 , TFLOPS: 70.5723764320566, Tokens per sec: 174886.12298767694, Loss: 3.483166217803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5682 , TFLOPS: 69.73585191745047, Tokens per sec: 172813.1230897004, Loss: 3.5036633014678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5683 , TFLOPS: 66.97979933985903, Tokens per sec: 165983.32119817415, Loss: 3.496152639389038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5684 , TFLOPS: 69.98858522628804, Tokens per sec: 173439.42406987216, Loss: 3.4804067611694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5685 , TFLOPS: 70.55328948818267, Tokens per sec: 174838.82343815686, Loss: 3.4816672801971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5686 , TFLOPS: 67.71224158298786, Tokens per sec: 167798.39376182124, Loss: 3.4955990314483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5687 , TFLOPS: 67.81494684007824, Tokens per sec: 168052.90870280925, Loss: 3.493872880935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5688 , TFLOPS: 70.54336778170521, Tokens per sec: 174814.23635653954, Loss: 3.4955801963806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5689 , TFLOPS: 68.7333898016051, Tokens per sec: 170328.91153631723, Loss: 3.4831957817077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5690 , TFLOPS: 67.04131582039454, Tokens per sec: 166135.7658135409, Loss: 3.48618745803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5691 , TFLOPS: 70.59006690157452, Tokens per sec: 174929.9619199084, Loss: 3.4969351291656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5692 , TFLOPS: 68.98584713959774, Tokens per sec: 170954.5286303346, Loss: 3.4843850135803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5693 , TFLOPS: 69.79258831813088, Tokens per sec: 172953.7221406169, Loss: 3.482900381088257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5694 , TFLOPS: 68.096489957502, Tokens per sec: 168750.60356232003, Loss: 3.499830484390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5695 , TFLOPS: 70.63229921116681, Tokens per sec: 175034.61823535108, Loss: 3.4948840141296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5696 , TFLOPS: 68.9510153811623, Tokens per sec: 170868.21169589664, Loss: 3.4851646423339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5697 , TFLOPS: 68.29915565423984, Tokens per sec: 169252.8314843059, Loss: 3.4988553524017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5698 , TFLOPS: 69.53282079558127, Tokens per sec: 172309.9896039841, Loss: 3.4723124504089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5699 , TFLOPS: 68.96308242371616, Tokens per sec: 170898.11515663957, Loss: 3.5020928382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5700 , TFLOPS: 68.85807483162979, Tokens per sec: 170637.89477590835, Loss: 3.4906070232391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5701 , TFLOPS: 68.96449645358359, Tokens per sec: 170901.61927841933, Loss: 3.4725849628448486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5702 , TFLOPS: 70.59817674374682, Tokens per sec: 174950.05900218422, Loss: 3.5017902851104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5703 , TFLOPS: 66.6629688225816, Tokens per sec: 165198.180274598, Loss: 3.505512237548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5704 , TFLOPS: 69.65110066093159, Tokens per sec: 172603.100139925, Loss: 3.488948106765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5705 , TFLOPS: 68.81749526810884, Tokens per sec: 170537.3341472955, Loss: 3.506014823913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5706 , TFLOPS: 67.35369776004185, Tokens per sec: 166909.88267169835, Loss: 3.476552724838257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5707 , TFLOPS: 68.68893396601868, Tokens per sec: 170218.74507851954, Loss: 3.500797748565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5708 , TFLOPS: 70.58347892660456, Tokens per sec: 174913.63619220752, Loss: 3.4809699058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5709 , TFLOPS: 68.09505571809127, Tokens per sec: 168747.04935906574, Loss: 3.5005135536193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5710 , TFLOPS: 67.9837220455734, Tokens per sec: 168471.15225414996, Loss: 3.494447946548462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5711 , TFLOPS: 70.54651836861652, Tokens per sec: 174822.04385797156, Loss: 3.4960403442382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5712 , TFLOPS: 69.71141649230637, Tokens per sec: 172752.56941440902, Loss: 3.5047011375427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5713 , TFLOPS: 68.19119273528065, Tokens per sec: 168985.28747802772, Loss: 3.470332622528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5714 , TFLOPS: 68.69241217185083, Tokens per sec: 170227.36445572757, Loss: 3.4779841899871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5715 , TFLOPS: 70.57337604579021, Tokens per sec: 174888.60014062535, Loss: 3.4966328144073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5716 , TFLOPS: 68.87819426527066, Tokens per sec: 170687.752948809, Loss: 3.5231757164001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5717 , TFLOPS: 67.97888899345999, Tokens per sec: 168459.1754186083, Loss: 3.509537696838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5718 , TFLOPS: 70.52874997429609, Tokens per sec: 174778.01181949532, Loss: 3.511213779449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5719 , TFLOPS: 68.91583677870709, Tokens per sec: 170781.03524376964, Loss: 3.506248712539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5720 , TFLOPS: 67.9070340468688, Tokens per sec: 168281.1109454795, Loss: 3.502753734588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5721 , TFLOPS: 69.60260127319049, Tokens per sec: 172482.9133144539, Loss: 3.5077691078186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5722 , TFLOPS: 70.62086649719139, Tokens per sec: 175006.28671070255, Loss: 3.501516819000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5723 , TFLOPS: 67.24341609238434, Tokens per sec: 166636.59255071447, Loss: 3.4946658611297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5724 , TFLOPS: 68.67526200347952, Tokens per sec: 170184.86444925648, Loss: 3.5035207271575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5725 , TFLOPS: 70.55236263012542, Tokens per sec: 174836.52658178913, Loss: 3.499228000640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5726 , TFLOPS: 68.00901282059287, Tokens per sec: 168533.82558065708, Loss: 3.5099313259124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5727 , TFLOPS: 68.33151897754993, Tokens per sec: 169333.0313645827, Loss: 3.499589443206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5728 , TFLOPS: 69.51754291840032, Tokens per sec: 172272.12934133262, Loss: 3.5108723640441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5729 , TFLOPS: 69.82664872805691, Tokens per sec: 173038.12758847754, Loss: 3.4970011711120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5730 , TFLOPS: 67.330929856916, Tokens per sec: 166853.4612996609, Loss: 3.497077465057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5731 , TFLOPS: 69.4865448162689, Tokens per sec: 172195.3126295281, Loss: 3.4889719486236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5732 , TFLOPS: 70.60322163793398, Tokens per sec: 174962.560805721, Loss: 3.5123095512390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5733 , TFLOPS: 68.32844991222699, Tokens per sec: 169325.42588262697, Loss: 3.5087523460388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5734 , TFLOPS: 68.84278364635435, Tokens per sec: 170600.00153433267, Loss: 3.5000076293945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5735 , TFLOPS: 69.65108575250545, Tokens per sec: 172603.0631952027, Loss: 3.492452383041382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5736 , TFLOPS: 68.90748955955888, Tokens per sec: 170760.34991519828, Loss: 3.4495251178741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5737 , TFLOPS: 67.87240579910414, Tokens per sec: 168195.29833290228, Loss: 3.50942063331604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5738 , TFLOPS: 69.54343660830656, Tokens per sec: 172336.2967573457, Loss: 3.4937033653259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5739 , TFLOPS: 68.89871603457617, Tokens per sec: 170738.60815380755, Loss: 3.4997568130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5740 , TFLOPS: 68.93482352251334, Tokens per sec: 170828.08648648404, Loss: 3.497422218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5741 , TFLOPS: 69.68067409981678, Tokens per sec: 172676.38637926572, Loss: 3.4878673553466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5742 , TFLOPS: 69.63445511025151, Tokens per sec: 172561.85063168095, Loss: 3.497908115386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5743 , TFLOPS: 67.23314260082604, Tokens per sec: 166611.1337069145, Loss: 3.5018436908721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5744 , TFLOPS: 68.77612946601658, Tokens per sec: 170434.8251328916, Loss: 3.4834017753601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5745 , TFLOPS: 70.5965982976461, Tokens per sec: 174946.14743886644, Loss: 3.4879889488220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5746 , TFLOPS: 67.8806036085784, Tokens per sec: 168215.6134078428, Loss: 3.484743595123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5747 , TFLOPS: 67.51908699900753, Tokens per sec: 167319.735424984, Loss: 3.505561351776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5748 , TFLOPS: 70.59107254144607, Tokens per sec: 174932.4540062905, Loss: 3.4870264530181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5749 , TFLOPS: 69.8249696912549, Tokens per sec: 173033.966750321, Loss: 3.5013198852539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5750 , TFLOPS: 67.82328118937868, Tokens per sec: 168073.5621384801, Loss: 3.497739315032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5751 , TFLOPS: 69.98581364173621, Tokens per sec: 173432.5557780375, Loss: 3.4757368564605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5752 , TFLOPS: 70.5585019316132, Tokens per sec: 174851.74044717578, Loss: 3.491184949874878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5753 , TFLOPS: 67.24752632553263, Tokens per sec: 166646.7781612355, Loss: 3.50425124168396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5754 , TFLOPS: 68.76075970732218, Tokens per sec: 170396.73717772815, Loss: 3.4987874031066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5755 , TFLOPS: 70.55453113173189, Tokens per sec: 174841.90036765017, Loss: 3.4922702312469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5756 , TFLOPS: 68.93617224249498, Tokens per sec: 170831.42876317227, Loss: 3.491128444671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5757 , TFLOPS: 66.97935424330248, Tokens per sec: 165982.21819987564, Loss: 3.504070281982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5758 , TFLOPS: 70.56396916995776, Tokens per sec: 174865.28886605962, Loss: 3.47336483001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5759 , TFLOPS: 68.8561441666335, Tokens per sec: 170633.11037536725, Loss: 3.4841561317443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5760 , TFLOPS: 69.79856921957551, Tokens per sec: 172968.54347324744, Loss: 3.507826328277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5761 , TFLOPS: 68.61499817432394, Tokens per sec: 170035.52404200015, Loss: 3.4736828804016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5762 , TFLOPS: 70.57948170781177, Tokens per sec: 174903.73064370896, Loss: 3.498988389968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5763 , TFLOPS: 67.96386564045078, Tokens per sec: 168421.9458948925, Loss: 3.520275592803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5764 , TFLOPS: 67.19875637431208, Tokens per sec: 166525.92084965736, Loss: 3.5003719329833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5765 , TFLOPS: 70.55147450252414, Tokens per sec: 174834.32570375787, Loss: 3.4478695392608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5766 , TFLOPS: 68.89636152487547, Tokens per sec: 170732.77341939855, Loss: 3.490269184112549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5767 , TFLOPS: 68.04583746273661, Tokens per sec: 168625.08110045848, Loss: 3.4932851791381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5768 , TFLOPS: 69.03630702194528, Tokens per sec: 171079.5737774067, Loss: 3.5018296241760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5769 , TFLOPS: 70.55486486981712, Tokens per sec: 174842.72740738967, Loss: 3.497040271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5770 , TFLOPS: 67.44315525452747, Tokens per sec: 167131.56819759027, Loss: 3.507058620452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5771 , TFLOPS: 69.59437180829804, Tokens per sec: 172462.51979390005, Loss: 3.5069756507873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5772 , TFLOPS: 69.53771546926293, Tokens per sec: 172322.11914456007, Loss: 3.503924608230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5773 , TFLOPS: 68.09147879535074, Tokens per sec: 168738.18535048413, Loss: 3.494898796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5774 , TFLOPS: 68.91670323954551, Tokens per sec: 170783.1824291758, Loss: 3.500084161758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5775 , TFLOPS: 69.59376259927511, Tokens per sec: 172461.0101068313, Loss: 3.4805073738098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5776 , TFLOPS: 68.85800257758491, Tokens per sec: 170637.7157224257, Loss: 3.491250514984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5777 , TFLOPS: 67.89396675753822, Tokens per sec: 168248.72876303762, Loss: 3.5179600715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5778 , TFLOPS: 70.57773864799464, Tokens per sec: 174899.41114946821, Loss: 3.4889869689941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5779 , TFLOPS: 68.72336595631795, Tokens per sec: 170304.07134347796, Loss: 3.4825329780578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5780 , TFLOPS: 69.19061749609469, Tokens per sec: 171461.97213105098, Loss: 3.506758451461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5781 , TFLOPS: 68.92545708290456, Tokens per sec: 170804.87541733496, Loss: 3.4993245601654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5782 , TFLOPS: 70.56541705114816, Tokens per sec: 174868.8768751467, Loss: 3.5179219245910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5783 , TFLOPS: 67.33830589347956, Tokens per sec: 166871.73993080296, Loss: 3.485489845275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5784 , TFLOPS: 67.89210049455707, Tokens per sec: 168244.10395778445, Loss: 3.510540246963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5785 , TFLOPS: 70.61212933971444, Tokens per sec: 174984.63507199733, Loss: 3.4985594749450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5786 , TFLOPS: 68.79305128332497, Tokens per sec: 170476.75926027383, Loss: 3.508258104324341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5787 , TFLOPS: 67.85198890215005, Tokens per sec: 168144.7030131432, Loss: 3.5059776306152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5788 , TFLOPS: 69.71937333935766, Tokens per sec: 172772.28735792116, Loss: 3.500014305114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5789 , TFLOPS: 70.6156315742604, Tokens per sec: 174993.31399500518, Loss: 3.496870517730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5790 , TFLOPS: 66.43459401548559, Tokens per sec: 164632.24234505228, Loss: 3.492521047592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5791 , TFLOPS: 69.67332779976492, Tokens per sec: 172658.1814384754, Loss: 3.5102734565734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5792 , TFLOPS: 70.58852236026293, Tokens per sec: 174926.13437639302, Loss: 3.5001296997070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5793 , TFLOPS: 68.09392483507897, Tokens per sec: 168744.24690638419, Loss: 3.5082504749298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5794 , TFLOPS: 68.7192517842148, Tokens per sec: 170293.87597179288, Loss: 3.501852512359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5795 , TFLOPS: 70.57611261683094, Tokens per sec: 174895.38166512246, Loss: 3.47210431098938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5796 , TFLOPS: 69.7455690476701, Tokens per sec: 172837.2032088149, Loss: 3.4784865379333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5797 , TFLOPS: 68.25097036335787, Tokens per sec: 169133.4230254524, Loss: 3.5056796073913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5798 , TFLOPS: 69.61386311341684, Tokens per sec: 172510.82139512897, Loss: 3.505368947982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5799 , TFLOPS: 69.6724967905396, Tokens per sec: 172656.12210607185, Loss: 3.4726197719573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5800 , TFLOPS: 68.89634161227866, Tokens per sec: 170732.72407379013, Loss: 3.5151546001434326 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/optimizer.pt +[rank0]:[2024-08-25 08:51:14,973] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006404701998690143, 'preprocessing_with_comm': 0.001288567997107748, 'state_converting': 0.9665857250001864, : 0.9756799189999583}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800, took 4.29s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0003800 + eval ppl=25.474849700927734, eval loss=3.237691640853882 +------------------------------------------------------------------ +iteration: 5801 , TFLOPS: 62.45731729075902, Tokens per sec: 154776.11248797973, Loss: 3.492762327194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5802 , TFLOPS: 70.53640001499743, Tokens per sec: 174796.96946307475, Loss: 3.492051601409912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5803 , TFLOPS: 68.94042575646277, Tokens per sec: 170841.96943934774, Loss: 3.4835891723632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5804 , TFLOPS: 69.11294151312465, Tokens per sec: 171269.48248853532, Loss: 3.4798266887664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5805 , TFLOPS: 67.9878775850619, Tokens per sec: 168481.4501387729, Loss: 3.5065088272094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5806 , TFLOPS: 68.8894466931992, Tokens per sec: 170715.6377047732, Loss: 3.5122556686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5807 , TFLOPS: 68.85855205398059, Tokens per sec: 170639.077385464, Loss: 3.4959194660186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5808 , TFLOPS: 68.85196376906204, Tokens per sec: 170622.75088967633, Loss: 3.4667463302612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5809 , TFLOPS: 69.43069326966442, Tokens per sec: 172056.90634448643, Loss: 3.50225567817688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5810 , TFLOPS: 67.54469319889255, Tokens per sec: 167383.1904090549, Loss: 3.5034878253936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5811 , TFLOPS: 68.61623927259082, Tokens per sec: 170038.59962022371, Loss: 3.519650936126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5812 , TFLOPS: 70.5442016386603, Tokens per sec: 174816.30274593114, Loss: 3.503324508666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5813 , TFLOPS: 68.50776703372219, Tokens per sec: 169769.79346310374, Loss: 3.493053913116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5814 , TFLOPS: 68.00601489210558, Tokens per sec: 168526.39638361646, Loss: 3.491987466812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5815 , TFLOPS: 70.55113644358873, Tokens per sec: 174833.48795647564, Loss: 3.493865489959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5816 , TFLOPS: 67.85241189691845, Tokens per sec: 168145.751240776, Loss: 3.5039150714874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5817 , TFLOPS: 68.75125645768854, Tokens per sec: 170373.18707826664, Loss: 3.4891984462738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5818 , TFLOPS: 68.39366979904128, Tokens per sec: 169487.04794671643, Loss: 3.4948017597198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5819 , TFLOPS: 70.56106230102176, Tokens per sec: 174858.08532461815, Loss: 3.507359504699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5820 , TFLOPS: 67.07251980133047, Tokens per sec: 166213.09271570243, Loss: 3.503105640411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5821 , TFLOPS: 68.88387408050636, Tokens per sec: 170701.82815664637, Loss: 3.485200881958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5822 , TFLOPS: 70.61826587937287, Tokens per sec: 174999.84209326655, Loss: 3.473255157470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5823 , TFLOPS: 68.1410927366344, Tokens per sec: 168861.13416241185, Loss: 3.489320993423462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5824 , TFLOPS: 68.65121366214863, Tokens per sec: 170125.27001029506, Loss: 3.4898953437805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5825 , TFLOPS: 68.66927995495169, Tokens per sec: 170170.04027402744, Loss: 3.4681358337402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5826 , TFLOPS: 69.74974123711578, Tokens per sec: 172847.5423538659, Loss: 3.4859516620635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5827 , TFLOPS: 68.34185731654107, Tokens per sec: 169358.6509074648, Loss: 3.497507333755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5828 , TFLOPS: 67.88224256211694, Tokens per sec: 168219.67491525694, Loss: 3.480984926223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5829 , TFLOPS: 69.76990131270463, Tokens per sec: 172897.50124199074, Loss: 3.5088486671447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5830 , TFLOPS: 68.89036322709015, Tokens per sec: 170717.90897671948, Loss: 3.5071723461151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5831 , TFLOPS: 66.4532414102151, Tokens per sec: 164678.4526433721, Loss: 3.452857732772827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5832 , TFLOPS: 70.55228572365743, Tokens per sec: 174836.33599908956, Loss: 3.492581605911255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5833 , TFLOPS: 68.89598267653739, Tokens per sec: 170731.8345914832, Loss: 3.4859726428985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5834 , TFLOPS: 68.89256223869637, Tokens per sec: 170723.35836972087, Loss: 3.4969403743743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5835 , TFLOPS: 68.91615356135372, Tokens per sec: 170781.8202660645, Loss: 3.5007760524749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5836 , TFLOPS: 68.89353059166314, Tokens per sec: 170725.75805504536, Loss: 3.4675047397613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5837 , TFLOPS: 68.88622057540242, Tokens per sec: 170707.64302948656, Loss: 3.500274896621704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5838 , TFLOPS: 68.74637607827574, Tokens per sec: 170361.09296046357, Loss: 3.4917709827423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5839 , TFLOPS: 69.61591160237613, Tokens per sec: 172515.89777642975, Loss: 3.4826042652130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5840 , TFLOPS: 67.41919718194845, Tokens per sec: 167072.19745453988, Loss: 3.499361991882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5841 , TFLOPS: 68.91939658684551, Tokens per sec: 170789.8568404819, Loss: 3.5025479793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5842 , TFLOPS: 70.5655454048763, Tokens per sec: 174869.19494982436, Loss: 3.509918212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5843 , TFLOPS: 67.94513063511445, Tokens per sec: 168375.5184878379, Loss: 3.4964160919189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5844 , TFLOPS: 68.76188207328013, Tokens per sec: 170399.5185242108, Loss: 3.469984292984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5845 , TFLOPS: 68.72497583802836, Tokens per sec: 170308.06080769945, Loss: 3.498671531677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5846 , TFLOPS: 68.72763730433667, Tokens per sec: 170314.65621439615, Loss: 3.4683609008789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5847 , TFLOPS: 68.17511421349153, Tokens per sec: 168945.44312983885, Loss: 3.4964075088500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5848 , TFLOPS: 68.77635803388651, Tokens per sec: 170435.3915492521, Loss: 3.5080344676971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5849 , TFLOPS: 70.5629061159937, Tokens per sec: 174862.6545012316, Loss: 3.497370719909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5850 , TFLOPS: 68.26936125073458, Tokens per sec: 169178.9976702954, Loss: 3.495760917663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5851 , TFLOPS: 68.66917029707592, Tokens per sec: 170169.76852973152, Loss: 3.4893441200256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5852 , TFLOPS: 69.83956258425891, Tokens per sec: 173070.12954672478, Loss: 3.48303484916687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5853 , TFLOPS: 68.88664882831102, Tokens per sec: 170708.70428737003, Loss: 3.498852491378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5854 , TFLOPS: 69.62703958378238, Tokens per sec: 172543.4741402028, Loss: 3.4918696880340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5855 , TFLOPS: 69.15648363125206, Tokens per sec: 171377.38465381527, Loss: 3.503185749053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5856 , TFLOPS: 68.82995316980723, Tokens per sec: 170568.20620005525, Loss: 3.489258050918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5857 , TFLOPS: 68.70855284562943, Tokens per sec: 170267.36282339293, Loss: 3.5076231956481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5858 , TFLOPS: 68.72284257790076, Tokens per sec: 170302.77435410526, Loss: 3.511110305786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5859 , TFLOPS: 70.54580182538618, Tokens per sec: 174820.268184913, Loss: 3.504293918609619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5860 , TFLOPS: 67.68449989562623, Tokens per sec: 167729.64680453998, Loss: 3.4819858074188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5861 , TFLOPS: 67.42288501437508, Tokens per sec: 167081.33630954137, Loss: 3.5220000743865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5862 , TFLOPS: 70.51999050656237, Tokens per sec: 174756.3048935161, Loss: 3.4918227195739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5863 , TFLOPS: 68.85878156405533, Tokens per sec: 170639.64613671185, Loss: 3.5195729732513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5864 , TFLOPS: 68.90629660869902, Tokens per sec: 170757.39365155317, Loss: 3.481306552886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5865 , TFLOPS: 67.4860322313245, Tokens per sec: 167237.8220693822, Loss: 3.481454610824585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5866 , TFLOPS: 70.59732551273672, Tokens per sec: 174947.94955797037, Loss: 3.477760076522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5867 , TFLOPS: 67.08991192499421, Tokens per sec: 166256.19231404175, Loss: 3.500727415084839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5868 , TFLOPS: 68.79931973803667, Tokens per sec: 170492.29318157674, Loss: 3.476977825164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5869 , TFLOPS: 70.56705664162055, Tokens per sec: 174872.9399609531, Loss: 3.5326170921325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5870 , TFLOPS: 68.14815590538211, Tokens per sec: 168878.63747264096, Loss: 3.4954240322113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5871 , TFLOPS: 69.6927180084842, Tokens per sec: 172706.23251168962, Loss: 3.4826581478118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5872 , TFLOPS: 67.97041667088983, Tokens per sec: 168438.18006997608, Loss: 3.4947385787963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5873 , TFLOPS: 68.90669862797148, Tokens per sec: 170758.3898995964, Loss: 3.482532262802124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5874 , TFLOPS: 69.63642399918317, Tokens per sec: 172566.72975534463, Loss: 3.489737033843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5875 , TFLOPS: 68.87455856542583, Tokens per sec: 170678.7432840881, Loss: 3.5039610862731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5876 , TFLOPS: 68.82857847519462, Tokens per sec: 170564.79955537006, Loss: 3.50122332572937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5877 , TFLOPS: 68.32575715162965, Tokens per sec: 169318.75292523467, Loss: 3.482830047607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5878 , TFLOPS: 68.69878923732277, Tokens per sec: 170243.1675264588, Loss: 3.5195858478546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5879 , TFLOPS: 69.56471869998511, Tokens per sec: 172389.03612494105, Loss: 3.498983383178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5880 , TFLOPS: 68.765596685261, Tokens per sec: 170408.72374189767, Loss: 3.5015711784362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5881 , TFLOPS: 67.50130247070875, Tokens per sec: 167275.66340473213, Loss: 3.500135660171509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5882 , TFLOPS: 70.55800125048634, Tokens per sec: 174850.49970418884, Loss: 3.4911153316497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5883 , TFLOPS: 68.09955513453082, Tokens per sec: 168758.19940865456, Loss: 3.4964394569396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5884 , TFLOPS: 68.84194873508783, Tokens per sec: 170597.93253224116, Loss: 3.496796131134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5885 , TFLOPS: 68.80015032545019, Tokens per sec: 170494.35146868418, Loss: 3.4855334758758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5886 , TFLOPS: 70.5125217143874, Tokens per sec: 174737.79640375118, Loss: 3.4968128204345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5887 , TFLOPS: 66.1495469774826, Tokens per sec: 163925.86438435744, Loss: 3.4998137950897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5888 , TFLOPS: 68.84746829762052, Tokens per sec: 170611.61061622718, Loss: 3.4961323738098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5889 , TFLOPS: 70.5754571883789, Tokens per sec: 174893.7574412169, Loss: 3.4877333641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5890 , TFLOPS: 67.92886376921494, Tokens per sec: 168335.2074021957, Loss: 3.5008926391601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5891 , TFLOPS: 69.19279531459343, Tokens per sec: 171467.36900519734, Loss: 3.506891965866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5892 , TFLOPS: 68.67782298663435, Tokens per sec: 170191.21084762973, Loss: 3.52311372756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5893 , TFLOPS: 69.68540488506987, Tokens per sec: 172688.1097862619, Loss: 3.5042736530303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5894 , TFLOPS: 68.24314144656778, Tokens per sec: 169114.0221072202, Loss: 3.4670472145080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5895 , TFLOPS: 68.77952697728236, Tokens per sec: 170443.24454007496, Loss: 3.485907793045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5896 , TFLOPS: 70.5332265769381, Tokens per sec: 174789.10533398073, Loss: 3.4891932010650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5897 , TFLOPS: 68.28427176398239, Tokens per sec: 169215.94756465207, Loss: 3.4749605655670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5898 , TFLOPS: 67.19805495567203, Tokens per sec: 166524.18265699965, Loss: 3.5031185150146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5899 , TFLOPS: 70.53440815363592, Tokens per sec: 174792.03341119934, Loss: 3.4961235523223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5900 , TFLOPS: 68.88566354989965, Tokens per sec: 170706.26265892925, Loss: 3.48590350151062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5901 , TFLOPS: 68.16471439541229, Tokens per sec: 168919.67123500397, Loss: 3.5001561641693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5902 , TFLOPS: 69.01667043032873, Tokens per sec: 171030.91214023175, Loss: 3.4891304969787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5903 , TFLOPS: 68.89178692509782, Tokens per sec: 170721.43705721578, Loss: 3.496865749359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5904 , TFLOPS: 68.80121234834074, Tokens per sec: 170496.98327839846, Loss: 3.5034945011138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5905 , TFLOPS: 68.43836234886035, Tokens per sec: 169597.80100845886, Loss: 3.4913129806518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5906 , TFLOPS: 70.53799723517562, Tokens per sec: 174800.92755062468, Loss: 3.5125598907470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5907 , TFLOPS: 67.40240145448769, Tokens per sec: 167030.57579169, Loss: 3.4805662631988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5908 , TFLOPS: 68.75633461462068, Tokens per sec: 170385.77131054967, Loss: 3.5237197875976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5909 , TFLOPS: 69.47506662271954, Tokens per sec: 172166.86840148535, Loss: 3.523235321044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5910 , TFLOPS: 68.993266075952, Tokens per sec: 170972.91357188372, Loss: 3.492217540740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5911 , TFLOPS: 68.80962758739761, Tokens per sec: 170517.83716779648, Loss: 3.4798221588134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5912 , TFLOPS: 68.72732746255956, Tokens per sec: 170313.8883923406, Loss: 3.504844903945923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5913 , TFLOPS: 68.64695402412775, Tokens per sec: 170114.71415804143, Loss: 3.4976556301116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5914 , TFLOPS: 68.82860172199459, Tokens per sec: 170564.85716350126, Loss: 3.502577304840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5915 , TFLOPS: 68.84328496981959, Tokens per sec: 170601.2438691049, Loss: 3.5179197788238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5916 , TFLOPS: 70.58877326953059, Tokens per sec: 174926.75615719825, Loss: 3.5080416202545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5917 , TFLOPS: 66.31630791390901, Tokens per sec: 164339.11635505574, Loss: 3.498228073120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5918 , TFLOPS: 68.84570795587885, Tokens per sec: 170607.2482954737, Loss: 3.5272765159606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5919 , TFLOPS: 69.83658752357277, Tokens per sec: 173062.75701861453, Loss: 3.4770684242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5920 , TFLOPS: 67.90722672049581, Tokens per sec: 168281.58841195202, Loss: 3.4827723503112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5921 , TFLOPS: 68.79812192930714, Tokens per sec: 170489.32487959543, Loss: 3.4964849948883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5922 , TFLOPS: 69.17023496213801, Tokens per sec: 171411.46196658481, Loss: 3.483293056488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5923 , TFLOPS: 68.96994791318501, Tokens per sec: 170915.12859583943, Loss: 3.4789555072784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5924 , TFLOPS: 67.98465843526097, Tokens per sec: 168473.4727309476, Loss: 3.4830105304718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5925 , TFLOPS: 68.7974381653273, Tokens per sec: 170487.63043713057, Loss: 3.4948716163635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5926 , TFLOPS: 70.62111757983448, Tokens per sec: 175006.90892115116, Loss: 3.5220019817352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5927 , TFLOPS: 67.1633182497404, Tokens per sec: 166438.101273137, Loss: 3.523729085922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5928 , TFLOPS: 68.23734681989748, Tokens per sec: 169099.66238400034, Loss: 3.4982810020446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5929 , TFLOPS: 70.53353300387141, Tokens per sec: 174789.86469367717, Loss: 3.503763198852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5930 , TFLOPS: 68.94482877966603, Tokens per sec: 170852.88061587844, Loss: 3.4797067642211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5931 , TFLOPS: 68.00853417943273, Tokens per sec: 168532.63945513556, Loss: 3.4959158897399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5932 , TFLOPS: 68.78067480561018, Tokens per sec: 170446.08898511503, Loss: 3.486733913421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5933 , TFLOPS: 70.52012730867172, Tokens per sec: 174756.6439042131, Loss: 3.4725656509399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5934 , TFLOPS: 68.20081710985411, Tokens per sec: 169009.1377384331, Loss: 3.497208595275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5935 , TFLOPS: 67.71458970254615, Tokens per sec: 167804.2126607534, Loss: 3.487771987915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5936 , TFLOPS: 70.54636450616314, Tokens per sec: 174821.66256986276, Loss: 3.4993250370025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5937 , TFLOPS: 68.17941978275654, Tokens per sec: 168956.11280477518, Loss: 3.5213804244995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5938 , TFLOPS: 69.48499045283106, Tokens per sec: 172191.46074570183, Loss: 3.525773763656616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5939 , TFLOPS: 68.85268609077187, Tokens per sec: 170624.5408824436, Loss: 3.4770212173461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5940 , TFLOPS: 68.74347768673117, Tokens per sec: 170353.91042693192, Loss: 3.504000425338745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5941 , TFLOPS: 68.88395484930938, Tokens per sec: 170702.02831063766, Loss: 3.523754835128784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5942 , TFLOPS: 67.75810598767764, Tokens per sec: 167912.05080902376, Loss: 3.4898698329925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5943 , TFLOPS: 69.86709744305355, Tokens per sec: 173138.363960033, Loss: 3.498528480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5944 , TFLOPS: 68.493857327879, Tokens per sec: 169735.32367974368, Loss: 3.515981674194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5945 , TFLOPS: 68.73961642743318, Tokens per sec: 170344.34180103868, Loss: 3.48063063621521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5946 , TFLOPS: 69.54128743885002, Tokens per sec: 172330.9708786818, Loss: 3.5026779174804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5947 , TFLOPS: 68.7805543550712, Tokens per sec: 170445.7904954107, Loss: 3.5239510536193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5948 , TFLOPS: 68.1472814871664, Tokens per sec: 168876.4705679773, Loss: 3.484984874725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5949 , TFLOPS: 69.980209249377, Tokens per sec: 173418.6674763966, Loss: 3.4870927333831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5950 , TFLOPS: 68.00707122462426, Tokens per sec: 168529.0140919624, Loss: 3.486637592315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5951 , TFLOPS: 68.89348151809234, Tokens per sec: 170725.63644533104, Loss: 3.4818484783172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5952 , TFLOPS: 68.76751996234962, Tokens per sec: 170413.48983438974, Loss: 3.495222330093384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5953 , TFLOPS: 69.67889717527642, Tokens per sec: 172671.98296451033, Loss: 3.501469612121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5954 , TFLOPS: 67.9624106961992, Tokens per sec: 168418.34038276176, Loss: 3.52787446975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5955 , TFLOPS: 68.81328954587796, Tokens per sec: 170526.9119043088, Loss: 3.515436887741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5956 , TFLOPS: 70.58290750470913, Tokens per sec: 174912.2201458033, Loss: 3.481295347213745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5957 , TFLOPS: 67.90923276520773, Tokens per sec: 168286.5596117322, Loss: 3.4883408546447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5958 , TFLOPS: 69.80552043814352, Tokens per sec: 172985.76935859886, Loss: 3.5034573078155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5959 , TFLOPS: 68.79013565505103, Tokens per sec: 170469.53401222805, Loss: 3.473345994949341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5960 , TFLOPS: 69.13083143081039, Tokens per sec: 171313.8156752977, Loss: 3.503399610519409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5961 , TFLOPS: 67.6688797065017, Tokens per sec: 167690.9382551835, Loss: 3.502077579498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5962 , TFLOPS: 68.75434201580228, Tokens per sec: 170380.83343117477, Loss: 3.513124465942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5963 , TFLOPS: 69.79705798861177, Tokens per sec: 172964.7984764434, Loss: 3.4981608390808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5964 , TFLOPS: 67.97262159889625, Tokens per sec: 168443.64412446888, Loss: 3.496305465698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5965 , TFLOPS: 68.40001730106037, Tokens per sec: 169502.7777559548, Loss: 3.525041103363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5966 , TFLOPS: 70.58473726372101, Tokens per sec: 174916.75449019932, Loss: 3.5040595531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5967 , TFLOPS: 68.86599426357873, Tokens per sec: 170657.5200006753, Loss: 3.503204822540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5968 , TFLOPS: 67.95950305345099, Tokens per sec: 168411.1349237284, Loss: 3.464653491973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5969 , TFLOPS: 68.93285022769757, Tokens per sec: 170823.19644455417, Loss: 3.536937952041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5970 , TFLOPS: 68.98661302077832, Tokens per sec: 170956.42656826883, Loss: 3.49479341506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5971 , TFLOPS: 68.93486521426401, Tokens per sec: 170828.1898032349, Loss: 3.528158187866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5972 , TFLOPS: 67.36179527679514, Tokens per sec: 166929.9492102281, Loss: 3.486816883087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5973 , TFLOPS: 70.55127050847554, Tokens per sec: 174833.82018403363, Loss: 3.496006488800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5974 , TFLOPS: 67.93120549529789, Tokens per sec: 168341.01045739165, Loss: 3.513503074645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5975 , TFLOPS: 68.07600922595395, Tokens per sec: 168699.85005340513, Loss: 3.5075490474700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5976 , TFLOPS: 69.64236471626961, Tokens per sec: 172581.45150670884, Loss: 3.514801502227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5977 , TFLOPS: 69.0452132780379, Tokens per sec: 171101.64446110933, Loss: 3.483133554458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5978 , TFLOPS: 69.20919314846097, Tokens per sec: 171508.00464389162, Loss: 3.469541311264038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5979 , TFLOPS: 67.79643555422788, Tokens per sec: 168007.03569728558, Loss: 3.506627082824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5980 , TFLOPS: 69.71188032777226, Tokens per sec: 172753.71884978918, Loss: 3.480858325958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5981 , TFLOPS: 68.02253218092577, Tokens per sec: 168567.32804484243, Loss: 3.4878475666046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5982 , TFLOPS: 68.81861886074961, Tokens per sec: 170540.11853363324, Loss: 3.4921112060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5983 , TFLOPS: 69.68634185872598, Tokens per sec: 172690.4317101979, Loss: 3.5115997791290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5984 , TFLOPS: 68.38445569471331, Tokens per sec: 169464.21438117436, Loss: 3.4975509643554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5985 , TFLOPS: 68.92131197713935, Tokens per sec: 170794.6033886277, Loss: 3.5036702156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5986 , TFLOPS: 69.84399941839484, Tokens per sec: 173081.12451047127, Loss: 3.509981632232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5987 , TFLOPS: 67.79416345214676, Tokens per sec: 168001.40517803584, Loss: 3.4932496547698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5988 , TFLOPS: 68.94269351118207, Tokens per sec: 170847.58918535596, Loss: 3.4738519191741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5989 , TFLOPS: 68.743299446783, Tokens per sec: 170353.4687287056, Loss: 3.532635450363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5990 , TFLOPS: 69.77160737376951, Tokens per sec: 172901.72904924766, Loss: 3.4993507862091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5991 , TFLOPS: 67.86784043963729, Tokens per sec: 168183.9848692276, Loss: 3.50419545173645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5992 , TFLOPS: 68.78972981978909, Tokens per sec: 170468.52830774253, Loss: 3.507681131362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5993 , TFLOPS: 70.58002601354262, Tokens per sec: 174905.07949327023, Loss: 3.471989393234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5994 , TFLOPS: 67.87797557502115, Tokens per sec: 168209.10085118611, Loss: 3.505784273147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5995 , TFLOPS: 68.20890901298206, Tokens per sec: 169029.1903657809, Loss: 3.4856410026550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5996 , TFLOPS: 70.5993848147359, Tokens per sec: 174953.05273517518, Loss: 3.465862512588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5997 , TFLOPS: 68.96029582943662, Tokens per sec: 170891.20966904613, Loss: 3.488449811935425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5998 , TFLOPS: 67.97118907318716, Tokens per sec: 168440.094167956, Loss: 3.5058741569519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5999 , TFLOPS: 68.80984693409754, Tokens per sec: 170518.38073308193, Loss: 3.490894079208374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6000 , TFLOPS: 70.53997325600592, Tokens per sec: 174805.82434791882, Loss: 3.504162073135376 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/optimizer.pt +[rank0]:[2024-08-25 09:16:59,488] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063918390005710535, 'preprocessing_with_comm': 0.0013649679967784323, 'state_converting': 0.9918018820026191, : 1.0010017349995906}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004000 + eval ppl=24.655515670776367, eval loss=3.205000638961792 +------------------------------------------------------------------ +iteration: 6001 , TFLOPS: 62.43501268153882, Tokens per sec: 154720.83920927654, Loss: 3.4662721157073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6002 , TFLOPS: 69.52305486882953, Tokens per sec: 172285.78856168766, Loss: 3.4995224475860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6003 , TFLOPS: 70.57683359569278, Tokens per sec: 174897.16833016448, Loss: 3.494412422180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6004 , TFLOPS: 68.85049838070262, Tokens per sec: 170619.11949589584, Loss: 3.5018131732940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6005 , TFLOPS: 67.01957091448965, Tokens per sec: 166081.8795413095, Loss: 3.4944252967834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6006 , TFLOPS: 70.5631437519135, Tokens per sec: 174863.24338921864, Loss: 3.5083565711975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6007 , TFLOPS: 69.54984573647158, Tokens per sec: 172352.17928296234, Loss: 3.4890308380126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6008 , TFLOPS: 69.68765994568658, Tokens per sec: 172693.69807488553, Loss: 3.512131690979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6009 , TFLOPS: 68.92013507376289, Tokens per sec: 170791.68689241423, Loss: 3.4819695949554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6010 , TFLOPS: 69.46563341556026, Tokens per sec: 172143.49187499113, Loss: 3.488884925842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6011 , TFLOPS: 68.96819019714634, Tokens per sec: 170910.7727818672, Loss: 3.492044687271118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6012 , TFLOPS: 68.5329119334048, Tokens per sec: 169832.1052944566, Loss: 3.505385637283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6013 , TFLOPS: 70.60842411667284, Tokens per sec: 174975.4531211362, Loss: 3.514570474624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6014 , TFLOPS: 68.84594531224576, Tokens per sec: 170607.83649069784, Loss: 3.483790397644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6015 , TFLOPS: 68.3184539784979, Tokens per sec: 169300.65485770328, Loss: 3.495882987976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6016 , TFLOPS: 69.60029026431987, Tokens per sec: 172477.1863798939, Loss: 3.4844791889190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6017 , TFLOPS: 69.53256973919692, Tokens per sec: 172309.36745860754, Loss: 3.5041966438293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6018 , TFLOPS: 68.89379815800257, Tokens per sec: 170726.42111390937, Loss: 3.490415096282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6019 , TFLOPS: 68.58538151127301, Tokens per sec: 169962.13068841517, Loss: 3.507479429244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6020 , TFLOPS: 69.50300205446982, Tokens per sec: 172236.09547870443, Loss: 3.516859292984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6021 , TFLOPS: 68.29255876776628, Tokens per sec: 169236.4836729174, Loss: 3.4829392433166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6022 , TFLOPS: 68.8322804661424, Tokens per sec: 170573.97349674662, Loss: 3.490277051925659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6023 , TFLOPS: 69.55406835516645, Tokens per sec: 172362.6433972481, Loss: 3.510706663131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6024 , TFLOPS: 67.89751741211668, Tokens per sec: 168257.5276762199, Loss: 3.5030174255371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6025 , TFLOPS: 68.72655710371563, Tokens per sec: 170311.9793582633, Loss: 3.525351047515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6026 , TFLOPS: 70.01564202177512, Tokens per sec: 173506.47378964126, Loss: 3.504910469055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6027 , TFLOPS: 68.61644567601212, Tokens per sec: 170039.1111106389, Loss: 3.494072437286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6028 , TFLOPS: 69.66150413660259, Tokens per sec: 172628.88109867516, Loss: 3.4862709045410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6029 , TFLOPS: 69.60319541967083, Tokens per sec: 172484.38567488285, Loss: 3.4940218925476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6030 , TFLOPS: 69.55900333485341, Tokens per sec: 172374.87282054525, Loss: 3.49564528465271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6031 , TFLOPS: 67.89145415744034, Tokens per sec: 168242.5022632093, Loss: 3.5038437843322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6032 , TFLOPS: 68.77569582296148, Tokens per sec: 170433.7505176313, Loss: 3.5012052059173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6033 , TFLOPS: 70.62976083437196, Tokens per sec: 175028.32785802815, Loss: 3.495293617248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6034 , TFLOPS: 68.71324272677683, Tokens per sec: 170278.98486550926, Loss: 3.499809741973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6035 , TFLOPS: 67.427217103616, Tokens per sec: 167092.0717038994, Loss: 3.4809823036193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6036 , TFLOPS: 70.58542483230198, Tokens per sec: 174918.4583608849, Loss: 3.484127998352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6037 , TFLOPS: 69.59054452731326, Tokens per sec: 172453.03537000992, Loss: 3.5041677951812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6038 , TFLOPS: 68.06561816298905, Tokens per sec: 168674.09985470804, Loss: 3.4913594722747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6039 , TFLOPS: 68.80731845565796, Tokens per sec: 170512.11488497377, Loss: 3.5186452865600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6040 , TFLOPS: 70.50387471808841, Tokens per sec: 174716.368194377, Loss: 3.504453659057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6041 , TFLOPS: 67.8831786563735, Tokens per sec: 168221.99465994394, Loss: 3.5485761165618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6042 , TFLOPS: 67.77522738956496, Tokens per sec: 167954.47952897864, Loss: 3.4985692501068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6043 , TFLOPS: 70.59625749042037, Tokens per sec: 174945.30288101808, Loss: 3.5143895149230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6044 , TFLOPS: 69.60296422371722, Tokens per sec: 172483.81274584148, Loss: 3.508551836013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6045 , TFLOPS: 69.23411581059632, Tokens per sec: 171569.76574612092, Loss: 3.4984705448150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6046 , TFLOPS: 68.10138109327596, Tokens per sec: 168762.724335571, Loss: 3.4942288398742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6047 , TFLOPS: 69.67046899992447, Tokens per sec: 172651.097017548, Loss: 3.4881229400634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6048 , TFLOPS: 69.69755981740218, Tokens per sec: 172718.23104755767, Loss: 3.485691785812378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6049 , TFLOPS: 68.71647791180696, Tokens per sec: 170287.00201039898, Loss: 3.510204792022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6050 , TFLOPS: 70.56265790157572, Tokens per sec: 174862.03939856077, Loss: 3.4911270141601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6051 , TFLOPS: 69.03015142247183, Tokens per sec: 171064.31952381623, Loss: 3.4925646781921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6052 , TFLOPS: 67.87570386078094, Tokens per sec: 168203.4712930489, Loss: 3.4920082092285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6053 , TFLOPS: 69.59354721885683, Tokens per sec: 172460.4763704287, Loss: 3.485250949859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6054 , TFLOPS: 69.07856846941598, Tokens per sec: 171184.30229972186, Loss: 3.495739698410034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6055 , TFLOPS: 67.10751576127484, Tokens per sec: 166299.8165595678, Loss: 3.5185015201568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6056 , TFLOPS: 70.54844640607583, Tokens per sec: 174826.8217471866, Loss: 3.4908881187438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6057 , TFLOPS: 68.81813522339273, Tokens per sec: 170538.92002698587, Loss: 3.4911701679229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6058 , TFLOPS: 69.03451100482283, Tokens per sec: 171075.12304912964, Loss: 3.4855685234069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6059 , TFLOPS: 68.81774786933126, Tokens per sec: 170537.96012095097, Loss: 3.497648000717163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6060 , TFLOPS: 70.55791906595644, Tokens per sec: 174850.29604187034, Loss: 3.477294921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6061 , TFLOPS: 66.3288112502294, Tokens per sec: 164370.1009998139, Loss: 3.490356683731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6062 , TFLOPS: 68.71305342487109, Tokens per sec: 170278.51575453352, Loss: 3.52451491355896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6063 , TFLOPS: 70.53964826770232, Tokens per sec: 174805.0189911021, Loss: 3.4971985816955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6064 , TFLOPS: 68.63455040291521, Tokens per sec: 170083.97661830508, Loss: 3.517086982727051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6065 , TFLOPS: 69.02516546802896, Tokens per sec: 171051.96377945857, Loss: 3.494368553161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6066 , TFLOPS: 68.74508326214087, Tokens per sec: 170357.88921966587, Loss: 3.505082845687866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6067 , TFLOPS: 69.56221865488179, Tokens per sec: 172382.84073777235, Loss: 3.4965949058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6068 , TFLOPS: 68.7008257854403, Tokens per sec: 170248.21431703836, Loss: 3.4783427715301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6069 , TFLOPS: 69.57977499598319, Tokens per sec: 172426.3472850122, Loss: 3.480741262435913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6070 , TFLOPS: 70.56390667471283, Tokens per sec: 174865.13399595823, Loss: 3.531757116317749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6071 , TFLOPS: 67.98832604106892, Tokens per sec: 168482.56146215982, Loss: 3.5048229694366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6072 , TFLOPS: 68.0372778857932, Tokens per sec: 168603.86952587747, Loss: 3.500631809234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6073 , TFLOPS: 70.64399399897518, Tokens per sec: 175063.59920782744, Loss: 3.489063262939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6074 , TFLOPS: 69.55816836103389, Tokens per sec: 172372.80366344054, Loss: 3.476198196411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6075 , TFLOPS: 67.06887483508974, Tokens per sec: 166204.06008783172, Loss: 3.4579451084136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6076 , TFLOPS: 69.89195740131598, Tokens per sec: 173199.96967515777, Loss: 3.495314598083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6077 , TFLOPS: 69.71530396264558, Tokens per sec: 172762.2029941498, Loss: 3.4852261543273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6078 , TFLOPS: 69.02703435497706, Tokens per sec: 171056.59508719086, Loss: 3.4767043590545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6079 , TFLOPS: 67.79503381496207, Tokens per sec: 168003.56203297057, Loss: 3.51371431350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6080 , TFLOPS: 70.51226189014133, Tokens per sec: 174737.1525306474, Loss: 3.4993791580200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6081 , TFLOPS: 68.74265046635973, Tokens per sec: 170351.86048372564, Loss: 3.486250877380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6082 , TFLOPS: 68.25634898544533, Tokens per sec: 169146.7518434893, Loss: 3.481213092803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6083 , TFLOPS: 69.62161436897276, Tokens per sec: 172530.0298602678, Loss: 3.451540231704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6084 , TFLOPS: 69.56167879318723, Tokens per sec: 172381.50290102253, Loss: 3.475684404373169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6085 , TFLOPS: 68.96512199623756, Tokens per sec: 170903.169442025, Loss: 3.5033419132232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6086 , TFLOPS: 67.82148272807342, Tokens per sec: 168069.1053532488, Loss: 3.483771324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6087 , TFLOPS: 70.62483257657914, Tokens per sec: 175016.1150923251, Loss: 3.479893922805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6088 , TFLOPS: 68.96570284755431, Tokens per sec: 170904.608855574, Loss: 3.520331382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6089 , TFLOPS: 69.63134985391603, Tokens per sec: 172554.15546440944, Loss: 3.488645076751709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6090 , TFLOPS: 69.67328945519287, Tokens per sec: 172658.0864164019, Loss: 3.48764967918396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6091 , TFLOPS: 68.26560463096295, Tokens per sec: 169169.68835267544, Loss: 3.5019054412841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6092 , TFLOPS: 68.74158697508364, Tokens per sec: 170349.22503519026, Loss: 3.498866558074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6093 , TFLOPS: 69.9305371178214, Tokens per sec: 173295.57446256484, Loss: 3.480829954147339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6094 , TFLOPS: 68.58662048989466, Tokens per sec: 169965.20101392438, Loss: 3.504201650619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6095 , TFLOPS: 68.8058961989112, Tokens per sec: 170508.59037608077, Loss: 3.5009632110595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6096 , TFLOPS: 69.12408573861606, Tokens per sec: 171297.09910694076, Loss: 3.5046801567077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6097 , TFLOPS: 68.5824814603557, Tokens per sec: 169954.94404277508, Loss: 3.492567539215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6098 , TFLOPS: 68.92032152594825, Tokens per sec: 170792.14894146894, Loss: 3.496596097946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6099 , TFLOPS: 68.71558811704813, Tokens per sec: 170284.79700096764, Loss: 3.499119997024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6100 , TFLOPS: 70.60612676523748, Tokens per sec: 174969.76003120528, Loss: 3.5136730670928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6101 , TFLOPS: 69.56115352640222, Tokens per sec: 172380.20123206638, Loss: 3.4999749660491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6102 , TFLOPS: 67.54209348002534, Tokens per sec: 167376.74801931926, Loss: 3.4981629848480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6103 , TFLOPS: 70.55816011923113, Tokens per sec: 174850.89339843925, Loss: 3.486419677734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6104 , TFLOPS: 69.65540841209355, Tokens per sec: 172613.77522184228, Loss: 3.4766221046447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6105 , TFLOPS: 67.72233832540404, Tokens per sec: 167823.41460177652, Loss: 3.473491907119751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6106 , TFLOPS: 68.76374190123919, Tokens per sec: 170404.1273827703, Loss: 3.5105953216552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6107 , TFLOPS: 70.54105061017508, Tokens per sec: 174808.49415023034, Loss: 3.495485782623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6108 , TFLOPS: 68.15254538826005, Tokens per sec: 168889.51509475394, Loss: 3.5013513565063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6109 , TFLOPS: 69.6148455430236, Tokens per sec: 172513.25596391805, Loss: 3.5125246047973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6110 , TFLOPS: 70.59958159324219, Tokens per sec: 174953.54037399072, Loss: 3.5151045322418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6111 , TFLOPS: 68.87678834205396, Tokens per sec: 170684.2689162028, Loss: 3.5062925815582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6112 , TFLOPS: 68.28099424547173, Tokens per sec: 169207.82551273418, Loss: 3.491962194442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6113 , TFLOPS: 68.88151721547172, Tokens per sec: 170695.9875854604, Loss: 3.5125746726989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6114 , TFLOPS: 69.52433808191572, Tokens per sec: 172288.96850507281, Loss: 3.4825117588043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6115 , TFLOPS: 68.96511948987047, Tokens per sec: 170903.16323097132, Loss: 3.4759633541107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6116 , TFLOPS: 68.58321626276269, Tokens per sec: 169956.76496408478, Loss: 3.4792444705963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6117 , TFLOPS: 69.68436098329519, Tokens per sec: 172685.52288266897, Loss: 3.5114967823028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6118 , TFLOPS: 69.80770268626013, Tokens per sec: 172991.17720982633, Loss: 3.5049643516540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6119 , TFLOPS: 68.16263508981274, Tokens per sec: 168914.51848667391, Loss: 3.47316837310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6120 , TFLOPS: 66.75297649721124, Tokens per sec: 165421.2292074941, Loss: 3.4708876609802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6121 , TFLOPS: 69.5432628648271, Tokens per sec: 172335.86620186415, Loss: 3.504075050354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6122 , TFLOPS: 68.00346574385011, Tokens per sec: 168520.0793134276, Loss: 3.494269371032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6123 , TFLOPS: 69.04245189748536, Tokens per sec: 171094.80145590898, Loss: 3.5079028606414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6124 , TFLOPS: 69.64452047689296, Tokens per sec: 172586.79371901017, Loss: 3.503636598587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6125 , TFLOPS: 68.84451910378067, Tokens per sec: 170604.3021890115, Loss: 3.5057566165924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6126 , TFLOPS: 68.67346182080914, Tokens per sec: 170180.40339829278, Loss: 3.490013599395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6127 , TFLOPS: 69.52969902964894, Tokens per sec: 172302.25352411286, Loss: 3.5089352130889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6128 , TFLOPS: 68.73579243071426, Tokens per sec: 170334.86551592115, Loss: 3.518173933029175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6129 , TFLOPS: 69.62691447499235, Tokens per sec: 172543.16410683922, Loss: 3.486095905303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6130 , TFLOPS: 70.61353409473386, Tokens per sec: 174988.116209682, Loss: 3.4868860244750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6131 , TFLOPS: 68.73554784653332, Tokens per sec: 170334.25940937764, Loss: 3.5212697982788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6132 , TFLOPS: 69.07820855177978, Tokens per sec: 171183.41038417118, Loss: 3.5032150745391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6133 , TFLOPS: 68.45179169440425, Tokens per sec: 169631.08040608122, Loss: 3.494509696960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6134 , TFLOPS: 69.56823381544879, Tokens per sec: 172397.74696828198, Loss: 3.5049498081207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6135 , TFLOPS: 67.96572063461605, Tokens per sec: 168426.54277477853, Loss: 3.48344087600708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6136 , TFLOPS: 68.88387052247084, Tokens per sec: 170701.8193394424, Loss: 3.4808542728424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6137 , TFLOPS: 70.57219430142653, Tokens per sec: 174885.67164791227, Loss: 3.4876205921173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6138 , TFLOPS: 68.20202647141281, Tokens per sec: 169012.13466959776, Loss: 3.4776785373687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6139 , TFLOPS: 67.93941061085829, Tokens per sec: 168361.3436376184, Loss: 3.4854066371917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6140 , TFLOPS: 70.59911719510013, Tokens per sec: 174952.38954423703, Loss: 3.472365617752075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6141 , TFLOPS: 69.5516241712856, Tokens per sec: 172356.58644034222, Loss: 3.4784274101257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6142 , TFLOPS: 67.98150065885689, Tokens per sec: 168465.64741316048, Loss: 3.500697135925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6143 , TFLOPS: 69.66901795111698, Tokens per sec: 172647.50115875684, Loss: 3.467467784881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6144 , TFLOPS: 69.6675878002983, Tokens per sec: 172643.95708748404, Loss: 3.513979911804199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6145 , TFLOPS: 68.60818078715857, Tokens per sec: 170018.62980560717, Loss: 3.5128326416015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6146 , TFLOPS: 68.67541719261088, Tokens per sec: 170185.24902501947, Loss: 3.5009677410125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6147 , TFLOPS: 70.5741908187104, Tokens per sec: 174890.61923767623, Loss: 3.478877544403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6148 , TFLOPS: 69.59008549738907, Tokens per sec: 172451.8978432915, Loss: 3.500438690185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6149 , TFLOPS: 68.18447796028607, Tokens per sec: 168968.6475258976, Loss: 3.5074541568756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6150 , TFLOPS: 70.6163285232728, Tokens per sec: 174995.0411114332, Loss: 3.4809813499450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6151 , TFLOPS: 69.62656877294575, Tokens per sec: 172542.30741908547, Loss: 3.4878900051116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6152 , TFLOPS: 68.81304297885411, Tokens per sec: 170526.3008840621, Loss: 3.510131359100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6153 , TFLOPS: 68.57613141342098, Tokens per sec: 169939.20792695545, Loss: 3.508033514022827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6154 , TFLOPS: 70.5558900518625, Tokens per sec: 174845.26792143125, Loss: 3.5035200119018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6155 , TFLOPS: 68.09876454482848, Tokens per sec: 168756.24024028156, Loss: 3.495239019393921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6156 , TFLOPS: 68.86845446684839, Tokens per sec: 170663.61665539126, Loss: 3.4806065559387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6157 , TFLOPS: 69.59162061821836, Tokens per sec: 172455.70204181396, Loss: 3.512683391571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6158 , TFLOPS: 68.82885976463831, Tokens per sec: 170565.49662159814, Loss: 3.4827818870544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6159 , TFLOPS: 68.7445428967287, Tokens per sec: 170356.55013464837, Loss: 3.5048599243164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6160 , TFLOPS: 68.91167032519832, Tokens per sec: 170770.71031300214, Loss: 3.466282844543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6161 , TFLOPS: 69.55024683968468, Tokens per sec: 172353.17326090953, Loss: 3.469536781311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6162 , TFLOPS: 68.982795289775, Tokens per sec: 170946.9658102848, Loss: 3.5029358863830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6163 , TFLOPS: 69.15264381561103, Tokens per sec: 171367.8691676662, Loss: 3.5018584728240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6164 , TFLOPS: 68.71592284311151, Tokens per sec: 170285.6264890258, Loss: 3.495180130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6165 , TFLOPS: 68.92749637775026, Tokens per sec: 170809.9290146095, Loss: 3.4865827560424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6166 , TFLOPS: 68.66606326571447, Tokens per sec: 170162.06896375006, Loss: 3.506078004837036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6167 , TFLOPS: 70.57810099156221, Tokens per sec: 174900.30907674407, Loss: 3.496277093887329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6168 , TFLOPS: 68.59699868197173, Tokens per sec: 169990.9193171436, Loss: 3.4841690063476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6169 , TFLOPS: 68.66811699581437, Tokens per sec: 170167.15833317468, Loss: 3.4969077110290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6170 , TFLOPS: 70.60411351688502, Tokens per sec: 174964.77098001027, Loss: 3.4891555309295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6171 , TFLOPS: 68.75199241692218, Tokens per sec: 170375.0108663202, Loss: 3.492978811264038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6172 , TFLOPS: 68.96442019403416, Tokens per sec: 170901.43029885515, Loss: 3.4822049140930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6173 , TFLOPS: 68.63280982948035, Tokens per sec: 170079.6632855935, Loss: 3.517551898956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6174 , TFLOPS: 70.59354473378022, Tokens per sec: 174938.58037122895, Loss: 3.487483263015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6175 , TFLOPS: 66.99766034522933, Tokens per sec: 166027.58273703395, Loss: 3.512388229370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6176 , TFLOPS: 68.93163656437306, Tokens per sec: 170820.18885314, Loss: 3.465111017227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6177 , TFLOPS: 70.53797874000992, Tokens per sec: 174800.88171756663, Loss: 3.4956812858581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6178 , TFLOPS: 68.80413614935973, Tokens per sec: 170504.22877940678, Loss: 3.4711015224456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6179 , TFLOPS: 68.82080560089547, Tokens per sec: 170545.537516604, Loss: 3.5073556900024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6180 , TFLOPS: 69.02628810374826, Tokens per sec: 171054.74579443954, Loss: 3.515387535095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6181 , TFLOPS: 69.55952904440703, Tokens per sec: 172376.17558673082, Loss: 3.505164384841919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6182 , TFLOPS: 67.87389882445301, Tokens per sec: 168198.99821418713, Loss: 3.4855339527130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6183 , TFLOPS: 68.74928012421684, Tokens per sec: 170368.28950621266, Loss: 3.5030179023742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6184 , TFLOPS: 69.6565716393554, Tokens per sec: 172616.6578271372, Loss: 3.4855735301971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6185 , TFLOPS: 69.6600342961421, Tokens per sec: 172625.23867209777, Loss: 3.5388612747192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6186 , TFLOPS: 68.09999811005636, Tokens per sec: 168759.29715080463, Loss: 3.5013132095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6187 , TFLOPS: 70.58240305198441, Tokens per sec: 174910.97005638143, Loss: 3.508573532104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6188 , TFLOPS: 69.68106386483191, Tokens per sec: 172677.35225990938, Loss: 3.513822317123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6189 , TFLOPS: 68.61271249026032, Tokens per sec: 170029.85986510158, Loss: 3.4877986907958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6190 , TFLOPS: 69.62092422723269, Tokens per sec: 172528.3196130105, Loss: 3.531555652618408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6191 , TFLOPS: 69.57543667505148, Tokens per sec: 172415.59644783795, Loss: 3.5052599906921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6192 , TFLOPS: 69.01248945513153, Tokens per sec: 171020.55122312065, Loss: 3.4842722415924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6193 , TFLOPS: 68.84025972862587, Tokens per sec: 170593.746988169, Loss: 3.493122100830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6194 , TFLOPS: 69.65307409262041, Tokens per sec: 172607.99052104136, Loss: 3.5115771293640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6195 , TFLOPS: 68.00990755315445, Tokens per sec: 168536.0428265076, Loss: 3.502730369567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6196 , TFLOPS: 68.93786783477064, Tokens per sec: 170835.63062761628, Loss: 3.5150983333587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6197 , TFLOPS: 69.50881593981804, Tokens per sec: 172250.50292704962, Loss: 3.5088706016540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6198 , TFLOPS: 69.57088466051886, Tokens per sec: 172404.3160543803, Loss: 3.516746997833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6199 , TFLOPS: 69.21114906426831, Tokens per sec: 171512.85161872412, Loss: 3.5036444664001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6200 , TFLOPS: 68.73793935494183, Tokens per sec: 170340.18583066034, Loss: 3.5022530555725098 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/optimizer.pt +[rank0]:[2024-08-25 09:42:39,318] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006338228995446116, 'preprocessing_with_comm': 0.0012650979988393374, 'state_converting': 1.0045310970017454, : 1.0135336189996451}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200, took 4.37s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004200 + eval ppl=24.931936264038086, eval loss=3.2161495685577393 +------------------------------------------------------------------ +iteration: 6201 , TFLOPS: 63.58848811755885, Tokens per sec: 157579.278405543, Loss: 3.477971076965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6202 , TFLOPS: 68.72641367370055, Tokens per sec: 170311.6239228858, Loss: 3.5005857944488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6203 , TFLOPS: 68.79514302810185, Tokens per sec: 170481.94283425563, Loss: 3.5017073154449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6204 , TFLOPS: 69.48608710256842, Tokens per sec: 172194.17836455698, Loss: 3.4929637908935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6205 , TFLOPS: 68.64913973989857, Tokens per sec: 170120.13060249801, Loss: 3.5267174243927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6206 , TFLOPS: 69.61094317389372, Tokens per sec: 172503.58546333277, Loss: 3.5159146785736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6207 , TFLOPS: 69.82258296127011, Tokens per sec: 173028.05217049867, Loss: 3.4987566471099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6208 , TFLOPS: 68.5785005155201, Tokens per sec: 169945.0788229352, Loss: 3.494887590408325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6209 , TFLOPS: 68.23179493525308, Tokens per sec: 169085.90420226095, Loss: 3.499405860900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6210 , TFLOPS: 69.69322729175863, Tokens per sec: 172707.49457174586, Loss: 3.4745447635650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6211 , TFLOPS: 69.56154418550773, Tokens per sec: 172381.16932836449, Loss: 3.514241933822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6212 , TFLOPS: 67.81725748952539, Tokens per sec: 168058.63474667832, Loss: 3.507672071456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6213 , TFLOPS: 68.63833391094961, Tokens per sec: 170093.35256800216, Loss: 3.4897265434265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6214 , TFLOPS: 70.58717646880316, Tokens per sec: 174922.79910909355, Loss: 3.495140552520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6215 , TFLOPS: 69.03886246808935, Tokens per sec: 171085.90645445735, Loss: 3.4944441318511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6216 , TFLOPS: 67.84106308533885, Tokens per sec: 168117.62763550697, Loss: 3.490365505218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6217 , TFLOPS: 70.58380373549032, Tokens per sec: 174914.44110440704, Loss: 3.504791021347046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6218 , TFLOPS: 69.68311688861408, Tokens per sec: 172682.43987900083, Loss: 3.503448486328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6219 , TFLOPS: 68.77621910438077, Tokens per sec: 170435.04726663252, Loss: 3.5104386806488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6220 , TFLOPS: 68.59715241552142, Tokens per sec: 169991.30028581488, Loss: 3.467508554458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6221 , TFLOPS: 69.57916135152824, Tokens per sec: 172424.82660645436, Loss: 3.5099539756774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6222 , TFLOPS: 68.80720848172184, Tokens per sec: 170511.84235744545, Loss: 3.496838331222534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6223 , TFLOPS: 68.83242584175811, Tokens per sec: 170574.33375353683, Loss: 3.495866060256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6224 , TFLOPS: 69.65627856758563, Tokens per sec: 172615.9315630072, Loss: 3.495243549346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6225 , TFLOPS: 69.73909331788994, Tokens per sec: 172821.15563705913, Loss: 3.4925806522369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6226 , TFLOPS: 69.60884138318458, Tokens per sec: 172498.37699442415, Loss: 3.484201431274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6227 , TFLOPS: 68.53054893390996, Tokens per sec: 169826.24952140247, Loss: 3.5166070461273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6228 , TFLOPS: 69.57513407275755, Tokens per sec: 172414.8465660192, Loss: 3.4915287494659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6229 , TFLOPS: 68.09960661375874, Tokens per sec: 168758.32697985228, Loss: 3.4895520210266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6230 , TFLOPS: 68.69544807938561, Tokens per sec: 170234.8877690316, Loss: 3.481619119644165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6231 , TFLOPS: 69.60882687726468, Tokens per sec: 172498.34104715666, Loss: 3.5012853145599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6232 , TFLOPS: 68.88246560436905, Tokens per sec: 170698.33779762164, Loss: 3.4985427856445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6233 , TFLOPS: 68.68155254274163, Tokens per sec: 170200.45309850806, Loss: 3.48818302154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6234 , TFLOPS: 69.53397882697992, Tokens per sec: 172312.85933335766, Loss: 3.511113166809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6235 , TFLOPS: 68.74314516450092, Tokens per sec: 170353.086400215, Loss: 3.4923131465911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6236 , TFLOPS: 68.04665380481974, Tokens per sec: 168627.10408606796, Loss: 3.4982218742370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6237 , TFLOPS: 70.557479212546, Tokens per sec: 174849.20603666545, Loss: 3.4785044193267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6238 , TFLOPS: 68.69111169692735, Tokens per sec: 170224.14173560805, Loss: 3.512943744659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6239 , TFLOPS: 69.71126648049004, Tokens per sec: 172752.19766860295, Loss: 3.475897789001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6240 , TFLOPS: 68.94616392047219, Tokens per sec: 170856.189241874, Loss: 3.510194778442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6241 , TFLOPS: 69.56991028190171, Tokens per sec: 172401.90143683038, Loss: 3.510807752609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6242 , TFLOPS: 67.82584679805358, Tokens per sec: 168079.9199994016, Loss: 3.5079345703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6243 , TFLOPS: 68.88024195708404, Tokens per sec: 170692.8273546944, Loss: 3.5014946460723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6244 , TFLOPS: 69.64177564467127, Tokens per sec: 172579.99172239655, Loss: 3.486485481262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6245 , TFLOPS: 69.5943131684513, Tokens per sec: 172462.37447790007, Loss: 3.4892685413360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6246 , TFLOPS: 69.0872911087412, Tokens per sec: 171205.91796084782, Loss: 3.5063467025756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6247 , TFLOPS: 69.7361735464, Tokens per sec: 172813.92012166764, Loss: 3.509495973587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6248 , TFLOPS: 69.56189850050379, Tokens per sec: 172382.04735995573, Loss: 3.483091115951538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6249 , TFLOPS: 67.79178329208989, Tokens per sec: 167995.50687921865, Loss: 3.4939286708831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6250 , TFLOPS: 69.09798789424968, Tokens per sec: 171232.425773686, Loss: 3.4744443893432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6251 , TFLOPS: 69.62620631791287, Tokens per sec: 172541.40921558632, Loss: 3.5067920684814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6252 , TFLOPS: 68.88952591539072, Tokens per sec: 170715.83402609083, Loss: 3.4907009601593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6253 , TFLOPS: 67.72945006656381, Tokens per sec: 167841.03827979343, Loss: 3.5300450325012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6254 , TFLOPS: 70.6253496292253, Tokens per sec: 175017.39640574032, Loss: 3.477912187576294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6255 , TFLOPS: 68.73282338609008, Tokens per sec: 170327.50789627517, Loss: 3.506503105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6256 , TFLOPS: 68.86133032403735, Tokens per sec: 170645.9622447164, Loss: 3.481982469558716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6257 , TFLOPS: 68.96493666707723, Tokens per sec: 170902.71017595002, Loss: 3.51094388961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6258 , TFLOPS: 69.5905821314228, Tokens per sec: 172453.12855713585, Loss: 3.5003902912139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6259 , TFLOPS: 69.70317569401793, Tokens per sec: 172732.1478084514, Loss: 3.4814019203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6260 , TFLOPS: 67.70315125935961, Tokens per sec: 167775.86693848865, Loss: 3.4921610355377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6261 , TFLOPS: 69.59494207428611, Tokens per sec: 172463.93297583846, Loss: 3.471055030822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6262 , TFLOPS: 69.74455288111953, Tokens per sec: 172834.6850361626, Loss: 3.487428903579712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6263 , TFLOPS: 68.2038565675766, Tokens per sec: 169016.66984949337, Loss: 3.490011692047119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6264 , TFLOPS: 69.64685751225026, Tokens per sec: 172592.58515007122, Loss: 3.486513137817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6265 , TFLOPS: 69.95027723069656, Tokens per sec: 173344.49263682243, Loss: 3.525883674621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6266 , TFLOPS: 68.7488903668025, Tokens per sec: 170367.3236444046, Loss: 3.491431951522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6267 , TFLOPS: 68.63073524860673, Tokens per sec: 170074.52224565472, Loss: 3.474781036376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6268 , TFLOPS: 69.6100716155456, Tokens per sec: 172501.42564573605, Loss: 3.486402750015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6269 , TFLOPS: 68.95030041449336, Tokens per sec: 170866.4399297309, Loss: 3.49619722366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6270 , TFLOPS: 69.10481436309063, Tokens per sec: 171249.3425154715, Loss: 3.5273261070251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6271 , TFLOPS: 68.62063291142196, Tokens per sec: 170049.48754124658, Loss: 3.5040392875671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6272 , TFLOPS: 68.71677414353462, Tokens per sec: 170287.73610525273, Loss: 3.505969762802124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6273 , TFLOPS: 69.64358034013826, Tokens per sec: 172584.46395656737, Loss: 3.4898808002471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6274 , TFLOPS: 69.8984936115445, Tokens per sec: 173216.16712412704, Loss: 3.513026237487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6275 , TFLOPS: 68.39212155460962, Tokens per sec: 169483.21122646157, Loss: 3.4988930225372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6276 , TFLOPS: 68.27836298779071, Tokens per sec: 169201.3049663437, Loss: 3.506545305252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6277 , TFLOPS: 69.59936507289045, Tokens per sec: 172474.89365361354, Loss: 3.5118303298950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6278 , TFLOPS: 69.71523531465206, Tokens per sec: 172762.03287685965, Loss: 3.5137083530426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6279 , TFLOPS: 68.79760409873185, Tokens per sec: 170488.04163838612, Loss: 3.4878504276275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6280 , TFLOPS: 68.76440480195897, Tokens per sec: 170405.77012377846, Loss: 3.515885353088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6281 , TFLOPS: 70.60360565396296, Tokens per sec: 174963.51243974367, Loss: 3.485837697982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6282 , TFLOPS: 68.12772309376356, Tokens per sec: 168828.0027145902, Loss: 3.5140700340270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6283 , TFLOPS: 67.99088248893729, Tokens per sec: 168488.89662159362, Loss: 3.4992849826812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6284 , TFLOPS: 70.60508475611181, Tokens per sec: 174967.1778178051, Loss: 3.482950448989868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6285 , TFLOPS: 69.63294174586352, Tokens per sec: 172558.10034801668, Loss: 3.48897385597229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6286 , TFLOPS: 67.80986495567848, Tokens per sec: 168040.31523345085, Loss: 3.493849515914917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6287 , TFLOPS: 69.62922004541997, Tokens per sec: 172548.8775643384, Loss: 3.4920527935028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6288 , TFLOPS: 69.62993046165765, Tokens per sec: 172550.63805403418, Loss: 3.4739038944244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6289 , TFLOPS: 68.95563601970939, Tokens per sec: 170879.66214722165, Loss: 3.4990122318267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6290 , TFLOPS: 68.49419695959104, Tokens per sec: 169736.16532453964, Loss: 3.4880177974700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6291 , TFLOPS: 69.63374750493125, Tokens per sec: 172560.0971077482, Loss: 3.4966249465942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6292 , TFLOPS: 69.65775786345974, Tokens per sec: 172619.5974211411, Loss: 3.5049996376037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6293 , TFLOPS: 68.986189690503, Tokens per sec: 170955.3775092129, Loss: 3.50028920173645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6294 , TFLOPS: 68.72338072882864, Tokens per sec: 170304.10795138677, Loss: 3.510343074798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6295 , TFLOPS: 69.56101191997631, Tokens per sec: 172379.85031574374, Loss: 3.4833695888519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6296 , TFLOPS: 68.41866653238897, Tokens per sec: 169548.9926055695, Loss: 3.480257034301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6297 , TFLOPS: 68.60037754893624, Tokens per sec: 169999.29252169456, Loss: 3.5159621238708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6298 , TFLOPS: 69.64885000892083, Tokens per sec: 172597.52277631254, Loss: 3.5046098232269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6299 , TFLOPS: 69.27102017410003, Tokens per sec: 171661.21882423433, Loss: 3.491426944732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6300 , TFLOPS: 68.7334575111235, Tokens per sec: 170329.07932796265, Loss: 3.4851226806640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6301 , TFLOPS: 69.52311402766797, Tokens per sec: 172285.93516380622, Loss: 3.4975106716156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6302 , TFLOPS: 68.78698693288526, Tokens per sec: 170461.73113184088, Loss: 3.4980416297912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6303 , TFLOPS: 68.08642157834875, Tokens per sec: 168725.65300965475, Loss: 3.5034432411193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6304 , TFLOPS: 69.6307119379952, Tokens per sec: 172552.57463848536, Loss: 3.501696825027466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6305 , TFLOPS: 69.59832138687659, Tokens per sec: 172472.3072846997, Loss: 3.4924933910369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6306 , TFLOPS: 69.66136960059936, Tokens per sec: 172628.54770363882, Loss: 3.5039258003234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6307 , TFLOPS: 68.99867065839781, Tokens per sec: 170986.3067225484, Loss: 3.485218048095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6308 , TFLOPS: 69.53916084362929, Tokens per sec: 172325.70094146102, Loss: 3.502274751663208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6309 , TFLOPS: 68.96492669545782, Tokens per sec: 170902.68546517869, Loss: 3.514651298522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6310 , TFLOPS: 68.78917889390297, Tokens per sec: 170467.1630527072, Loss: 3.490647792816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6311 , TFLOPS: 70.60164483985099, Tokens per sec: 174958.6533263727, Loss: 3.485602855682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6312 , TFLOPS: 68.95247115897605, Tokens per sec: 170871.81927368627, Loss: 3.505277156829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6313 , TFLOPS: 68.0646047080147, Tokens per sec: 168671.5884016403, Loss: 3.5056333541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6314 , TFLOPS: 69.63720362457177, Tokens per sec: 172568.66175294042, Loss: 3.4951817989349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6315 , TFLOPS: 69.55422486419916, Tokens per sec: 172363.03124387204, Loss: 3.507905960083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6316 , TFLOPS: 67.20462033989082, Tokens per sec: 166540.45240233061, Loss: 3.488316535949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6317 , TFLOPS: 69.62995249469122, Tokens per sec: 172550.6926543185, Loss: 3.5117318630218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6318 , TFLOPS: 69.65091466868166, Tokens per sec: 172602.63923064093, Loss: 3.4954512119293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6319 , TFLOPS: 69.61909043122988, Tokens per sec: 172523.77526450626, Loss: 3.491131544113159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6320 , TFLOPS: 68.66569081096186, Tokens per sec: 170161.14597984383, Loss: 3.5071487426757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6321 , TFLOPS: 70.55336545655908, Tokens per sec: 174839.01169616223, Loss: 3.4857916831970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6322 , TFLOPS: 68.76191008862511, Tokens per sec: 170399.5879493218, Loss: 3.4972095489501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6323 , TFLOPS: 68.00578435757303, Tokens per sec: 168525.82509364936, Loss: 3.4999847412109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6324 , TFLOPS: 69.92974791795504, Tokens per sec: 173293.61873835843, Loss: 3.5023651123046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6325 , TFLOPS: 69.67423600579706, Tokens per sec: 172660.43207307023, Loss: 3.495213508605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6326 , TFLOPS: 69.64732501509931, Tokens per sec: 172593.74367363084, Loss: 3.496901035308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6327 , TFLOPS: 68.64089702628821, Tokens per sec: 170099.70425016168, Loss: 3.508178949356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6328 , TFLOPS: 69.6087324655094, Tokens per sec: 172498.10708442677, Loss: 3.4995532035827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6329 , TFLOPS: 69.83862529056485, Tokens per sec: 173067.80682969946, Loss: 3.4791202545166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6330 , TFLOPS: 68.27253284987505, Tokens per sec: 169186.85724234596, Loss: 3.4732577800750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6331 , TFLOPS: 69.47539533226681, Tokens per sec: 172167.68297995388, Loss: 3.4792721271514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6332 , TFLOPS: 69.58910731543666, Tokens per sec: 172449.47380065778, Loss: 3.5078327655792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6333 , TFLOPS: 67.81217161081246, Tokens per sec: 168046.03137897188, Loss: 3.4885025024414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6334 , TFLOPS: 68.734617919387, Tokens per sec: 170331.95494746912, Loss: 3.4926416873931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6335 , TFLOPS: 69.57621783389676, Tokens per sec: 172417.53224550828, Loss: 3.4760167598724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6336 , TFLOPS: 68.17100294883616, Tokens per sec: 168935.2549631297, Loss: 3.4770007133483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6337 , TFLOPS: 68.59224088942113, Tokens per sec: 169979.1289830891, Loss: 3.5144362449645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6338 , TFLOPS: 69.61393220266602, Tokens per sec: 172510.9926058992, Loss: 3.4943366050720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6339 , TFLOPS: 69.6312271839045, Tokens per sec: 172553.85147460763, Loss: 3.5136938095092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6340 , TFLOPS: 69.64289809927756, Tokens per sec: 172582.77328855955, Loss: 3.5061450004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6341 , TFLOPS: 69.94229526822974, Tokens per sec: 173324.71245454386, Loss: 3.4943201541900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6342 , TFLOPS: 67.88311064460119, Tokens per sec: 168221.82611928004, Loss: 3.508608341217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6343 , TFLOPS: 68.87899759888309, Tokens per sec: 170689.7436979951, Loss: 3.4856655597686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6344 , TFLOPS: 69.6747926171014, Tokens per sec: 172661.81141719822, Loss: 3.5000061988830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6345 , TFLOPS: 68.67404614299245, Tokens per sec: 170181.85141303143, Loss: 3.5090889930725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6346 , TFLOPS: 69.67666699091232, Tokens per sec: 172666.45632198098, Loss: 3.499114990234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6347 , TFLOPS: 68.81709469163532, Tokens per sec: 170536.34147466696, Loss: 3.5216612815856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6348 , TFLOPS: 70.61305622423137, Tokens per sec: 174986.93199393502, Loss: 3.4907591342926025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6349 , TFLOPS: 68.08776551706892, Tokens per sec: 168728.98343785037, Loss: 3.4891324043273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6350 , TFLOPS: 68.06475805967871, Tokens per sec: 168671.96842395683, Loss: 3.503371477127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6351 , TFLOPS: 70.59183502351152, Tokens per sec: 174934.34352084302, Loss: 3.5397167205810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6352 , TFLOPS: 68.64215660260501, Tokens per sec: 170102.82561902847, Loss: 3.4822397232055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6353 , TFLOPS: 67.83428242961976, Tokens per sec: 168100.82442368416, Loss: 3.4802165031433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6354 , TFLOPS: 69.68587386857484, Tokens per sec: 172689.27197905, Loss: 3.494122266769409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6355 , TFLOPS: 69.15580346599424, Tokens per sec: 171375.69912938005, Loss: 3.5052506923675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6356 , TFLOPS: 68.74131114673888, Tokens per sec: 170348.54150216695, Loss: 3.5060510635375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6357 , TFLOPS: 69.1291028106325, Tokens per sec: 171309.53196407366, Loss: 3.4977774620056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6358 , TFLOPS: 69.56734127799875, Tokens per sec: 172395.53516215968, Loss: 3.486881971359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6359 , TFLOPS: 69.76498048129864, Tokens per sec: 172885.30687969236, Loss: 3.513139009475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6360 , TFLOPS: 68.75689899778877, Tokens per sec: 170387.1699142121, Loss: 3.497544765472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6361 , TFLOPS: 69.5545276776362, Tokens per sec: 172363.78164892667, Loss: 3.4948997497558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6362 , TFLOPS: 69.69257527876644, Tokens per sec: 172705.87881172606, Loss: 3.504030704498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6363 , TFLOPS: 68.39164535095756, Tokens per sec: 169482.0311413536, Loss: 3.4990439414978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6364 , TFLOPS: 68.68950435491095, Tokens per sec: 170220.1585650281, Loss: 3.4856534004211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6365 , TFLOPS: 69.53883933058064, Tokens per sec: 172324.9041967089, Loss: 3.4945075511932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6366 , TFLOPS: 69.72316575358109, Tokens per sec: 172781.68537813573, Loss: 3.5116236209869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6367 , TFLOPS: 67.52382509204604, Tokens per sec: 167331.47694147748, Loss: 3.502528667449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6368 , TFLOPS: 70.61335067788369, Tokens per sec: 174987.66168252222, Loss: 3.500131368637085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6369 , TFLOPS: 68.89035392246058, Tokens per sec: 170717.88591882243, Loss: 3.5176541805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6370 , TFLOPS: 68.86591772918442, Tokens per sec: 170657.3303400152, Loss: 3.4945333003997803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6371 , TFLOPS: 68.89589059196487, Tokens per sec: 170731.60639576858, Loss: 3.4880988597869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6372 , TFLOPS: 69.57108864682391, Tokens per sec: 172404.82155491516, Loss: 3.508497953414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6373 , TFLOPS: 68.808574453272, Tokens per sec: 170515.2273854209, Loss: 3.4918477535247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6374 , TFLOPS: 68.96161868835635, Tokens per sec: 170894.4878591742, Loss: 3.500736713409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6375 , TFLOPS: 68.6821431860688, Tokens per sec: 170201.91677773753, Loss: 3.480167865753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6376 , TFLOPS: 68.98870927172072, Tokens per sec: 170961.62130902524, Loss: 3.4842240810394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6377 , TFLOPS: 68.91542221962457, Tokens per sec: 170780.0079206957, Loss: 3.512653350830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6378 , TFLOPS: 69.59582424129508, Tokens per sec: 172466.1190828654, Loss: 3.4975547790527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6379 , TFLOPS: 69.5557963170769, Tokens per sec: 172366.9254772129, Loss: 3.514516830444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6380 , TFLOPS: 69.0522113917993, Tokens per sec: 171118.98655791595, Loss: 3.479525089263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6381 , TFLOPS: 69.69572631262746, Tokens per sec: 172713.68742074867, Loss: 3.5089526176452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6382 , TFLOPS: 69.46136970958828, Tokens per sec: 172132.92594190658, Loss: 3.4998044967651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6383 , TFLOPS: 68.17195975229386, Tokens per sec: 168937.62602749845, Loss: 3.50219464302063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6384 , TFLOPS: 69.67589911741132, Tokens per sec: 172664.55344685726, Loss: 3.4797332286834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6385 , TFLOPS: 69.50406010152699, Tokens per sec: 172238.7174358656, Loss: 3.4974589347839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6386 , TFLOPS: 68.82802513735682, Tokens per sec: 170563.4283232528, Loss: 3.513753890991211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6387 , TFLOPS: 68.71460467571728, Tokens per sec: 170282.35992501478, Loss: 3.503932476043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6388 , TFLOPS: 70.54275878239002, Tokens per sec: 174812.72718914968, Loss: 3.495378017425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6389 , TFLOPS: 68.58425203500339, Tokens per sec: 169959.33172179697, Loss: 3.501779794692993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6390 , TFLOPS: 68.77538649476331, Tokens per sec: 170432.98396828093, Loss: 3.4906704425811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6391 , TFLOPS: 69.7467399558753, Tokens per sec: 172840.10484833358, Loss: 3.505195140838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6392 , TFLOPS: 69.52442958071778, Tokens per sec: 172289.19524918374, Loss: 3.511060953140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6393 , TFLOPS: 67.87537660901344, Tokens per sec: 168202.66032711926, Loss: 3.5168371200561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6394 , TFLOPS: 68.65100565177876, Tokens per sec: 170124.75453768426, Loss: 3.526360034942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6395 , TFLOPS: 69.67157802182192, Tokens per sec: 172653.84529597874, Loss: 3.508936882019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6396 , TFLOPS: 69.72834294755265, Tokens per sec: 172794.51503511227, Loss: 3.511232376098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6397 , TFLOPS: 67.43622933525026, Tokens per sec: 167114.40500666833, Loss: 3.493659496307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6398 , TFLOPS: 70.54019385774313, Tokens per sec: 174806.3710233249, Loss: 3.527355670928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6399 , TFLOPS: 69.78480622337432, Tokens per sec: 172934.43725254165, Loss: 3.4911041259765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6400 , TFLOPS: 68.48633502112766, Tokens per sec: 169716.6825749634, Loss: 3.5233325958251953 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/optimizer.pt +[rank0]:[2024-08-25 10:08:18,614] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006410309004422743, 'preprocessing_with_comm': 0.0013160829985281453, 'state_converting': 0.96311259399954, : 0.9722356269994634}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400, took 4.32s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004400 + eval ppl=24.710468292236328, eval loss=3.2072269916534424 +------------------------------------------------------------------ +iteration: 6401 , TFLOPS: 64.0448548640749, Tokens per sec: 158710.20547635708, Loss: 3.498770236968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6402 , TFLOPS: 69.58779237764503, Tokens per sec: 172446.21523995776, Loss: 3.480515241622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6403 , TFLOPS: 68.0780792667359, Tokens per sec: 168704.97984249663, Loss: 3.483703851699829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6404 , TFLOPS: 69.70481914404213, Tokens per sec: 172736.22045865233, Loss: 3.4931397438049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6405 , TFLOPS: 68.5770235515744, Tokens per sec: 169941.41874357674, Loss: 3.517773151397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6406 , TFLOPS: 69.02055112729097, Tokens per sec: 171040.52893479823, Loss: 3.5089406967163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6407 , TFLOPS: 68.67438938126317, Tokens per sec: 170182.7019952771, Loss: 3.509918212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6408 , TFLOPS: 69.48328831222324, Tokens per sec: 172187.2426537694, Loss: 3.4887382984161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6409 , TFLOPS: 69.03790279991789, Tokens per sec: 171083.52829101228, Loss: 3.4888508319854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6410 , TFLOPS: 67.9912352427931, Tokens per sec: 168489.7707845074, Loss: 3.4835004806518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6411 , TFLOPS: 70.56978071993446, Tokens per sec: 174879.69052709694, Loss: 3.4721062183380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6412 , TFLOPS: 68.57079598141597, Tokens per sec: 169925.98613870065, Loss: 3.483288288116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6413 , TFLOPS: 67.9835069477565, Tokens per sec: 168470.61921806468, Loss: 3.505767822265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6414 , TFLOPS: 69.57617533534928, Tokens per sec: 172417.42692942603, Loss: 3.5143232345581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6415 , TFLOPS: 69.63679095710903, Tokens per sec: 172567.6391175087, Loss: 3.4909286499023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6416 , TFLOPS: 67.87621365835831, Tokens per sec: 168204.73462760446, Loss: 3.5047099590301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6417 , TFLOPS: 69.12581359350753, Tokens per sec: 171301.3809217008, Loss: 3.4866816997528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6418 , TFLOPS: 69.63631442731624, Tokens per sec: 172566.45822418804, Loss: 3.5128133296966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6419 , TFLOPS: 69.713887103243, Tokens per sec: 172758.69186047022, Loss: 3.519697666168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6420 , TFLOPS: 68.60150862316574, Tokens per sec: 170002.0954482334, Loss: 3.509589672088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6421 , TFLOPS: 69.65984964940485, Tokens per sec: 172624.78109714235, Loss: 3.4834961891174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6422 , TFLOPS: 69.60196015064098, Tokens per sec: 172481.32454215075, Loss: 3.4828855991363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6423 , TFLOPS: 68.15699250399791, Tokens per sec: 168900.5355374426, Loss: 3.489565372467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6424 , TFLOPS: 69.76257850026704, Tokens per sec: 172879.35450609602, Loss: 3.494175434112549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6425 , TFLOPS: 69.57671518437607, Tokens per sec: 172418.76473478373, Loss: 3.4991960525512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6426 , TFLOPS: 69.6117008480468, Tokens per sec: 172505.46306334878, Loss: 3.4923453330993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6427 , TFLOPS: 67.8317956849485, Tokens per sec: 168094.66199645124, Loss: 3.5065388679504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6428 , TFLOPS: 70.60597446936865, Tokens per sec: 174969.3826252655, Loss: 3.4820899963378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6429 , TFLOPS: 68.7011342828294, Tokens per sec: 170248.97880755234, Loss: 3.4777095317840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6430 , TFLOPS: 67.6044207831778, Tokens per sec: 167531.20194245028, Loss: 3.4758365154266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6431 , TFLOPS: 69.98983496928965, Tokens per sec: 173442.52107070075, Loss: 3.481454610824585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6432 , TFLOPS: 69.58274441184386, Tokens per sec: 172433.70582462312, Loss: 3.4832241535186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6433 , TFLOPS: 68.99428762109288, Tokens per sec: 170975.44507327533, Loss: 3.4853076934814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6434 , TFLOPS: 67.5401167611253, Tokens per sec: 167371.8494921314, Loss: 3.4829726219177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6435 , TFLOPS: 69.60741864372768, Tokens per sec: 172494.85128932225, Loss: 3.477602958679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6436 , TFLOPS: 68.74843174971816, Tokens per sec: 170366.18714074878, Loss: 3.4909932613372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6437 , TFLOPS: 68.90026557440261, Tokens per sec: 170742.44808419337, Loss: 3.484306573867798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6438 , TFLOPS: 68.63225838924865, Tokens per sec: 170078.2967559532, Loss: 3.507967472076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6439 , TFLOPS: 70.60757801537703, Tokens per sec: 174973.35638891946, Loss: 3.494297981262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6440 , TFLOPS: 69.0953095436673, Tokens per sec: 171225.78852589818, Loss: 3.4969444274902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6441 , TFLOPS: 69.64999103554786, Tokens per sec: 172600.35036598874, Loss: 3.4789934158325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6442 , TFLOPS: 68.2207007940232, Tokens per sec: 169058.41169817277, Loss: 3.4834656715393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6443 , TFLOPS: 68.03056821100203, Tokens per sec: 168587.24221261323, Loss: 3.494960069656372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6444 , TFLOPS: 69.73191252275001, Tokens per sec: 172803.3608356725, Loss: 3.5078041553497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6445 , TFLOPS: 68.71702250964584, Tokens per sec: 170288.35158383613, Loss: 3.4953393936157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6446 , TFLOPS: 69.71942711304874, Tokens per sec: 172772.42061505135, Loss: 3.4757137298583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6447 , TFLOPS: 68.51080498463811, Tokens per sec: 169777.32184013128, Loss: 3.50683331489563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6448 , TFLOPS: 70.60772991324389, Tokens per sec: 174973.73280856645, Loss: 3.5008819103240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6449 , TFLOPS: 67.60297241765217, Tokens per sec: 167527.6127331271, Loss: 3.4896411895751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6450 , TFLOPS: 68.7715194973211, Tokens per sec: 170423.4011226342, Loss: 3.5046443939208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6451 , TFLOPS: 69.6790831794179, Tokens per sec: 172672.44390326293, Loss: 3.5101206302642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6452 , TFLOPS: 69.04751972854481, Tokens per sec: 171107.36009954195, Loss: 3.472665548324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6453 , TFLOPS: 67.97664175659338, Tokens per sec: 168453.60651809926, Loss: 3.4952259063720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6454 , TFLOPS: 68.69072288396185, Tokens per sec: 170223.1782142483, Loss: 3.500401496887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6455 , TFLOPS: 69.59490871949563, Tokens per sec: 172463.85031899333, Loss: 3.5233068466186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6456 , TFLOPS: 69.78439839975694, Tokens per sec: 172933.4266206923, Loss: 3.504239797592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6457 , TFLOPS: 67.92949750123539, Tokens per sec: 168336.77785995367, Loss: 3.5037436485290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6458 , TFLOPS: 70.58289942152578, Tokens per sec: 174912.20011478453, Loss: 3.5084292888641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6459 , TFLOPS: 69.84594157569504, Tokens per sec: 173085.9373902074, Loss: 3.5029220581054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6460 , TFLOPS: 67.87269508515323, Tokens per sec: 168196.01521559941, Loss: 3.4920833110809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6461 , TFLOPS: 69.60100913948462, Tokens per sec: 172478.96783174278, Loss: 3.468972682952881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6462 , TFLOPS: 69.58818912573473, Tokens per sec: 172447.19842542944, Loss: 3.473576307296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6463 , TFLOPS: 68.11584952046566, Tokens per sec: 168798.5786919759, Loss: 3.477029800415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6464 , TFLOPS: 68.6895865668056, Tokens per sec: 170220.36229515943, Loss: 3.507812976837158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6465 , TFLOPS: 69.59959296021489, Tokens per sec: 172475.45838350736, Loss: 3.4958932399749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6466 , TFLOPS: 69.69546158242281, Tokens per sec: 172713.03139013916, Loss: 3.519969940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6467 , TFLOPS: 68.70755707798403, Tokens per sec: 170264.8952014748, Loss: 3.51194167137146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6468 , TFLOPS: 69.81908623620276, Tokens per sec: 173019.38690058573, Loss: 3.507099151611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6469 , TFLOPS: 69.56694431958833, Tokens per sec: 172394.5514554901, Loss: 3.504366159439087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6470 , TFLOPS: 68.39169716667459, Tokens per sec: 169482.1595464084, Loss: 3.489394426345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6471 , TFLOPS: 68.4820705641082, Tokens per sec: 169706.11478070042, Loss: 3.5207903385162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6472 , TFLOPS: 69.58827127108154, Tokens per sec: 172447.4019906479, Loss: 3.534437417984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6473 , TFLOPS: 68.85178042645654, Tokens per sec: 170622.29654650306, Loss: 3.508624792098999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6474 , TFLOPS: 68.65281006099113, Tokens per sec: 170129.22606248458, Loss: 3.5010013580322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6475 , TFLOPS: 69.70404173908943, Tokens per sec: 172734.2939635413, Loss: 3.4867587089538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6476 , TFLOPS: 69.05819621123665, Tokens per sec: 171133.81759976398, Loss: 3.5083999633789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6477 , TFLOPS: 68.04434337001317, Tokens per sec: 168621.37857410184, Loss: 3.503629446029663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6478 , TFLOPS: 70.57499551593176, Tokens per sec: 174892.61336603548, Loss: 3.488825798034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6479 , TFLOPS: 68.5831587103848, Tokens per sec: 169956.62234295244, Loss: 3.4790964126586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6480 , TFLOPS: 68.70966922768291, Tokens per sec: 170270.1293411011, Loss: 3.462909698486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6481 , TFLOPS: 69.69299255247809, Tokens per sec: 172706.91286195003, Loss: 3.5176548957824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6482 , TFLOPS: 69.54147440803267, Tokens per sec: 172331.43420891275, Loss: 3.479565143585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6483 , TFLOPS: 68.84776792490153, Tokens per sec: 170612.3531256362, Loss: 3.4855406284332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6484 , TFLOPS: 69.0938626333604, Tokens per sec: 171222.20292276738, Loss: 3.482811689376831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6485 , TFLOPS: 69.66002787120419, Tokens per sec: 172625.22275039388, Loss: 3.505567789077759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6486 , TFLOPS: 68.63364665102405, Tokens per sec: 170081.73702156232, Loss: 3.5174787044525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6487 , TFLOPS: 69.65490875388551, Tokens per sec: 172612.53701376086, Loss: 3.511409282684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6488 , TFLOPS: 69.66858778122253, Tokens per sec: 172646.43515037125, Loss: 3.50311541557312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6489 , TFLOPS: 69.58209674151726, Tokens per sec: 172432.10082620702, Loss: 3.4962220191955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6490 , TFLOPS: 67.82844564810222, Tokens per sec: 168086.36023610912, Loss: 3.4986865520477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6491 , TFLOPS: 69.6625520558806, Tokens per sec: 172631.4779580839, Loss: 3.503526210784912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6492 , TFLOPS: 69.60198137904403, Tokens per sec: 172481.377148472, Loss: 3.5023763179779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6493 , TFLOPS: 68.18856048995276, Tokens per sec: 168978.76448413945, Loss: 3.494678497314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6494 , TFLOPS: 67.80690896134679, Tokens per sec: 168032.98995386728, Loss: 3.480870008468628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6495 , TFLOPS: 70.61631484708785, Tokens per sec: 174995.00722034037, Loss: 3.479440450668335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6496 , TFLOPS: 68.30066174238405, Tokens per sec: 169256.56373663654, Loss: 3.4999241828918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6497 , TFLOPS: 66.9811655951154, Tokens per sec: 165986.7069292049, Loss: 3.5094246864318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6498 , TFLOPS: 70.57216446226253, Tokens per sec: 174885.5977031768, Loss: 3.5091848373413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6499 , TFLOPS: 69.71631162072251, Tokens per sec: 172764.7000818671, Loss: 3.5087344646453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6500 , TFLOPS: 69.7122630986211, Tokens per sec: 172754.6673981183, Loss: 3.4982075691223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6501 , TFLOPS: 68.11017224247207, Tokens per sec: 168784.509771704, Loss: 3.514127254486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6502 , TFLOPS: 69.55551102840208, Tokens per sec: 172366.21850044955, Loss: 3.507402181625366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6503 , TFLOPS: 68.94593290275623, Tokens per sec: 170855.61675452517, Loss: 3.5063443183898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6504 , TFLOPS: 67.93392458547255, Tokens per sec: 168347.74866237698, Loss: 3.4942409992218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6505 , TFLOPS: 69.67768830540422, Tokens per sec: 172668.987251799, Loss: 3.5022504329681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6506 , TFLOPS: 70.54863310513794, Tokens per sec: 174827.28440802908, Loss: 3.498807430267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6507 , TFLOPS: 69.09479511977754, Tokens per sec: 171224.51372683092, Loss: 3.505303382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6508 , TFLOPS: 68.78094016467912, Tokens per sec: 170446.74657411955, Loss: 3.500572919845581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6509 , TFLOPS: 69.57948502988556, Tokens per sec: 172425.62871707988, Loss: 3.484184741973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6510 , TFLOPS: 68.02056771181113, Tokens per sec: 168562.45987397223, Loss: 3.5074667930603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6511 , TFLOPS: 69.5691788941304, Tokens per sec: 172400.08897736404, Loss: 3.4956512451171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6512 , TFLOPS: 68.64086246173497, Tokens per sec: 170099.6185953911, Loss: 3.5128371715545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6513 , TFLOPS: 68.86507365749547, Tokens per sec: 170655.2386373878, Loss: 3.5191235542297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6514 , TFLOPS: 68.68793459635712, Tokens per sec: 170216.26853040853, Loss: 3.505843162536621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6515 , TFLOPS: 70.53956827811143, Tokens per sec: 174804.82076808426, Loss: 3.4768528938293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6516 , TFLOPS: 68.8039641668605, Tokens per sec: 170503.80258782822, Loss: 3.4991469383239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6517 , TFLOPS: 68.84687153074834, Tokens per sec: 170610.13176217861, Loss: 3.4685981273651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6518 , TFLOPS: 69.94461858497363, Tokens per sec: 173330.46988936968, Loss: 3.5262465476989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6519 , TFLOPS: 68.64663024990924, Tokens per sec: 170113.91180986128, Loss: 3.5233893394470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6520 , TFLOPS: 69.61722605603286, Tokens per sec: 172519.15513739013, Loss: 3.4890353679656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6521 , TFLOPS: 68.67829384732319, Tokens per sec: 170192.37769228627, Loss: 3.485281467437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6522 , TFLOPS: 69.52100570617786, Tokens per sec: 172280.7105109029, Loss: 3.491713762283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6523 , TFLOPS: 68.79503037551463, Tokens per sec: 170481.6636687348, Loss: 3.479583978652954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6524 , TFLOPS: 68.16337158581469, Tokens per sec: 168916.34360489846, Loss: 3.4844765663146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6525 , TFLOPS: 70.5512155286349, Tokens per sec: 174833.683937932, Loss: 3.502674102783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6526 , TFLOPS: 69.93829310278198, Tokens per sec: 173314.7946476894, Loss: 3.498534917831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6527 , TFLOPS: 68.59001127265539, Tokens per sec: 169973.60374713104, Loss: 3.471194267272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6528 , TFLOPS: 69.69467485373438, Tokens per sec: 172711.0817897839, Loss: 3.500596523284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6529 , TFLOPS: 69.52419937338681, Tokens per sec: 172288.62477005814, Loss: 3.4973509311676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6530 , TFLOPS: 68.8470548561355, Tokens per sec: 170610.586062683, Loss: 3.5121119022369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6531 , TFLOPS: 68.69727494421025, Tokens per sec: 170239.41494131283, Loss: 3.4892759323120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6532 , TFLOPS: 69.63743266657924, Tokens per sec: 172569.229344266, Loss: 3.492954730987549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6533 , TFLOPS: 68.8251818066477, Tokens per sec: 170556.3822365369, Loss: 3.5012052059173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6534 , TFLOPS: 69.52007191351659, Tokens per sec: 172278.3964698225, Loss: 3.484511375427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6535 , TFLOPS: 70.02699887213313, Tokens per sec: 173534.61731588875, Loss: 3.494783401489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6536 , TFLOPS: 68.76579692336323, Tokens per sec: 170409.21995397305, Loss: 3.4781575202941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6537 , TFLOPS: 68.87870106005302, Tokens per sec: 170689.00884210772, Loss: 3.506195068359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6538 , TFLOPS: 68.66913605139052, Tokens per sec: 170169.68366515057, Loss: 3.490027666091919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6539 , TFLOPS: 69.69821941093286, Tokens per sec: 172719.86559298693, Loss: 3.4927783012390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6540 , TFLOPS: 69.67281317752389, Tokens per sec: 172656.90614787186, Loss: 3.497349739074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6541 , TFLOPS: 67.75614791694062, Tokens per sec: 167907.19849403814, Loss: 3.500714063644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6542 , TFLOPS: 70.59158653964896, Tokens per sec: 174933.72775045884, Loss: 3.5036847591400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6543 , TFLOPS: 69.0770568550394, Tokens per sec: 171180.55635277863, Loss: 3.494353771209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6544 , TFLOPS: 68.03461236189307, Tokens per sec: 168597.26406402123, Loss: 3.5138847827911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6545 , TFLOPS: 69.54422691462024, Tokens per sec: 172338.2552234504, Loss: 3.485138416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6546 , TFLOPS: 69.7207892190383, Tokens per sec: 172775.79606374181, Loss: 3.5003137588500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6547 , TFLOPS: 68.78206031208107, Tokens per sec: 170449.52242277603, Loss: 3.502295732498169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6548 , TFLOPS: 69.68927719947844, Tokens per sec: 172697.70580793696, Loss: 3.4998090267181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6549 , TFLOPS: 68.61406486360258, Tokens per sec: 170033.21119521995, Loss: 3.4899582862854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6550 , TFLOPS: 69.02187810769446, Tokens per sec: 171043.81733841705, Loss: 3.522052764892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6551 , TFLOPS: 68.72396830050164, Tokens per sec: 170305.56401871884, Loss: 3.496337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6552 , TFLOPS: 69.67320314807137, Tokens per sec: 172657.8725378473, Loss: 3.4954705238342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6553 , TFLOPS: 68.54959068943343, Tokens per sec: 169873.43708921218, Loss: 3.5106513500213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6554 , TFLOPS: 68.9884270786999, Tokens per sec: 170960.92200363352, Loss: 3.507136106491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6555 , TFLOPS: 69.64134743249343, Tokens per sec: 172578.93056544842, Loss: 3.5037550926208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6556 , TFLOPS: 68.687433242601, Tokens per sec: 170215.02612057218, Loss: 3.494143009185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6557 , TFLOPS: 68.44481117350301, Tokens per sec: 169613.78190632007, Loss: 3.4774506092071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6558 , TFLOPS: 69.62767089438638, Tokens per sec: 172545.03859742408, Loss: 3.5071117877960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6559 , TFLOPS: 69.62561284066621, Tokens per sec: 172539.93851359218, Loss: 3.50832200050354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6560 , TFLOPS: 69.49393782442526, Tokens per sec: 172213.63331814663, Loss: 3.486313819885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6561 , TFLOPS: 68.83044853480786, Tokens per sec: 170569.43376909575, Loss: 3.4951417446136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6562 , TFLOPS: 70.62621206932698, Tokens per sec: 175019.53362731793, Loss: 3.4854540824890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6563 , TFLOPS: 68.73706738817471, Tokens per sec: 170338.02500095632, Loss: 3.489788770675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6564 , TFLOPS: 67.15247389089785, Tokens per sec: 166411.22775732045, Loss: 3.4920077323913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6565 , TFLOPS: 70.47073985722338, Tokens per sec: 174634.25635904723, Loss: 3.4979140758514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6566 , TFLOPS: 69.54317033902831, Tokens per sec: 172335.6369127422, Loss: 3.4995763301849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6567 , TFLOPS: 69.67367673524193, Tokens per sec: 172659.04613902586, Loss: 3.4937891960144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6568 , TFLOPS: 68.10156294028604, Tokens per sec: 168763.17497249407, Loss: 3.5010197162628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6569 , TFLOPS: 69.66957219618178, Tokens per sec: 172648.87463908253, Loss: 3.495985269546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6570 , TFLOPS: 69.01098671327809, Tokens per sec: 171016.82726326716, Loss: 3.4943628311157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6571 , TFLOPS: 67.65562254015884, Tokens per sec: 167658.08553658848, Loss: 3.49198842048645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6572 , TFLOPS: 70.57315091853586, Tokens per sec: 174888.04225048923, Loss: 3.487062454223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6573 , TFLOPS: 69.785401672505, Tokens per sec: 172935.91284108165, Loss: 3.497915744781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6574 , TFLOPS: 68.17312785057841, Tokens per sec: 168940.52070372424, Loss: 3.4860451221466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6575 , TFLOPS: 68.77741054845973, Tokens per sec: 170437.9997963086, Loss: 3.5283126831054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6576 , TFLOPS: 69.58342342611637, Tokens per sec: 172435.38849679, Loss: 3.488847255706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6577 , TFLOPS: 68.10795144252228, Tokens per sec: 168779.0063847871, Loss: 3.484375476837158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6578 , TFLOPS: 68.66910881510039, Tokens per sec: 170169.61617062334, Loss: 3.50929856300354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6579 , TFLOPS: 69.56453203541324, Tokens per sec: 172388.57354956915, Loss: 3.525669574737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6580 , TFLOPS: 69.65520986025793, Tokens per sec: 172613.2831885213, Loss: 3.501290798187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6581 , TFLOPS: 68.78823827374552, Tokens per sec: 170464.83209233917, Loss: 3.487440586090088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6582 , TFLOPS: 69.60359838284137, Tokens per sec: 172485.38426200952, Loss: 3.481472969055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6583 , TFLOPS: 68.91436555754069, Tokens per sec: 170777.38939565094, Loss: 3.49316668510437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6584 , TFLOPS: 68.8753996977119, Tokens per sec: 170680.82770255135, Loss: 3.4878294467926025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6585 , TFLOPS: 69.95111214391513, Tokens per sec: 173346.56164375128, Loss: 3.4947023391723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6586 , TFLOPS: 69.55089773635999, Tokens per sec: 172354.78625457318, Loss: 3.4830803871154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6587 , TFLOPS: 69.70238128470683, Tokens per sec: 172730.17917466708, Loss: 3.513726234436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6588 , TFLOPS: 68.75297919662766, Tokens per sec: 170377.45621513313, Loss: 3.5089282989501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6589 , TFLOPS: 69.68640703771594, Tokens per sec: 172690.59323091508, Loss: 3.497796058654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6590 , TFLOPS: 68.86684804137782, Tokens per sec: 170659.6357561128, Loss: 3.46047043800354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6591 , TFLOPS: 68.17944009964594, Tokens per sec: 168956.16315226522, Loss: 3.490090847015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6592 , TFLOPS: 70.57738411713403, Tokens per sec: 174898.53258294082, Loss: 3.5238051414489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6593 , TFLOPS: 68.5873319147948, Tokens per sec: 169966.96400319686, Loss: 3.482696294784546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6594 , TFLOPS: 67.91539414042049, Tokens per sec: 168301.82817824095, Loss: 3.5330283641815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6595 , TFLOPS: 69.65075017379792, Tokens per sec: 172602.23159419838, Loss: 3.5026116371154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6596 , TFLOPS: 69.65921785597145, Tokens per sec: 172623.21544341662, Loss: 3.4883170127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6597 , TFLOPS: 68.02667170740358, Tokens per sec: 168577.58624746097, Loss: 3.4844844341278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6598 , TFLOPS: 69.67734252999966, Tokens per sec: 172668.1303822562, Loss: 3.5142598152160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6599 , TFLOPS: 69.52806478552444, Tokens per sec: 172298.2036871455, Loss: 3.5198285579681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6600 , TFLOPS: 69.740526828579, Tokens per sec: 172824.70803446084, Loss: 3.519648551940918 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/model.pt +[rank0]:[2024-08-25 10:33:57,772] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006419751996872947, 'preprocessing_with_comm': 0.0014744200016139075, 'state_converting': 0.9731246089941124, : 0.982449839997571}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004600 + eval ppl=24.1011905670166, eval loss=3.1822612285614014 +------------------------------------------------------------------ +iteration: 6601 , TFLOPS: 63.66790251736399, Tokens per sec: 157776.07603647848, Loss: 3.482259511947632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6602 , TFLOPS: 70.47449433168038, Tokens per sec: 174643.56036034113, Loss: 3.4815287590026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6603 , TFLOPS: 68.88241029106557, Tokens per sec: 170698.20072516234, Loss: 3.4880011081695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6604 , TFLOPS: 67.38861083775387, Tokens per sec: 166996.40112426179, Loss: 3.492591381072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6605 , TFLOPS: 69.9067860885223, Tokens per sec: 173236.71679556975, Loss: 3.4981606006622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6606 , TFLOPS: 69.50554075902741, Tokens per sec: 172242.38666825954, Loss: 3.5075743198394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6607 , TFLOPS: 68.82799062555809, Tokens per sec: 170563.34279921374, Loss: 3.5006518363952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6608 , TFLOPS: 67.70832476684359, Tokens per sec: 167788.6874599428, Loss: 3.4756650924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6609 , TFLOPS: 69.6109219857734, Tokens per sec: 172503.53295683654, Loss: 3.4739978313446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6610 , TFLOPS: 68.92688373049857, Tokens per sec: 170808.41080723112, Loss: 3.5019664764404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6611 , TFLOPS: 68.83176214799221, Tokens per sec: 170572.68904727302, Loss: 3.48332142829895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6612 , TFLOPS: 69.47199583027245, Tokens per sec: 172159.25863952577, Loss: 3.4997568130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6613 , TFLOPS: 68.95392964763562, Tokens per sec: 170875.43356925045, Loss: 3.512725830078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6614 , TFLOPS: 68.99588092656646, Tokens per sec: 170979.393459756, Loss: 3.469759464263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6615 , TFLOPS: 68.11840019506575, Tokens per sec: 168804.8995446149, Loss: 3.4706482887268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6616 , TFLOPS: 69.64453046276294, Tokens per sec: 172586.81846509597, Loss: 3.495328664779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6617 , TFLOPS: 68.335526372317, Tokens per sec: 169342.96213026647, Loss: 3.487443208694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6618 , TFLOPS: 69.71462319837745, Tokens per sec: 172760.515985301, Loss: 3.489706516265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6619 , TFLOPS: 68.51356525582607, Tokens per sec: 169784.162096204, Loss: 3.4859790802001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6620 , TFLOPS: 69.68214571934932, Tokens per sec: 172680.0332145796, Loss: 3.4836649894714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6621 , TFLOPS: 68.7837561240407, Tokens per sec: 170453.7248316211, Loss: 3.492142915725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6622 , TFLOPS: 70.52772310044426, Tokens per sec: 174775.46711297097, Loss: 3.497620105743408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6623 , TFLOPS: 66.57426659385487, Tokens per sec: 164978.36638045605, Loss: 3.4876904487609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6624 , TFLOPS: 68.03789451134328, Tokens per sec: 168605.39759191748, Loss: 3.4868640899658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6625 , TFLOPS: 70.57476316258037, Tokens per sec: 174892.03756883487, Loss: 3.480232000350952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6626 , TFLOPS: 69.64486467143465, Tokens per sec: 172587.64667100064, Loss: 3.4887611865997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6627 , TFLOPS: 68.27444938810618, Tokens per sec: 169191.6066352068, Loss: 3.48868989944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6628 , TFLOPS: 68.69254898938749, Tokens per sec: 170227.70350465513, Loss: 3.4843733310699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6629 , TFLOPS: 70.60458497661499, Tokens per sec: 174965.93930915667, Loss: 3.5034687519073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6630 , TFLOPS: 64.37060087668783, Tokens per sec: 159517.4399794975, Loss: 3.502807855606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6631 , TFLOPS: 68.82259735063583, Tokens per sec: 170549.97766983815, Loss: 3.460737943649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6632 , TFLOPS: 70.57068529979215, Tokens per sec: 174881.932175632, Loss: 3.5030877590179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6633 , TFLOPS: 69.63446942533987, Tokens per sec: 172561.88610604685, Loss: 3.508880138397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6634 , TFLOPS: 66.65480564054484, Tokens per sec: 165177.95101026216, Loss: 3.515565872192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6635 , TFLOPS: 69.55201484209815, Tokens per sec: 172357.5545656517, Loss: 3.5225582122802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6636 , TFLOPS: 69.53702658801399, Tokens per sec: 172320.41202093865, Loss: 3.4972801208496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6637 , TFLOPS: 68.78754680563871, Tokens per sec: 170463.11855819903, Loss: 3.4754490852355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6638 , TFLOPS: 68.57954958573572, Tokens per sec: 169947.67853449413, Loss: 3.4838805198669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6639 , TFLOPS: 69.4826254305701, Tokens per sec: 172185.59996001047, Loss: 3.509479522705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6640 , TFLOPS: 69.6851872859389, Tokens per sec: 172687.5705516448, Loss: 3.5264639854431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6641 , TFLOPS: 68.40794398625981, Tokens per sec: 169522.42095507547, Loss: 3.4905433654785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6642 , TFLOPS: 70.52750969831266, Tokens per sec: 174774.9382789807, Loss: 3.493744373321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6643 , TFLOPS: 68.65324955379742, Tokens per sec: 170130.31517407275, Loss: 3.484801769256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6644 , TFLOPS: 68.91123149105707, Tokens per sec: 170769.62283365856, Loss: 3.502448558807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6645 , TFLOPS: 69.05021953729123, Tokens per sec: 171114.05052302405, Loss: 3.496676445007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6646 , TFLOPS: 69.65061730126367, Tokens per sec: 172601.9023214214, Loss: 3.4846339225769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6647 , TFLOPS: 69.636501404079, Tokens per sec: 172566.92157320335, Loss: 3.493980646133423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6648 , TFLOPS: 68.64347548068551, Tokens per sec: 170106.09394419836, Loss: 3.492056131362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6649 , TFLOPS: 69.43038702656656, Tokens per sec: 172056.14744035449, Loss: 3.480656147003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6650 , TFLOPS: 68.62913573942424, Tokens per sec: 170070.55848570005, Loss: 3.4754228591918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6651 , TFLOPS: 68.33161453365258, Tokens per sec: 169333.26816313143, Loss: 3.488084554672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6652 , TFLOPS: 69.44202240404803, Tokens per sec: 172084.9811874962, Loss: 3.4917712211608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6653 , TFLOPS: 69.65719598888188, Tokens per sec: 172618.2050340416, Loss: 3.497943639755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6654 , TFLOPS: 68.88990672749034, Tokens per sec: 170716.7777204236, Loss: 3.489710569381714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6655 , TFLOPS: 68.91770142755348, Tokens per sec: 170785.65604901902, Loss: 3.5195424556732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6656 , TFLOPS: 68.19090429366862, Tokens per sec: 168984.57268793863, Loss: 3.4946365356445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6657 , TFLOPS: 68.84442665544736, Tokens per sec: 170604.07309185754, Loss: 3.516962766647339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6658 , TFLOPS: 68.9207868803426, Tokens per sec: 170793.30214092115, Loss: 3.50765061378479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6659 , TFLOPS: 69.68703731559766, Tokens per sec: 172692.1551289368, Loss: 3.487149715423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6660 , TFLOPS: 68.66947702284499, Tokens per sec: 170170.52862997586, Loss: 3.5161209106445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6661 , TFLOPS: 69.68498896791354, Tokens per sec: 172687.07909773142, Loss: 3.488347053527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6662 , TFLOPS: 69.86072276354808, Tokens per sec: 173122.56680198954, Loss: 3.496809959411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6663 , TFLOPS: 68.61011398488668, Tokens per sec: 170023.42048253672, Loss: 3.4933340549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6664 , TFLOPS: 67.8370307505903, Tokens per sec: 168107.6350658017, Loss: 3.4809913635253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6665 , TFLOPS: 69.73574575249523, Tokens per sec: 172812.8600012463, Loss: 3.505894660949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6666 , TFLOPS: 69.5927276618883, Tokens per sec: 172458.44541797903, Loss: 3.5008351802825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6667 , TFLOPS: 68.70155984522465, Tokens per sec: 170250.03339804735, Loss: 3.4836196899414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6668 , TFLOPS: 68.6934098268396, Tokens per sec: 170229.83675469054, Loss: 3.482961654663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6669 , TFLOPS: 70.61456311099676, Tokens per sec: 174990.66622533696, Loss: 3.497814178466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6670 , TFLOPS: 68.63502354440331, Tokens per sec: 170085.14911503342, Loss: 3.4880383014678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6671 , TFLOPS: 67.72967602793338, Tokens per sec: 167841.59823695887, Loss: 3.49056077003479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6672 , TFLOPS: 70.49456895998289, Tokens per sec: 174693.30750067762, Loss: 3.5061521530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6673 , TFLOPS: 69.54865845238297, Tokens per sec: 172349.23706220064, Loss: 3.4907853603363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6674 , TFLOPS: 68.9195337628467, Tokens per sec: 170790.19677772402, Loss: 3.491914749145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6675 , TFLOPS: 67.76245302397648, Tokens per sec: 167922.82324383783, Loss: 3.492779493331909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6676 , TFLOPS: 69.5413127772526, Tokens per sec: 172331.03367003467, Loss: 3.5126733779907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6677 , TFLOPS: 68.77260115651603, Tokens per sec: 170426.08159327385, Loss: 3.5160770416259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6678 , TFLOPS: 68.90932964810693, Tokens per sec: 170764.90985732284, Loss: 3.5089774131774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6679 , TFLOPS: 69.51657380316672, Tokens per sec: 172269.72776702687, Loss: 3.4988298416137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6680 , TFLOPS: 69.04939670636575, Tokens per sec: 171112.01145734842, Loss: 3.4953689575195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6681 , TFLOPS: 69.65860818938803, Tokens per sec: 172621.70462246257, Loss: 3.5188491344451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6682 , TFLOPS: 68.68238002176777, Tokens per sec: 170202.50368268922, Loss: 3.47888445854187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6683 , TFLOPS: 69.60104858490668, Tokens per sec: 172479.0655818439, Loss: 3.5119731426239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6684 , TFLOPS: 69.03171823011466, Tokens per sec: 171068.2022457533, Loss: 3.4967474937438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6685 , TFLOPS: 69.0408308622637, Tokens per sec: 171090.78435205793, Loss: 3.503089666366577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6686 , TFLOPS: 68.49295193419267, Tokens per sec: 169733.08001445164, Loss: 3.4970312118530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6687 , TFLOPS: 69.780252781621, Tokens per sec: 172923.15332227395, Loss: 3.4788875579833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6688 , TFLOPS: 68.69206415444775, Tokens per sec: 170226.50203026546, Loss: 3.4990386962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6689 , TFLOPS: 69.67028712164355, Tokens per sec: 172650.64630313235, Loss: 3.473818063735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6690 , TFLOPS: 68.68879248422532, Tokens per sec: 170218.39447105007, Loss: 3.5065529346466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6691 , TFLOPS: 68.80439124558723, Tokens per sec: 170504.86093595964, Loss: 3.505539655685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6692 , TFLOPS: 69.9263709141377, Tokens per sec: 173285.2501508914, Loss: 3.5028374195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6693 , TFLOPS: 68.53912668808566, Tokens per sec: 169847.50614116748, Loss: 3.498974561691284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6694 , TFLOPS: 68.95585572342105, Tokens per sec: 170880.20659722152, Loss: 3.5133743286132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6695 , TFLOPS: 68.75439737366592, Tokens per sec: 170380.97061405904, Loss: 3.4994890689849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6696 , TFLOPS: 70.53536074896712, Tokens per sec: 174794.3940473671, Loss: 3.488356113433838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6697 , TFLOPS: 67.30770303773348, Tokens per sec: 166795.90268308128, Loss: 3.5018420219421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6698 , TFLOPS: 68.82436683530092, Tokens per sec: 170554.3626477631, Loss: 3.5071945190429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6699 , TFLOPS: 70.48543562904565, Tokens per sec: 174670.6741004957, Loss: 3.4630026817321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6700 , TFLOPS: 69.58421176064769, Tokens per sec: 172437.34207660085, Loss: 3.48671555519104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6701 , TFLOPS: 68.06144473290884, Tokens per sec: 168663.757635228, Loss: 3.502660036087036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6702 , TFLOPS: 69.74435419744337, Tokens per sec: 172834.19267612626, Loss: 3.5187783241271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6703 , TFLOPS: 69.45294488139902, Tokens per sec: 172112.04828958216, Loss: 3.478959560394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6704 , TFLOPS: 68.73362745488528, Tokens per sec: 170329.50046732518, Loss: 3.5048987865448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6705 , TFLOPS: 68.77708312126529, Tokens per sec: 170437.1883956517, Loss: 3.5046017169952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6706 , TFLOPS: 69.55066785629023, Tokens per sec: 172354.21658643693, Loss: 3.507830858230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6707 , TFLOPS: 69.73684638189609, Tokens per sec: 172815.58748214593, Loss: 3.476349115371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6708 , TFLOPS: 68.6040696644997, Tokens per sec: 170008.44199078094, Loss: 3.503781795501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6709 , TFLOPS: 70.62532641527275, Tokens per sec: 175017.3388790087, Loss: 3.5167510509490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6710 , TFLOPS: 68.77394157057422, Tokens per sec: 170429.40328696885, Loss: 3.4998435974121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6711 , TFLOPS: 68.81502845401415, Tokens per sec: 170531.22111022685, Loss: 3.4940812587738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6712 , TFLOPS: 68.52133591443331, Tokens per sec: 169803.41864424144, Loss: 3.492347002029419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6713 , TFLOPS: 69.60245677115725, Tokens per sec: 172482.55522249738, Loss: 3.501960515975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6714 , TFLOPS: 68.8872064972566, Tokens per sec: 170710.08625244946, Loss: 3.4932055473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6715 , TFLOPS: 67.76027356487053, Tokens per sec: 167917.4223040861, Loss: 3.4857821464538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6716 , TFLOPS: 70.5754679510437, Tokens per sec: 174893.78411228588, Loss: 3.5031931400299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6717 , TFLOPS: 68.75916126679351, Tokens per sec: 170392.7760660154, Loss: 3.4978785514831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6718 , TFLOPS: 68.18324145557119, Tokens per sec: 168965.5833310019, Loss: 3.516361951828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6719 , TFLOPS: 69.571527185361, Tokens per sec: 172405.90830171912, Loss: 3.482999801635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6720 , TFLOPS: 69.54064422101507, Tokens per sec: 172329.3769140305, Loss: 3.4907965660095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6721 , TFLOPS: 69.74368505740173, Tokens per sec: 172832.53447339122, Loss: 3.487701416015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6722 , TFLOPS: 69.13059534661049, Tokens per sec: 171313.2306326435, Loss: 3.464066982269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6723 , TFLOPS: 68.62262122747875, Tokens per sec: 170054.41480746635, Loss: 3.4908640384674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6724 , TFLOPS: 68.82340195558028, Tokens per sec: 170551.97156952505, Loss: 3.4946112632751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6725 , TFLOPS: 68.78715455865881, Tokens per sec: 170462.14652697326, Loss: 3.5072431564331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6726 , TFLOPS: 69.50604331362595, Tokens per sec: 172243.63205391567, Loss: 3.4821393489837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6727 , TFLOPS: 68.90228602741911, Tokens per sec: 170747.45498933955, Loss: 3.487025499343872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6728 , TFLOPS: 69.64440259767652, Tokens per sec: 172586.50160132634, Loss: 3.5094423294067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6729 , TFLOPS: 69.8938277917566, Tokens per sec: 173204.60470871016, Loss: 3.49406099319458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6730 , TFLOPS: 68.6996038491894, Tokens per sec: 170245.18622440123, Loss: 3.4623312950134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6731 , TFLOPS: 68.25941886394332, Tokens per sec: 169154.3593405823, Loss: 3.489398241043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6732 , TFLOPS: 69.70393877277311, Tokens per sec: 172734.0388016668, Loss: 3.4748809337615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6733 , TFLOPS: 69.58902750966726, Tokens per sec: 172449.27603316988, Loss: 3.474208354949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6734 , TFLOPS: 67.81622509621768, Tokens per sec: 168056.0763623343, Loss: 3.5042550563812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6735 , TFLOPS: 68.7765602466413, Tokens per sec: 170435.89265473402, Loss: 3.5110785961151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6736 , TFLOPS: 70.57942751200962, Tokens per sec: 174903.59634054106, Loss: 3.5114526748657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6737 , TFLOPS: 68.83548516436248, Tokens per sec: 170581.9150919627, Loss: 3.5111279487609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6738 , TFLOPS: 66.47465121880157, Tokens per sec: 164731.50850753445, Loss: 3.5199952125549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6739 , TFLOPS: 70.5665732581825, Tokens per sec: 174871.74208354452, Loss: 3.4867966175079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6740 , TFLOPS: 69.5698600662132, Tokens per sec: 172401.7769968225, Loss: 3.479731559753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6741 , TFLOPS: 68.6047123949607, Tokens per sec: 170010.03474766586, Loss: 3.485225200653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6742 , TFLOPS: 68.75030670570146, Tokens per sec: 170370.83348821884, Loss: 3.485408067703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6743 , TFLOPS: 69.67845987728462, Tokens per sec: 172670.8992919143, Loss: 3.5014452934265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6744 , TFLOPS: 69.04356708210265, Tokens per sec: 171097.56500623832, Loss: 3.501635789871216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6745 , TFLOPS: 68.57152746937327, Tokens per sec: 169927.79884643888, Loss: 3.4830501079559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6746 , TFLOPS: 70.56099725290673, Tokens per sec: 174857.92412822347, Loss: 3.495216131210327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6747 , TFLOPS: 68.7075470557678, Tokens per sec: 170264.87036531896, Loss: 3.5095577239990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6748 , TFLOPS: 69.56642081063961, Tokens per sec: 172393.25414264586, Loss: 3.4713826179504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6749 , TFLOPS: 68.51165477439908, Tokens per sec: 169779.42771277105, Loss: 3.4871842861175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6750 , TFLOPS: 69.6164391794016, Tokens per sec: 172517.20517041624, Loss: 3.496397018432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6751 , TFLOPS: 68.07727134427165, Tokens per sec: 168702.9777216302, Loss: 3.5050172805786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6752 , TFLOPS: 69.0870795098397, Tokens per sec: 171205.39359546034, Loss: 3.4924156665802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6753 , TFLOPS: 68.623662788824, Tokens per sec: 170056.99591121738, Loss: 3.4856009483337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6754 , TFLOPS: 68.77834754875047, Tokens per sec: 170440.32178624815, Loss: 3.4682111740112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6755 , TFLOPS: 68.72787616692649, Tokens per sec: 170315.24814220663, Loss: 3.4985246658325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6756 , TFLOPS: 69.436765315972, Tokens per sec: 172071.9535441277, Loss: 3.485008955001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6757 , TFLOPS: 68.91622263621439, Tokens per sec: 170781.99144117846, Loss: 3.475409746170044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6758 , TFLOPS: 68.02356764020689, Tokens per sec: 168569.89402700635, Loss: 3.5075559616088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6759 , TFLOPS: 70.56360719722545, Tokens per sec: 174864.39185775438, Loss: 3.512662172317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6760 , TFLOPS: 68.59336550110419, Tokens per sec: 169981.9158947259, Loss: 3.503492593765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6761 , TFLOPS: 69.64404858680992, Tokens per sec: 172585.62432340137, Loss: 3.4853339195251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6762 , TFLOPS: 68.6974654427769, Tokens per sec: 170239.88701774614, Loss: 3.492196798324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6763 , TFLOPS: 69.6151267509735, Tokens per sec: 172513.95282819553, Loss: 3.4909443855285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6764 , TFLOPS: 68.21362901363189, Tokens per sec: 169040.88704734217, Loss: 3.4825973510742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6765 , TFLOPS: 68.75349233787621, Tokens per sec: 170378.72783567422, Loss: 3.4995176792144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6766 , TFLOPS: 70.53596518485321, Tokens per sec: 174795.89190607693, Loss: 3.5166046619415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6767 , TFLOPS: 68.47976518854958, Tokens per sec: 169700.40180610807, Loss: 3.5136446952819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6768 , TFLOPS: 69.00880602247086, Tokens per sec: 171011.4232712239, Loss: 3.517500877380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6769 , TFLOPS: 69.6067138848356, Tokens per sec: 172493.10481915207, Loss: 3.480787992477417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6770 , TFLOPS: 69.64509018172599, Tokens per sec: 172588.20551034456, Loss: 3.5136542320251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6771 , TFLOPS: 68.67242262196726, Tokens per sec: 170177.82814908537, Loss: 3.5248701572418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6772 , TFLOPS: 69.05428663749038, Tokens per sec: 171124.1292453456, Loss: 3.515681505203247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6773 , TFLOPS: 69.54190487300775, Tokens per sec: 172332.50094854063, Loss: 3.504410982131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6774 , TFLOPS: 68.78708733309581, Tokens per sec: 170461.97993462265, Loss: 3.5062060356140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6775 , TFLOPS: 67.69464027320406, Tokens per sec: 167754.77577723263, Loss: 3.4936416149139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6776 , TFLOPS: 70.58643481948587, Tokens per sec: 174920.96122038589, Loss: 3.5020411014556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6777 , TFLOPS: 68.45017685065172, Tokens per sec: 169627.0786453727, Loss: 3.484297037124634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6778 , TFLOPS: 67.94990715518477, Tokens per sec: 168387.3552307493, Loss: 3.4969935417175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6779 , TFLOPS: 69.9693496415273, Tokens per sec: 173391.7561718605, Loss: 3.472219467163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6780 , TFLOPS: 69.6166435885866, Tokens per sec: 172517.71171889425, Loss: 3.4938855171203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6781 , TFLOPS: 69.57829265542495, Tokens per sec: 172422.6738818156, Loss: 3.501767873764038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6782 , TFLOPS: 68.52997352665726, Tokens per sec: 169824.82359884397, Loss: 3.505784749984741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6783 , TFLOPS: 70.59237250752733, Tokens per sec: 174935.67546544294, Loss: 3.4920990467071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6784 , TFLOPS: 68.9274828903233, Tokens per sec: 170809.8955912797, Loss: 3.4893999099731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6785 , TFLOPS: 68.18934717634095, Tokens per sec: 168980.71397967002, Loss: 3.494591236114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6786 , TFLOPS: 69.6065481162128, Tokens per sec: 172492.69402624387, Loss: 3.484111785888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6787 , TFLOPS: 69.55592092374903, Tokens per sec: 172367.234266273, Loss: 3.5237841606140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6788 , TFLOPS: 69.70162762751066, Tokens per sec: 172728.3115291128, Loss: 3.5070345401763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6789 , TFLOPS: 68.0205265418127, Tokens per sec: 168562.3578501809, Loss: 3.4992752075195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6790 , TFLOPS: 69.6302653085968, Tokens per sec: 172551.46784163584, Loss: 3.500725269317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6791 , TFLOPS: 68.91693250050108, Tokens per sec: 170783.75056307882, Loss: 3.4999008178710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6792 , TFLOPS: 68.81372592605456, Tokens per sec: 170527.99330245765, Loss: 3.4852232933044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6793 , TFLOPS: 68.52423842384097, Tokens per sec: 169810.61138229145, Loss: 3.5056023597717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6794 , TFLOPS: 68.83513688609436, Tokens per sec: 170581.05202004826, Loss: 3.4953911304473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6795 , TFLOPS: 69.62762763692251, Tokens per sec: 172544.93140066345, Loss: 3.500884771347046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6796 , TFLOPS: 69.96408573821617, Tokens per sec: 173378.71163958867, Loss: 3.495535135269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6797 , TFLOPS: 67.92557955505531, Tokens per sec: 168327.06875772207, Loss: 3.5141654014587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6798 , TFLOPS: 68.82750703543674, Tokens per sec: 170562.14440962125, Loss: 3.4950807094573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6799 , TFLOPS: 69.70762074875829, Tokens per sec: 172743.16314376122, Loss: 3.5090389251708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6800 , TFLOPS: 68.51793266708677, Tokens per sec: 169794.98502241808, Loss: 3.503232002258301 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/optimizer.pt +[rank0]:[2024-08-25 10:59:38,805] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006337515005725436, 'preprocessing_with_comm': 0.0016147679998539388, 'state_converting': 0.9983186489989748, : 1.0076670369962812}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800, took 4.38s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0004800 + eval ppl=26.348073959350586, eval loss=3.271395206451416 +------------------------------------------------------------------ +iteration: 6801 , TFLOPS: 64.61926281160115, Tokens per sec: 160133.6516465861, Loss: 3.5010335445404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6802 , TFLOPS: 69.44048706329679, Tokens per sec: 172081.17644398284, Loss: 3.5053529739379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6803 , TFLOPS: 70.54697755347331, Tokens per sec: 174823.18176863008, Loss: 3.4900314807891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6804 , TFLOPS: 67.04716416718415, Tokens per sec: 166150.25866113417, Loss: 3.4836232662200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6805 , TFLOPS: 70.60068147179572, Tokens per sec: 174956.265994208, Loss: 3.5051558017730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6806 , TFLOPS: 68.9005465713616, Tokens per sec: 170743.14442561215, Loss: 3.501323938369751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6807 , TFLOPS: 69.53087826084808, Tokens per sec: 172305.17578892742, Loss: 3.487006664276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6808 , TFLOPS: 68.61814389660789, Tokens per sec: 170043.31948834966, Loss: 3.514199733734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6809 , TFLOPS: 69.43168591521643, Tokens per sec: 172059.36622951325, Loss: 3.5109219551086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6810 , TFLOPS: 69.48978996014516, Tokens per sec: 172203.35445353555, Loss: 3.4857921600341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6811 , TFLOPS: 67.8130375719834, Tokens per sec: 168048.17732614698, Loss: 3.485682725906372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6812 , TFLOPS: 68.74056267994229, Tokens per sec: 170346.68671899458, Loss: 3.4986257553100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6813 , TFLOPS: 70.52896719850841, Tokens per sec: 174778.55012502268, Loss: 3.4961280822753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6814 , TFLOPS: 68.78955212653061, Tokens per sec: 170468.08796427347, Loss: 3.5201499462127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6815 , TFLOPS: 67.9679830998864, Tokens per sec: 168432.14941294957, Loss: 3.485060214996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6816 , TFLOPS: 70.57276606732701, Tokens per sec: 174887.08854679886, Loss: 3.4934682846069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6817 , TFLOPS: 69.56758304213514, Tokens per sec: 172396.13428032215, Loss: 3.488865375518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6818 , TFLOPS: 68.72917671382234, Tokens per sec: 170318.47104068162, Loss: 3.4971117973327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6819 , TFLOPS: 67.71357643712172, Tokens per sec: 167801.70167741124, Loss: 3.4837398529052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6820 , TFLOPS: 69.6418128368666, Tokens per sec: 172580.08388875364, Loss: 3.474949359893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6821 , TFLOPS: 68.9345989265469, Tokens per sec: 170827.5299129379, Loss: 3.505985736846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6822 , TFLOPS: 68.7676820713483, Tokens per sec: 170413.89155834625, Loss: 3.5113773345947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6823 , TFLOPS: 68.54113886462513, Tokens per sec: 169852.49253629183, Loss: 3.4791810512542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6824 , TFLOPS: 69.71860388166775, Tokens per sec: 172770.3805570028, Loss: 3.496788263320923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6825 , TFLOPS: 69.92943557549528, Tokens per sec: 173292.84471933576, Loss: 3.4980835914611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6826 , TFLOPS: 67.80654017597112, Tokens per sec: 168032.0760630813, Loss: 3.5119576454162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6827 , TFLOPS: 69.64677196681977, Tokens per sec: 172592.37315907085, Loss: 3.5097270011901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6828 , TFLOPS: 68.97309467308445, Tokens per sec: 170922.9266135145, Loss: 3.466193914413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6829 , TFLOPS: 68.75254222048528, Tokens per sec: 170376.37334011542, Loss: 3.5183169841766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6830 , TFLOPS: 68.54228101271623, Tokens per sec: 169855.3229050796, Loss: 3.4904448986053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6831 , TFLOPS: 68.79868385178916, Tokens per sec: 170490.71738540672, Loss: 3.5183656215667725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6832 , TFLOPS: 68.86232745501394, Tokens per sec: 170648.4332451194, Loss: 3.522148609161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6833 , TFLOPS: 69.57459361741923, Tokens per sec: 172413.50725815474, Loss: 3.521300792694092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6834 , TFLOPS: 68.82979999273097, Tokens per sec: 170567.82661038646, Loss: 3.509551763534546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6835 , TFLOPS: 69.7031657464087, Tokens per sec: 172732.12315717989, Loss: 3.4951202869415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6836 , TFLOPS: 69.72241299662402, Tokens per sec: 172779.81996347307, Loss: 3.5042002201080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6837 , TFLOPS: 68.49511409220747, Tokens per sec: 169738.43808019365, Loss: 3.494370222091675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6838 , TFLOPS: 68.72645584529779, Tokens per sec: 170311.7284287494, Loss: 3.518054962158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6839 , TFLOPS: 68.84788880795662, Tokens per sec: 170612.6526871631, Loss: 3.5026397705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6840 , TFLOPS: 69.60691274604734, Tokens per sec: 172493.59761914116, Loss: 3.490227699279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6841 , TFLOPS: 67.44911402644316, Tokens per sec: 167146.33469081027, Loss: 3.508648633956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6842 , TFLOPS: 68.77501528313414, Tokens per sec: 170432.06406497152, Loss: 3.4857892990112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6843 , TFLOPS: 70.59930063206298, Tokens per sec: 174952.8441212382, Loss: 3.496208667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6844 , TFLOPS: 68.96250847037201, Tokens per sec: 170896.69283702658, Loss: 3.49741268157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6845 , TFLOPS: 68.82530654712407, Tokens per sec: 170556.69135717864, Loss: 3.4877090454101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6846 , TFLOPS: 68.85925247336378, Tokens per sec: 170640.8131018531, Loss: 3.4948978424072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6847 , TFLOPS: 69.6047452845356, Tokens per sec: 172488.22641074946, Loss: 3.4847164154052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6848 , TFLOPS: 67.90973619381553, Tokens per sec: 168287.80716327956, Loss: 3.5070273876190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6849 , TFLOPS: 68.73721022898702, Tokens per sec: 170338.37897622446, Loss: 3.487769842147827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6850 , TFLOPS: 69.53658824688056, Tokens per sec: 172319.32576332273, Loss: 3.5153284072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6851 , TFLOPS: 68.63324317618151, Tokens per sec: 170080.73716645606, Loss: 3.490821123123169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6852 , TFLOPS: 68.63765145965537, Tokens per sec: 170091.66137851682, Loss: 3.493535280227661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6853 , TFLOPS: 69.92754766085979, Tokens per sec: 173288.16625889807, Loss: 3.4906253814697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6854 , TFLOPS: 68.85597264936453, Tokens per sec: 170632.68533668053, Loss: 3.494554042816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6855 , TFLOPS: 69.81594598653224, Tokens per sec: 173011.60501597507, Loss: 3.5138332843780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6856 , TFLOPS: 69.30075081947865, Tokens per sec: 171734.89463858353, Loss: 3.5078284740448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6857 , TFLOPS: 69.58452811456176, Tokens per sec: 172438.12603644916, Loss: 3.515362501144409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6858 , TFLOPS: 68.71212086021575, Tokens per sec: 170276.2047565871, Loss: 3.499992847442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6859 , TFLOPS: 68.82077289458505, Tokens per sec: 170545.45646676386, Loss: 3.470982313156128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6860 , TFLOPS: 69.49095369134936, Tokens per sec: 172206.23830765518, Loss: 3.4983818531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6861 , TFLOPS: 68.4220652401804, Tokens per sec: 169557.41497787525, Loss: 3.5110864639282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6862 , TFLOPS: 68.7896349980593, Tokens per sec: 170468.29332905062, Loss: 3.4991016387939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6863 , TFLOPS: 69.64886062829358, Tokens per sec: 172597.549092288, Loss: 3.5007386207580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6864 , TFLOPS: 69.51537982333733, Tokens per sec: 172266.76895348192, Loss: 3.4826674461364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6865 , TFLOPS: 69.06486389193499, Tokens per sec: 171150.34084704012, Loss: 3.4945409297943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6866 , TFLOPS: 68.86994442296329, Tokens per sec: 170667.30893077617, Loss: 3.4770288467407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6867 , TFLOPS: 68.54580690331892, Tokens per sec: 169864.06045040197, Loss: 3.4726290702819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6868 , TFLOPS: 68.999332664884, Tokens per sec: 170987.94724754718, Loss: 3.502612352371216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6869 , TFLOPS: 68.68502143780766, Tokens per sec: 170209.04940261206, Loss: 3.4846479892730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6870 , TFLOPS: 69.56884685585362, Tokens per sec: 172399.26614993712, Loss: 3.4715373516082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6871 , TFLOPS: 68.82178043663109, Tokens per sec: 170547.9532669439, Loss: 3.4884848594665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6872 , TFLOPS: 70.57228937876627, Tokens per sec: 174885.90726003377, Loss: 3.5150532722473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6873 , TFLOPS: 69.08036518436393, Tokens per sec: 171188.75475728736, Loss: 3.505133628845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6874 , TFLOPS: 68.6487725335558, Tokens per sec: 170119.2206247295, Loss: 3.4955928325653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6875 , TFLOPS: 69.27452334564971, Tokens per sec: 171669.90006924066, Loss: 3.507594585418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6876 , TFLOPS: 69.4686125486479, Tokens per sec: 172150.87449496088, Loss: 3.502004623413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6877 , TFLOPS: 69.58585612656448, Tokens per sec: 172441.41699648448, Loss: 3.4818997383117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6878 , TFLOPS: 67.40277799913292, Tokens per sec: 167031.5089108007, Loss: 3.5192062854766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6879 , TFLOPS: 68.7107824178574, Tokens per sec: 170272.88794898152, Loss: 3.49837589263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6880 , TFLOPS: 70.63847033053547, Tokens per sec: 175049.91094895152, Loss: 3.487480401992798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6881 , TFLOPS: 68.71910723404673, Tokens per sec: 170293.51776055287, Loss: 3.4778780937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6882 , TFLOPS: 67.2153269653668, Tokens per sec: 166566.98459969222, Loss: 3.482089042663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6883 , TFLOPS: 70.61380212505895, Tokens per sec: 174988.78041835353, Loss: 3.5053727626800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6884 , TFLOPS: 69.49973707397946, Tokens per sec: 172228.00449738203, Loss: 3.5144271850585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6885 , TFLOPS: 67.79025431349224, Tokens per sec: 167991.7179018228, Loss: 3.484225034713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6886 , TFLOPS: 68.87553148370309, Tokens per sec: 170681.15428275493, Loss: 3.5193610191345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6887 , TFLOPS: 69.63754972526824, Tokens per sec: 172569.51942859244, Loss: 3.504457950592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6888 , TFLOPS: 68.99358602536475, Tokens per sec: 170973.7064417739, Loss: 3.5093631744384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6889 , TFLOPS: 67.86154668087418, Tokens per sec: 168168.38824169783, Loss: 3.515948534011841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6890 , TFLOPS: 70.54828582314758, Tokens per sec: 174826.4238050007, Loss: 3.4866979122161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6891 , TFLOPS: 68.9401559336493, Tokens per sec: 170841.30078869252, Loss: 3.4820938110351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6892 , TFLOPS: 69.95221048736468, Tokens per sec: 173349.28345981182, Loss: 3.4807233810424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6893 , TFLOPS: 68.52417160477988, Tokens per sec: 169810.44579729738, Loss: 3.506864070892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6894 , TFLOPS: 69.61993264505914, Tokens per sec: 172525.86236315267, Loss: 3.5021557807922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6895 , TFLOPS: 68.96257810537182, Tokens per sec: 170896.86540022705, Loss: 3.5003771781921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6896 , TFLOPS: 69.51387802819568, Tokens per sec: 172263.0473396845, Loss: 3.491447925567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6897 , TFLOPS: 68.64516773228603, Tokens per sec: 170110.28753008135, Loss: 3.4926013946533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6898 , TFLOPS: 68.83135483708537, Tokens per sec: 170571.67968597688, Loss: 3.50045108795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6899 , TFLOPS: 68.8608551410628, Tokens per sec: 170644.78468895968, Loss: 3.503859043121338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6900 , TFLOPS: 69.4088843218863, Tokens per sec: 172002.86136943832, Loss: 3.4979135990142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6901 , TFLOPS: 68.79970625888767, Tokens per sec: 170493.2510228243, Loss: 3.5002400875091553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6902 , TFLOPS: 68.77105916688089, Tokens per sec: 170422.2603730938, Loss: 3.46516752243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6903 , TFLOPS: 69.78227528692126, Tokens per sec: 172928.16531320533, Loss: 3.5053014755249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6904 , TFLOPS: 68.59937324368214, Tokens per sec: 169996.80374264152, Loss: 3.507117748260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6905 , TFLOPS: 68.76655512170129, Tokens per sec: 170411.0988529772, Loss: 3.512772560119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6906 , TFLOPS: 68.89575829771559, Tokens per sec: 170731.2785560454, Loss: 3.5074169635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6907 , TFLOPS: 69.50293247356434, Tokens per sec: 172235.92304955568, Loss: 3.486823320388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6908 , TFLOPS: 68.96226458761079, Tokens per sec: 170896.08846867835, Loss: 3.5231661796569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6909 , TFLOPS: 68.54117198010881, Tokens per sec: 169852.5746001084, Loss: 3.5163493156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6910 , TFLOPS: 70.59236196563647, Tokens per sec: 174935.6493414761, Loss: 3.4838294982910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6911 , TFLOPS: 68.62026319069534, Tokens per sec: 170048.57133255788, Loss: 3.4906997680664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6912 , TFLOPS: 69.70968744417185, Tokens per sec: 172748.2846426613, Loss: 3.4827263355255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6913 , TFLOPS: 68.86117858365618, Tokens per sec: 170645.58621533622, Loss: 3.4982833862304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6914 , TFLOPS: 69.63987627161615, Tokens per sec: 172575.28486673275, Loss: 3.530487298965454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6915 , TFLOPS: 68.00716898539724, Tokens per sec: 168529.25635392708, Loss: 3.4873039722442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6916 , TFLOPS: 69.49348839612286, Tokens per sec: 172212.51958530466, Loss: 3.4926652908325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6917 , TFLOPS: 69.55060153293677, Tokens per sec: 172354.0522298609, Loss: 3.5030674934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6918 , TFLOPS: 68.77700951819212, Tokens per sec: 170437.0059991283, Loss: 3.4936470985412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6919 , TFLOPS: 68.7683159108388, Tokens per sec: 170415.46228242686, Loss: 3.4828219413757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6920 , TFLOPS: 69.90080279678773, Tokens per sec: 173221.88953953754, Loss: 3.4977355003356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6921 , TFLOPS: 68.79271489361065, Tokens per sec: 170475.92564950537, Loss: 3.4961442947387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6922 , TFLOPS: 68.74983957072615, Tokens per sec: 170369.6758762909, Loss: 3.4995689392089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6923 , TFLOPS: 68.97918280722277, Tokens per sec: 170938.0136805726, Loss: 3.5117313861846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6924 , TFLOPS: 69.53780052776955, Tokens per sec: 172322.32992890946, Loss: 3.4935431480407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6925 , TFLOPS: 68.8954519767767, Tokens per sec: 170730.51945901485, Loss: 3.5234427452087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6926 , TFLOPS: 68.58290382166813, Tokens per sec: 169955.99070063423, Loss: 3.4833879470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6927 , TFLOPS: 70.6196818337396, Tokens per sec: 175003.35098416757, Loss: 3.500663995742798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6928 , TFLOPS: 68.22604026936524, Tokens per sec: 169071.6435062621, Loss: 3.5043821334838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6929 , TFLOPS: 68.71302508355703, Tokens per sec: 170278.4455216352, Loss: 3.4986565113067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6930 , TFLOPS: 69.6237108607458, Tokens per sec: 172535.22519782904, Loss: 3.4975881576538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6931 , TFLOPS: 69.4940004639393, Tokens per sec: 172213.7885457627, Loss: 3.5058751106262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6932 , TFLOPS: 69.14194589529505, Tokens per sec: 171341.35854265, Loss: 3.4919843673706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6933 , TFLOPS: 68.85029218729522, Tokens per sec: 170618.60852591821, Loss: 3.5122599601745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6934 , TFLOPS: 68.59778777265556, Tokens per sec: 169992.87477078364, Loss: 3.5054030418395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6935 , TFLOPS: 68.9133740322663, Tokens per sec: 170774.93228679543, Loss: 3.483438491821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6936 , TFLOPS: 69.49985545844204, Tokens per sec: 172228.2978671214, Loss: 3.500957727432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6937 , TFLOPS: 68.70933420576816, Tokens per sec: 170269.29911989067, Loss: 3.490597724914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6938 , TFLOPS: 68.84690148652761, Tokens per sec: 170610.20599589957, Loss: 3.4749109745025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6939 , TFLOPS: 70.54540195055964, Tokens per sec: 174819.27725104298, Loss: 3.4906129837036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6940 , TFLOPS: 69.04248528310399, Tokens per sec: 171094.88418914963, Loss: 3.497589111328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6941 , TFLOPS: 68.6069485508207, Tokens per sec: 170015.5761882199, Loss: 3.505059242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6942 , TFLOPS: 69.15155843324861, Tokens per sec: 171365.1794706074, Loss: 3.493647813796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6943 , TFLOPS: 68.68315269631564, Tokens per sec: 170204.41845533595, Loss: 3.4941012859344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6944 , TFLOPS: 68.6331474775139, Tokens per sec: 170080.50001461565, Loss: 3.50129771232605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6945 , TFLOPS: 68.92316849711041, Tokens per sec: 170799.20404962852, Loss: 3.488109588623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6946 , TFLOPS: 68.7690949149548, Tokens per sec: 170417.39274044076, Loss: 3.5090205669403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6947 , TFLOPS: 70.5524843456793, Tokens per sec: 174836.82820633976, Loss: 3.5344841480255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6948 , TFLOPS: 68.52923393043335, Tokens per sec: 169822.99079792787, Loss: 3.4933457374572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6949 , TFLOPS: 68.93021820993461, Tokens per sec: 170816.67401459685, Loss: 3.500617742538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6950 , TFLOPS: 69.92962828606728, Tokens per sec: 173293.32227736205, Loss: 3.4921789169311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6951 , TFLOPS: 69.54525416137544, Tokens per sec: 172340.80085407034, Loss: 3.4744420051574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6952 , TFLOPS: 68.55616182765496, Tokens per sec: 169889.72109359576, Loss: 3.495784282684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6953 , TFLOPS: 68.80800798733534, Tokens per sec: 170513.82362042848, Loss: 3.4835941791534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6954 , TFLOPS: 69.69054108122836, Tokens per sec: 172700.83784614134, Loss: 3.472287893295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6955 , TFLOPS: 68.74692876461194, Tokens per sec: 170362.46257808805, Loss: 3.5239107608795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6956 , TFLOPS: 68.19827060800479, Tokens per sec: 169002.82722633018, Loss: 3.485367774963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6957 , TFLOPS: 69.69469068907927, Tokens per sec: 172711.12103151294, Loss: 3.5044326782226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6958 , TFLOPS: 69.68705234940448, Tokens per sec: 172692.1923843662, Loss: 3.479356527328491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6959 , TFLOPS: 68.938126867953, Tokens per sec: 170836.27254037705, Loss: 3.472623586654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6960 , TFLOPS: 68.7975019781774, Tokens per sec: 170487.78857240264, Loss: 3.485924482345581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6961 , TFLOPS: 69.56246656193701, Tokens per sec: 172383.45507876438, Loss: 3.507038116455078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6962 , TFLOPS: 68.83901272404091, Tokens per sec: 170590.65677343862, Loss: 3.513653039932251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6963 , TFLOPS: 67.70127064269754, Tokens per sec: 167771.20656323296, Loss: 3.4800713062286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6964 , TFLOPS: 69.60041161751109, Tokens per sec: 172477.4871064699, Loss: 3.501683235168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6965 , TFLOPS: 69.05119487608543, Tokens per sec: 171116.46751999852, Loss: 3.48391056060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6966 , TFLOPS: 68.80645901744361, Tokens per sec: 170509.98510240368, Loss: 3.4920058250427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6967 , TFLOPS: 69.58585325479228, Tokens per sec: 172441.40987991664, Loss: 3.502819061279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6968 , TFLOPS: 68.72379896646537, Tokens per sec: 170305.1443903233, Loss: 3.483839511871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6969 , TFLOPS: 68.76017353361124, Tokens per sec: 170395.28457470008, Loss: 3.507481336593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6970 , TFLOPS: 69.66185656510878, Tokens per sec: 172629.75445533672, Loss: 3.4972643852233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6971 , TFLOPS: 68.74588510948865, Tokens per sec: 170359.87628572446, Loss: 3.5084164142608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6972 , TFLOPS: 69.04488037859663, Tokens per sec: 171100.81949962195, Loss: 3.4849021434783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6973 , TFLOPS: 68.80817932826027, Tokens per sec: 170514.24822211525, Loss: 3.494317054748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6974 , TFLOPS: 68.58460872496799, Tokens per sec: 169960.21563882192, Loss: 3.497422218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6975 , TFLOPS: 68.82024141156914, Tokens per sec: 170544.1393933028, Loss: 3.5002336502075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6976 , TFLOPS: 69.50054207069446, Tokens per sec: 172229.99936791952, Loss: 3.486309289932251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6977 , TFLOPS: 70.54289847384095, Tokens per sec: 174813.07335995336, Loss: 3.5103652477264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6978 , TFLOPS: 68.92995565549253, Tokens per sec: 170816.02337576664, Loss: 3.4932637214660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6979 , TFLOPS: 69.145611072088, Tokens per sec: 171350.44125449553, Loss: 3.4874582290649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6980 , TFLOPS: 68.81403175320887, Tokens per sec: 170528.75117583567, Loss: 3.481628179550171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6981 , TFLOPS: 68.28789568570696, Tokens per sec: 169224.9280418935, Loss: 3.500633716583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6982 , TFLOPS: 68.78131002832544, Tokens per sec: 170447.66313697898, Loss: 3.4918413162231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6983 , TFLOPS: 68.80878588767476, Tokens per sec: 170515.75134316235, Loss: 3.473144769668579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6984 , TFLOPS: 69.55696391415158, Tokens per sec: 172369.81891138537, Loss: 3.5122861862182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6985 , TFLOPS: 68.13142326660639, Tokens per sec: 168837.17215050236, Loss: 3.497610092163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6986 , TFLOPS: 69.64151539146289, Tokens per sec: 172579.34678627693, Loss: 3.518864631652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6987 , TFLOPS: 69.9637449269801, Tokens per sec: 173377.86707180223, Loss: 3.4883999824523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6988 , TFLOPS: 69.6309075937049, Tokens per sec: 172553.05949488725, Loss: 3.491323947906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6989 , TFLOPS: 67.98860313250606, Tokens per sec: 168483.2481252652, Loss: 3.492076873779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6990 , TFLOPS: 69.07389896190652, Tokens per sec: 171172.73074572472, Loss: 3.503718137741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6991 , TFLOPS: 69.61271563658445, Tokens per sec: 172507.97782113327, Loss: 3.475156784057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6992 , TFLOPS: 68.01591393775497, Tokens per sec: 168550.92730920555, Loss: 3.5022029876708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6993 , TFLOPS: 68.83286058538005, Tokens per sec: 170575.41109612296, Loss: 3.4880030155181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6994 , TFLOPS: 70.58813650378617, Tokens per sec: 174925.17818153804, Loss: 3.500297784805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6995 , TFLOPS: 68.12883141137554, Tokens per sec: 168830.7492477239, Loss: 3.499660015106201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6996 , TFLOPS: 69.4470597191729, Tokens per sec: 172097.4642092817, Loss: 3.5029897689819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6997 , TFLOPS: 70.59244967077707, Tokens per sec: 174935.86668447603, Loss: 3.5203933715820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6998 , TFLOPS: 69.57843116838603, Tokens per sec: 172423.01713219163, Loss: 3.5179197788238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6999 , TFLOPS: 68.9083362766462, Tokens per sec: 170762.44817341422, Loss: 3.4911446571350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7000 , TFLOPS: 68.60917164043373, Tokens per sec: 170021.0852491745, Loss: 3.503998041152954 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/model.pt +[rank0]:[2024-08-25 11:25:18,701] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006401355996786151, 'preprocessing_with_comm': 0.0012949700030731037, 'state_converting': 0.9990345830010483, : 1.00816841199412}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000, took 4.31s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005000 + eval ppl=24.88047218322754, eval loss=3.214083194732666 +------------------------------------------------------------------ +iteration: 7001 , TFLOPS: 65.29838520820938, Tokens per sec: 161816.59175688916, Loss: 3.5175421237945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7002 , TFLOPS: 68.72484733048918, Tokens per sec: 170307.74235186112, Loss: 3.4876556396484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7003 , TFLOPS: 68.23509217159419, Tokens per sec: 169094.07511713478, Loss: 3.492464780807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7004 , TFLOPS: 69.52252033470495, Tokens per sec: 172284.46392724314, Loss: 3.496530771255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7005 , TFLOPS: 69.7417869567178, Tokens per sec: 172827.83077080338, Loss: 3.504633665084839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7006 , TFLOPS: 69.08313513325514, Tokens per sec: 171195.61899577486, Loss: 3.5158095359802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7007 , TFLOPS: 68.651190802001, Tokens per sec: 170125.21336033093, Loss: 3.525817632675171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7008 , TFLOPS: 69.625251988758, Tokens per sec: 172539.0442828125, Loss: 3.510435104370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7009 , TFLOPS: 68.7675268221943, Tokens per sec: 170413.50683384054, Loss: 3.486895799636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7010 , TFLOPS: 68.71182614887779, Tokens per sec: 170275.4744294265, Loss: 3.510812759399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7011 , TFLOPS: 68.64490131051184, Tokens per sec: 170109.62730757613, Loss: 3.495426893234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7012 , TFLOPS: 68.97103440370421, Tokens per sec: 170917.82103903286, Loss: 3.490983486175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7013 , TFLOPS: 68.42756525579061, Tokens per sec: 169571.04462243404, Loss: 3.479536294937134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7014 , TFLOPS: 70.56984572778126, Tokens per sec: 174879.85162370253, Loss: 3.4718236923217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7015 , TFLOPS: 67.94513668657315, Tokens per sec: 168375.53348401916, Loss: 3.491652488708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7016 , TFLOPS: 68.87502088279034, Tokens per sec: 170679.8889574458, Loss: 3.502440929412842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7017 , TFLOPS: 69.72504924147107, Tokens per sec: 172786.3528686104, Loss: 3.487334728240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7018 , TFLOPS: 68.70147770287214, Tokens per sec: 170249.82984024906, Loss: 3.489567995071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7019 , TFLOPS: 68.88105900496518, Tokens per sec: 170694.85208934915, Loss: 3.50663161277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7020 , TFLOPS: 68.71177308007904, Tokens per sec: 170275.34291909722, Loss: 3.4748611450195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7021 , TFLOPS: 69.56093021368882, Tokens per sec: 172379.64783856264, Loss: 3.497302532196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7022 , TFLOPS: 68.00416075504597, Tokens per sec: 168521.8016277327, Loss: 3.4764339923858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7023 , TFLOPS: 68.87697693436898, Tokens per sec: 170684.73626873456, Loss: 3.5199530124664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7024 , TFLOPS: 69.57924289952108, Tokens per sec: 172425.0286913638, Loss: 3.4932003021240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7025 , TFLOPS: 70.58125003350274, Tokens per sec: 174908.11274956836, Loss: 3.494340181350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7026 , TFLOPS: 68.75305412313736, Tokens per sec: 170377.641891278, Loss: 3.4830336570739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7027 , TFLOPS: 68.92300687443357, Tokens per sec: 170798.8035308311, Loss: 3.496471643447876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7028 , TFLOPS: 69.58112839519691, Tokens per sec: 172429.70115735306, Loss: 3.5110902786254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7029 , TFLOPS: 68.74037279907859, Tokens per sec: 170346.2161732972, Loss: 3.498006820678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7030 , TFLOPS: 67.87636061909767, Tokens per sec: 168205.09881250563, Loss: 3.487865447998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7031 , TFLOPS: 69.56607905457741, Tokens per sec: 172392.40723347623, Loss: 3.4954521656036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7032 , TFLOPS: 69.0277990989331, Tokens per sec: 171058.49020695745, Loss: 3.4843239784240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7033 , TFLOPS: 68.65291370205233, Tokens per sec: 170129.48289645123, Loss: 3.476677417755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7034 , TFLOPS: 69.96542395039276, Tokens per sec: 173382.0278767785, Loss: 3.5086560249328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7035 , TFLOPS: 69.56588338831837, Tokens per sec: 172391.92235093188, Loss: 3.4681525230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7036 , TFLOPS: 68.2565763906485, Tokens per sec: 169147.31537863347, Loss: 3.4985435009002686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7037 , TFLOPS: 68.7212700820888, Tokens per sec: 170298.87753625851, Loss: 3.492948055267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7038 , TFLOPS: 69.60645919231027, Tokens per sec: 172492.47366301765, Loss: 3.476463556289673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7039 , TFLOPS: 69.0554328644215, Tokens per sec: 171126.96972194826, Loss: 3.5087926387786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7040 , TFLOPS: 67.68961801280825, Tokens per sec: 167742.33006272418, Loss: 3.523876428604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7041 , TFLOPS: 69.70502544002095, Tokens per sec: 172736.73168281323, Loss: 3.4834835529327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7042 , TFLOPS: 68.86288947735163, Tokens per sec: 170649.82599838407, Loss: 3.5162353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7043 , TFLOPS: 68.86748585116591, Tokens per sec: 170661.2163190282, Loss: 3.47221040725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7044 , TFLOPS: 69.62124607864179, Tokens per sec: 172529.1171962571, Loss: 3.4874520301818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7045 , TFLOPS: 69.83557826494638, Tokens per sec: 173060.2559645593, Loss: 3.495835304260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7046 , TFLOPS: 69.00922730966712, Tokens per sec: 171012.46726730486, Loss: 3.4948041439056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7047 , TFLOPS: 68.6890006345525, Tokens per sec: 170218.91029049046, Loss: 3.4797186851501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7048 , TFLOPS: 68.51571005963007, Tokens per sec: 169789.47715629998, Loss: 3.5123887062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7049 , TFLOPS: 68.76245464773471, Tokens per sec: 170400.93742678352, Loss: 3.4812655448913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7050 , TFLOPS: 68.91181622476789, Tokens per sec: 170771.0718682077, Loss: 3.5100796222686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7051 , TFLOPS: 69.60861036939923, Tokens per sec: 172497.80451681578, Loss: 3.497753620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7052 , TFLOPS: 68.08426355401618, Tokens per sec: 168720.30518763262, Loss: 3.5016589164733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7053 , TFLOPS: 69.54564434041575, Tokens per sec: 172341.767760714, Loss: 3.4768638610839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7054 , TFLOPS: 69.65576860527386, Tokens per sec: 172614.66782022163, Loss: 3.48915958404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7055 , TFLOPS: 68.28023676581795, Tokens per sec: 169205.94839470886, Loss: 3.493865728378296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7056 , TFLOPS: 67.95705207670707, Tokens per sec: 168405.0611333596, Loss: 3.506626844406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7057 , TFLOPS: 69.29680152652476, Tokens per sec: 171725.10785559326, Loss: 3.4916296005249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7058 , TFLOPS: 69.60955564845196, Tokens per sec: 172500.14702243955, Loss: 3.4972898960113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7059 , TFLOPS: 68.70943535139024, Tokens per sec: 170269.5497698843, Loss: 3.490955114364624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7060 , TFLOPS: 68.70511265411668, Tokens per sec: 170258.83764985597, Loss: 3.501805305480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7061 , TFLOPS: 69.62659826661313, Tokens per sec: 172542.38050764226, Loss: 3.498023509979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7062 , TFLOPS: 69.85518808419656, Tokens per sec: 173108.8512568612, Loss: 3.5134031772613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7063 , TFLOPS: 68.0185532489596, Tokens per sec: 168557.46781311478, Loss: 3.5058043003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7064 , TFLOPS: 70.62353689756935, Tokens per sec: 175012.90425700802, Loss: 3.4898440837860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7065 , TFLOPS: 69.59784908883739, Tokens per sec: 172471.13687813064, Loss: 3.4833054542541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7066 , TFLOPS: 67.87444237452566, Tokens per sec: 168200.3451911441, Loss: 3.5075151920318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7067 , TFLOPS: 68.69269856155618, Tokens per sec: 170228.07416096586, Loss: 3.493302822113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7068 , TFLOPS: 69.5600078779353, Tokens per sec: 172377.36218896176, Loss: 3.4939002990722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7069 , TFLOPS: 68.92370328482016, Tokens per sec: 170800.529312485, Loss: 3.5059733390808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7070 , TFLOPS: 68.80853643759181, Tokens per sec: 170515.1331783776, Loss: 3.495983600616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7071 , TFLOPS: 69.59440192413881, Tokens per sec: 172462.594424271, Loss: 3.4791948795318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7072 , TFLOPS: 69.75854055923688, Tokens per sec: 172869.3480434073, Loss: 3.4915335178375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7073 , TFLOPS: 68.97689370778669, Tokens per sec: 170932.34104000486, Loss: 3.498410940170288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7074 , TFLOPS: 68.60385774388959, Tokens per sec: 170007.9168281641, Loss: 3.4953229427337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7075 , TFLOPS: 69.57927414530559, Tokens per sec: 172425.10612185986, Loss: 3.483628988265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7076 , TFLOPS: 68.90987701104386, Tokens per sec: 170766.26628297754, Loss: 3.507415294647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7077 , TFLOPS: 68.29223991355973, Tokens per sec: 169235.69351706887, Loss: 3.5176215171813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7078 , TFLOPS: 69.56499539904391, Tokens per sec: 172389.72181568967, Loss: 3.5001394748687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7079 , TFLOPS: 68.86366941634671, Tokens per sec: 170651.75877313127, Loss: 3.5001957416534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7080 , TFLOPS: 68.59593161365811, Tokens per sec: 169988.27500431516, Loss: 3.489311456680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7081 , TFLOPS: 69.60186336929203, Tokens per sec: 172481.08470730673, Loss: 3.503119945526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7082 , TFLOPS: 69.07004245013913, Tokens per sec: 171163.17388473527, Loss: 3.5016074180603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7083 , TFLOPS: 68.84745692743141, Tokens per sec: 170611.58243964604, Loss: 3.505282402038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7084 , TFLOPS: 70.00257209897912, Tokens per sec: 173474.0850811819, Loss: 3.491276979446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7085 , TFLOPS: 69.50464467403239, Tokens per sec: 172240.16607092996, Loss: 3.510262966156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7086 , TFLOPS: 68.90209865049357, Tokens per sec: 170746.99064867682, Loss: 3.508005142211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7087 , TFLOPS: 68.68008810012323, Tokens per sec: 170196.82404837804, Loss: 3.494529962539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7088 , TFLOPS: 69.53383573307748, Tokens per sec: 172312.50473090427, Loss: 3.488851308822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7089 , TFLOPS: 68.00550660197926, Tokens per sec: 168525.1367846907, Loss: 3.5085368156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7090 , TFLOPS: 68.8855488170248, Tokens per sec: 170705.97833822653, Loss: 3.4874987602233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7091 , TFLOPS: 69.56654811532654, Tokens per sec: 172393.56961768385, Loss: 3.4917001724243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7092 , TFLOPS: 68.80511890033243, Tokens per sec: 170506.6641445761, Loss: 3.503730535507202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7093 , TFLOPS: 69.71138410884286, Tokens per sec: 172752.48916461912, Loss: 3.4981868267059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7094 , TFLOPS: 68.95046066561346, Tokens per sec: 170866.83704965963, Loss: 3.484980821609497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7095 , TFLOPS: 69.59177336291866, Tokens per sec: 172456.08056000754, Loss: 3.521014928817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7096 , TFLOPS: 68.83372048636414, Tokens per sec: 170577.54202548746, Loss: 3.512178897857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7097 , TFLOPS: 68.93036309320688, Tokens per sec: 170817.03305130574, Loss: 3.4824328422546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7098 , TFLOPS: 69.61134351759092, Tokens per sec: 172504.5775591159, Loss: 3.5001282691955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7099 , TFLOPS: 68.87006706811833, Tokens per sec: 170667.61285898086, Loss: 3.489865779876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7100 , TFLOPS: 68.58061780566324, Tokens per sec: 169950.32570114866, Loss: 3.5048930644989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7101 , TFLOPS: 69.8519781846275, Tokens per sec: 173100.89677212943, Loss: 3.4940130710601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7102 , TFLOPS: 68.87407974128381, Tokens per sec: 170677.5567051173, Loss: 3.487281322479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7103 , TFLOPS: 68.78666909339368, Tokens per sec: 170460.94349056785, Loss: 3.477736234664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7104 , TFLOPS: 68.81387636622739, Tokens per sec: 170528.3661097783, Loss: 3.4786767959594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7105 , TFLOPS: 69.57827584881734, Tokens per sec: 172422.63223319055, Loss: 3.4796805381774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7106 , TFLOPS: 68.90222178685347, Tokens per sec: 170747.2957941412, Loss: 3.5119380950927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7107 , TFLOPS: 68.62581116466276, Tokens per sec: 170062.31982320896, Loss: 3.490044116973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7108 , TFLOPS: 69.55234443548608, Tokens per sec: 172358.37133437468, Loss: 3.500419855117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7109 , TFLOPS: 68.86363181447211, Tokens per sec: 170651.6655915438, Loss: 3.507516622543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7110 , TFLOPS: 68.77170246628594, Tokens per sec: 170423.8545398848, Loss: 3.5008370876312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7111 , TFLOPS: 68.79796980577093, Tokens per sec: 170488.9479007149, Loss: 3.515456438064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7112 , TFLOPS: 70.56116980878828, Tokens per sec: 174858.35174070657, Loss: 3.4904279708862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7113 , TFLOPS: 69.1794724464486, Tokens per sec: 171434.35347029936, Loss: 3.521070718765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7114 , TFLOPS: 67.71895254281176, Tokens per sec: 167815.02425953813, Loss: 3.4850757122039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7115 , TFLOPS: 69.54157211765163, Tokens per sec: 172331.67634411217, Loss: 3.482598066329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7116 , TFLOPS: 68.98304116041226, Tokens per sec: 170947.57510480884, Loss: 3.5025784969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7117 , TFLOPS: 68.79882057448162, Tokens per sec: 170491.0561992999, Loss: 3.5155506134033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7118 , TFLOPS: 68.50975170444546, Tokens per sec: 169774.7116957855, Loss: 3.520094633102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7119 , TFLOPS: 68.90909868883261, Tokens per sec: 170764.33751479883, Loss: 3.4880101680755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7120 , TFLOPS: 69.66360533229428, Tokens per sec: 172634.08809306502, Loss: 3.523573398590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7121 , TFLOPS: 69.6413828105033, Tokens per sec: 172579.0182360541, Loss: 3.49914288520813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7122 , TFLOPS: 69.52518715304355, Tokens per sec: 172291.07259686047, Loss: 3.496661424636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7123 , TFLOPS: 68.3815072791712, Tokens per sec: 169456.90788266622, Loss: 3.486844301223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7124 , TFLOPS: 69.75062309665162, Tokens per sec: 172849.72769894055, Loss: 3.5116240978240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7125 , TFLOPS: 69.64929492516006, Tokens per sec: 172598.62532776492, Loss: 3.4907760620117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7126 , TFLOPS: 67.70424995307745, Tokens per sec: 167778.58962255254, Loss: 3.5067896842956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7127 , TFLOPS: 68.71810339173348, Tokens per sec: 170291.03012871827, Loss: 3.500892400741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7128 , TFLOPS: 69.54661102332753, Tokens per sec: 172344.1633074583, Loss: 3.4764256477355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7129 , TFLOPS: 68.93784963965088, Tokens per sec: 170835.58553810514, Loss: 3.4951012134552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7130 , TFLOPS: 68.21729607849602, Tokens per sec: 169049.97443803604, Loss: 3.5094783306121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7131 , TFLOPS: 70.55458715098136, Tokens per sec: 174842.0391895214, Loss: 3.4968881607055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7132 , TFLOPS: 69.9419742391627, Tokens per sec: 173323.9169091515, Loss: 3.4817848205566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7133 , TFLOPS: 67.37195379048525, Tokens per sec: 166955.12312620488, Loss: 3.4773366451263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7134 , TFLOPS: 68.71328745959237, Tokens per sec: 170279.0957183539, Loss: 3.4971537590026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7135 , TFLOPS: 69.587916970942, Tokens per sec: 172446.52399587276, Loss: 3.5032241344451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7136 , TFLOPS: 68.9335309328782, Tokens per sec: 170824.88330697763, Loss: 3.495666027069092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7137 , TFLOPS: 67.61647165838305, Tokens per sec: 167561.06533872237, Loss: 3.5069596767425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7138 , TFLOPS: 69.64807684785985, Tokens per sec: 172595.6067980327, Loss: 3.5060877799987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7139 , TFLOPS: 69.75116026190226, Tokens per sec: 172851.05885360527, Loss: 3.5119833946228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7140 , TFLOPS: 66.34546170254505, Tokens per sec: 164411.3626548508, Loss: 3.4820480346679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7141 , TFLOPS: 69.60819898337456, Tokens per sec: 172496.78505692873, Loss: 3.497567892074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7142 , TFLOPS: 69.55956966740327, Tokens per sec: 172376.2762549905, Loss: 3.467730760574341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7143 , TFLOPS: 68.92076461198175, Tokens per sec: 170793.24695746994, Loss: 3.522686243057251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7144 , TFLOPS: 68.24789236797328, Tokens per sec: 169125.7954138202, Loss: 3.5005364418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7145 , TFLOPS: 69.66416501862284, Tokens per sec: 172635.47505744168, Loss: 3.5045583248138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7146 , TFLOPS: 68.92685036687509, Tokens per sec: 170808.32812849683, Loss: 3.510538339614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7147 , TFLOPS: 68.77131756437868, Tokens per sec: 170422.90071055814, Loss: 3.5011210441589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7148 , TFLOPS: 69.55180238793122, Tokens per sec: 172357.02808082226, Loss: 3.461456298828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7149 , TFLOPS: 68.98440757858931, Tokens per sec: 170950.96123957494, Loss: 3.5016679763793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7150 , TFLOPS: 68.79550687640483, Tokens per sec: 170482.84449043163, Loss: 3.5005264282226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7151 , TFLOPS: 69.04130086151706, Tokens per sec: 171091.94906198242, Loss: 3.49835205078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7152 , TFLOPS: 69.62318636891541, Tokens per sec: 172533.92544929576, Loss: 3.4999923706054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7153 , TFLOPS: 68.94242501733143, Tokens per sec: 170846.92382801702, Loss: 3.5176219940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7154 , TFLOPS: 68.75451500044073, Tokens per sec: 170381.2621061647, Loss: 3.4939703941345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7155 , TFLOPS: 69.22722158220891, Tokens per sec: 171552.68108871498, Loss: 3.536269187927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7156 , TFLOPS: 68.92701214214377, Tokens per sec: 170808.72902543363, Loss: 3.4769625663757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7157 , TFLOPS: 68.70090784606236, Tokens per sec: 170248.41767229952, Loss: 3.482029914855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7158 , TFLOPS: 69.62380715352904, Tokens per sec: 172535.4638219534, Loss: 3.501244068145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7159 , TFLOPS: 68.7516923906047, Tokens per sec: 170374.26736805486, Loss: 3.4907240867614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7160 , TFLOPS: 69.66795903800654, Tokens per sec: 172644.87705542028, Loss: 3.492380380630493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7161 , TFLOPS: 68.602116264425, Tokens per sec: 170003.60125021104, Loss: 3.488715648651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7162 , TFLOPS: 69.5043225858046, Tokens per sec: 172239.36790082054, Loss: 3.4941508769989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7163 , TFLOPS: 68.68241736359828, Tokens per sec: 170202.5962198587, Loss: 3.5123414993286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7164 , TFLOPS: 68.73447918514657, Tokens per sec: 170331.61114873848, Loss: 3.500805377960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7165 , TFLOPS: 69.58900702089961, Tokens per sec: 172449.22525974663, Loss: 3.499666213989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7166 , TFLOPS: 67.84201629139635, Tokens per sec: 168119.98978512175, Loss: 3.4799187183380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7167 , TFLOPS: 69.71042726892995, Tokens per sec: 172750.1180099103, Loss: 3.5242209434509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7168 , TFLOPS: 69.94899783877858, Tokens per sec: 173341.32216271298, Loss: 3.5138347148895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7169 , TFLOPS: 68.70869515785888, Tokens per sec: 170267.71548877453, Loss: 3.4940872192382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7170 , TFLOPS: 67.87554743533072, Tokens per sec: 168203.08365355156, Loss: 3.500009298324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7171 , TFLOPS: 69.63323583933384, Tokens per sec: 172558.8291440332, Loss: 3.527643918991089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7172 , TFLOPS: 69.65612063661396, Tokens per sec: 172615.5401926619, Loss: 3.519171714782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7173 , TFLOPS: 68.85986273679323, Tokens per sec: 170642.32540185738, Loss: 3.510683536529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7174 , TFLOPS: 67.75420612179073, Tokens per sec: 167902.38651175032, Loss: 3.4932708740234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7175 , TFLOPS: 69.56792792091201, Tokens per sec: 172396.9889279228, Loss: 3.4907684326171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7176 , TFLOPS: 68.95195440571194, Tokens per sec: 170870.53870217272, Loss: 3.4891011714935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7177 , TFLOPS: 68.73531799706518, Tokens per sec: 170333.68981707556, Loss: 3.485914468765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7178 , TFLOPS: 68.70177409951329, Tokens per sec: 170250.5643437766, Loss: 3.50895619392395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7179 , TFLOPS: 70.55990857852787, Tokens per sec: 174855.22627318525, Loss: 3.506222724914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7180 , TFLOPS: 69.09032644351132, Tokens per sec: 171213.43985477788, Loss: 3.4950754642486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7181 , TFLOPS: 68.66609545119277, Tokens per sec: 170162.14872291082, Loss: 3.504457712173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7182 , TFLOPS: 69.56231578293519, Tokens per sec: 172383.0814317882, Loss: 3.4973602294921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7183 , TFLOPS: 68.95545273527809, Tokens per sec: 170879.20794821045, Loss: 3.4847774505615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7184 , TFLOPS: 68.74132453471533, Tokens per sec: 170348.57467904742, Loss: 3.4837915897369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7185 , TFLOPS: 68.69375924826696, Tokens per sec: 170230.7026594795, Loss: 3.511277437210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7186 , TFLOPS: 68.84597610880128, Tokens per sec: 170607.91280795494, Loss: 3.498750925064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7187 , TFLOPS: 69.69087072284282, Tokens per sec: 172701.65473437498, Loss: 3.5011630058288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7188 , TFLOPS: 68.70222083443865, Tokens per sec: 170251.67140213356, Loss: 3.4802045822143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7189 , TFLOPS: 69.58600328519313, Tokens per sec: 172441.78167177716, Loss: 3.5011038780212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7190 , TFLOPS: 69.18200851664518, Tokens per sec: 171440.63813161768, Loss: 3.481508255004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7191 , TFLOPS: 68.93175417019123, Tokens per sec: 170820.48029331275, Loss: 3.5018768310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7192 , TFLOPS: 68.68183699748938, Tokens per sec: 170201.1580087083, Loss: 3.4837775230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7193 , TFLOPS: 68.93413879782302, Tokens per sec: 170826.38966327274, Loss: 3.513031482696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7194 , TFLOPS: 68.85511612346782, Tokens per sec: 170630.56277115436, Loss: 3.4990592002868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7195 , TFLOPS: 69.55280717990263, Tokens per sec: 172359.5180660144, Loss: 3.462834358215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7196 , TFLOPS: 68.36781900639654, Tokens per sec: 169422.98683484286, Loss: 3.499645471572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7197 , TFLOPS: 69.05788321062113, Tokens per sec: 171133.04194975883, Loss: 3.481175422668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7198 , TFLOPS: 69.90239329977034, Tokens per sec: 173225.8309811368, Loss: 3.5039892196655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7199 , TFLOPS: 69.81510534828423, Tokens per sec: 173009.5218217926, Loss: 3.5162370204925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7200 , TFLOPS: 68.87384846274871, Tokens per sec: 170676.98357142994, Loss: 3.504263401031494 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/model.pt +[rank0]:[2024-08-25 11:50:59,232] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006362084997817874, 'preprocessing_with_comm': 0.001245090999873355, 'state_converting': 0.9817123500033631, : 0.9907006129942602}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200, took 4.49s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005200 + eval ppl=23.766551971435547, eval loss=3.1682791709899902 +------------------------------------------------------------------ +iteration: 7201 , TFLOPS: 64.65187947778878, Tokens per sec: 160214.4793384215, Loss: 3.506098747253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7202 , TFLOPS: 69.57503539107454, Tokens per sec: 172414.602021938, Loss: 3.484001398086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7203 , TFLOPS: 68.21046098654246, Tokens per sec: 169033.03632722757, Loss: 3.5045876502990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7204 , TFLOPS: 68.66376721448518, Tokens per sec: 170156.37909587295, Loss: 3.5042405128479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7205 , TFLOPS: 69.56693011458798, Tokens per sec: 172394.5162539344, Loss: 3.4859094619750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7206 , TFLOPS: 69.83116681632764, Tokens per sec: 173049.3239089209, Loss: 3.5079047679901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7207 , TFLOPS: 67.71535122610574, Tokens per sec: 167806.09980002278, Loss: 3.5132577419281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7208 , TFLOPS: 69.77800882093847, Tokens per sec: 172917.59254050994, Loss: 3.4951136112213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7209 , TFLOPS: 69.640995577987, Tokens per sec: 172578.0586312215, Loss: 3.4872143268585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7210 , TFLOPS: 68.00804839905113, Tokens per sec: 168531.43563783684, Loss: 3.5000290870666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7211 , TFLOPS: 68.52286408277888, Tokens per sec: 169807.20561374343, Loss: 3.4806737899780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7212 , TFLOPS: 69.53347658062783, Tokens per sec: 172311.61471157018, Loss: 3.5160329341888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7213 , TFLOPS: 68.79883435135315, Tokens per sec: 170491.0903399052, Loss: 3.470452070236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7214 , TFLOPS: 68.86027835627677, Tokens per sec: 170643.35535272185, Loss: 3.4816436767578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7215 , TFLOPS: 69.49958265292315, Tokens per sec: 172227.62182499358, Loss: 3.495431900024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7216 , TFLOPS: 68.78817336088773, Tokens per sec: 170464.67123112682, Loss: 3.465841054916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7217 , TFLOPS: 67.91929579432937, Tokens per sec: 168311.4969064298, Loss: 3.501896619796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7218 , TFLOPS: 70.57286390248679, Tokens per sec: 174887.33099310222, Loss: 3.501729965209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7219 , TFLOPS: 68.53798380213684, Tokens per sec: 169844.67394388703, Loss: 3.4923312664031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7220 , TFLOPS: 69.61962280773756, Tokens per sec: 172525.09455213838, Loss: 3.50024151802063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7221 , TFLOPS: 69.0462989444623, Tokens per sec: 171104.33486210502, Loss: 3.4586496353149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7222 , TFLOPS: 68.57206859087727, Tokens per sec: 169929.13980513526, Loss: 3.5317773818969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7223 , TFLOPS: 68.90506408818338, Tokens per sec: 170754.33932994207, Loss: 3.490603446960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7224 , TFLOPS: 68.4348127032285, Tokens per sec: 169589.00459555743, Loss: 3.491651773452759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7225 , TFLOPS: 69.5947473433122, Tokens per sec: 172463.45041103376, Loss: 3.5148632526397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7226 , TFLOPS: 69.5391785447209, Tokens per sec: 172325.744806716, Loss: 3.5092177391052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7227 , TFLOPS: 69.16380524976023, Tokens per sec: 171395.5284310214, Loss: 3.5047059059143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7228 , TFLOPS: 69.7489731304878, Tokens per sec: 172845.63890102782, Loss: 3.5205559730529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7229 , TFLOPS: 68.64483379309854, Tokens per sec: 170109.45999198832, Loss: 3.494345188140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7230 , TFLOPS: 68.00159036440706, Tokens per sec: 168515.43191657757, Loss: 3.4795498847961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7231 , TFLOPS: 69.6545309196732, Tokens per sec: 172611.60069896092, Loss: 3.5135738849639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7232 , TFLOPS: 69.57886056389978, Tokens per sec: 172424.08122157652, Loss: 3.499342679977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7233 , TFLOPS: 68.73032099740249, Tokens per sec: 170321.30670144648, Loss: 3.496736526489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7234 , TFLOPS: 68.79939260202636, Tokens per sec: 170492.4737465698, Loss: 3.493631601333618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7235 , TFLOPS: 70.6029680084993, Tokens per sec: 174961.9322840423, Loss: 3.4956681728363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7236 , TFLOPS: 68.79672744384726, Tokens per sec: 170485.86919101057, Loss: 3.48639178276062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7237 , TFLOPS: 67.81110738414438, Tokens per sec: 168043.39410805414, Loss: 3.5119621753692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7238 , TFLOPS: 69.91634669116756, Tokens per sec: 173260.40902211206, Loss: 3.471487045288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7239 , TFLOPS: 69.61425316565759, Tokens per sec: 172511.78798754932, Loss: 3.5192174911499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7240 , TFLOPS: 69.6601508076324, Tokens per sec: 172625.52740040552, Loss: 3.477932929992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7241 , TFLOPS: 68.67729302325789, Tokens per sec: 170189.89754000254, Loss: 3.4898667335510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7242 , TFLOPS: 69.60613392686054, Tokens per sec: 172491.6676194022, Loss: 3.4905292987823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7243 , TFLOPS: 69.65488901436818, Tokens per sec: 172612.48809706242, Loss: 3.5081279277801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7244 , TFLOPS: 67.27817735531755, Tokens per sec: 166722.7347895322, Loss: 3.510709285736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7245 , TFLOPS: 70.57600559033817, Tokens per sec: 174895.11644168344, Loss: 3.503103256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7246 , TFLOPS: 69.86078770688205, Tokens per sec: 173122.7277387252, Loss: 3.491002321243286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7247 , TFLOPS: 68.89595232838526, Tokens per sec: 170731.75938541914, Loss: 3.5069925785064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7248 , TFLOPS: 68.78185860317562, Tokens per sec: 170449.0225658882, Loss: 3.476099729537964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7249 , TFLOPS: 69.61428041533108, Tokens per sec: 172511.85551524203, Loss: 3.5002384185791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7250 , TFLOPS: 68.75729380187988, Tokens per sec: 170388.14828224116, Loss: 3.505120277404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7251 , TFLOPS: 68.94364085618402, Tokens per sec: 170849.93681062947, Loss: 3.513683795928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7252 , TFLOPS: 68.52076524669893, Tokens per sec: 169802.00446673142, Loss: 3.4974679946899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7253 , TFLOPS: 68.92325128717235, Tokens per sec: 170799.4092125221, Loss: 3.5214955806732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7254 , TFLOPS: 69.67469497288279, Tokens per sec: 172661.5694440681, Loss: 3.51271915435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7255 , TFLOPS: 68.97170393078417, Tokens per sec: 170919.48020089153, Loss: 3.5039491653442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7256 , TFLOPS: 68.79388697548617, Tokens per sec: 170478.83019750836, Loss: 3.5120151042938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7257 , TFLOPS: 68.72303414611716, Tokens per sec: 170303.24908124848, Loss: 3.487931251525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7258 , TFLOPS: 69.64492359807676, Tokens per sec: 172587.7926977111, Loss: 3.4915919303894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7259 , TFLOPS: 69.48942320909036, Tokens per sec: 172202.44560402085, Loss: 3.483191967010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7260 , TFLOPS: 69.74694701457219, Tokens per sec: 172840.61796259385, Loss: 3.4847686290740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7261 , TFLOPS: 68.90134715355278, Tokens per sec: 170745.1283564733, Loss: 3.506866693496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7262 , TFLOPS: 70.58011628509165, Tokens per sec: 174905.30319611297, Loss: 3.4981563091278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7263 , TFLOPS: 67.97228574982742, Tokens per sec: 168442.81185347945, Loss: 3.500552177429199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7264 , TFLOPS: 67.90178710911242, Tokens per sec: 168268.10845572132, Loss: 3.484894037246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7265 , TFLOPS: 70.60089516633013, Tokens per sec: 174956.79555280466, Loss: 3.521418809890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7266 , TFLOPS: 68.65285540945904, Tokens per sec: 170129.3384409836, Loss: 3.494504451751709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7267 , TFLOPS: 68.9240944706551, Tokens per sec: 170801.49871407668, Loss: 3.513723850250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7268 , TFLOPS: 69.6827158931725, Tokens per sec: 172681.4461681233, Loss: 3.52022647857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7269 , TFLOPS: 69.5831844051224, Tokens per sec: 172434.7961764366, Loss: 3.4944345951080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7270 , TFLOPS: 67.88842288394828, Tokens per sec: 168234.99043357364, Loss: 3.5013492107391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7271 , TFLOPS: 68.68467402108416, Tokens per sec: 170208.18846570014, Loss: 3.4986181259155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7272 , TFLOPS: 69.53432509874693, Tokens per sec: 172313.71743294128, Loss: 3.5194456577301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7273 , TFLOPS: 69.65919763747445, Tokens per sec: 172623.16533975376, Loss: 3.5087411403656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7274 , TFLOPS: 68.15965276147197, Tokens per sec: 168907.12794851488, Loss: 3.5169284343719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7275 , TFLOPS: 69.66761764035883, Tokens per sec: 172644.03103444117, Loss: 3.5183448791503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7276 , TFLOPS: 69.64543756095857, Tokens per sec: 172589.0663543499, Loss: 3.472604274749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7277 , TFLOPS: 68.27054559520626, Tokens per sec: 169181.9326063625, Loss: 3.503661870956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7278 , TFLOPS: 69.58130931449729, Tokens per sec: 172430.1494953093, Loss: 3.506493091583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7279 , TFLOPS: 69.60461069322899, Tokens per sec: 172487.8928786652, Loss: 3.5109260082244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7280 , TFLOPS: 69.70671591217906, Tokens per sec: 172740.92085904191, Loss: 3.464273691177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7281 , TFLOPS: 67.79204463455696, Tokens per sec: 167996.15451464133, Loss: 3.500375747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7282 , TFLOPS: 70.61192884661222, Tokens per sec: 174984.1382280039, Loss: 3.4770348072052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7283 , TFLOPS: 68.91742798780022, Tokens per sec: 170784.9784351886, Loss: 3.4888827800750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7284 , TFLOPS: 67.90118820537994, Tokens per sec: 168266.62430629754, Loss: 3.4920167922973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7285 , TFLOPS: 69.57886742203448, Tokens per sec: 172424.0982167898, Loss: 3.4958648681640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7286 , TFLOPS: 69.56170323241831, Tokens per sec: 172381.56346412937, Loss: 3.4765233993530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7287 , TFLOPS: 69.55270424878096, Tokens per sec: 172359.2629913561, Loss: 3.4843482971191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7288 , TFLOPS: 69.08675494235403, Tokens per sec: 171204.5892814768, Loss: 3.515045166015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7289 , TFLOPS: 68.67162089299966, Tokens per sec: 170175.84137638594, Loss: 3.5110161304473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7290 , TFLOPS: 68.94963009956042, Tokens per sec: 170864.77881548583, Loss: 3.4910624027252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7291 , TFLOPS: 68.69919121002255, Tokens per sec: 170244.1636590898, Loss: 3.5061521530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7292 , TFLOPS: 70.58672895707647, Tokens per sec: 174921.6901257373, Loss: 3.4920883178710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7293 , TFLOPS: 69.52947270458911, Tokens per sec: 172301.69266568284, Loss: 3.4864611625671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7294 , TFLOPS: 68.12802048667508, Tokens per sec: 168828.73968698547, Loss: 3.495972156524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7295 , TFLOPS: 69.59312076135882, Tokens per sec: 172459.41956177042, Loss: 3.502774238586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7296 , TFLOPS: 68.74614761143994, Tokens per sec: 170360.5267944767, Loss: 3.5039122104644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7297 , TFLOPS: 68.08406926163123, Tokens per sec: 168719.82370969967, Loss: 3.5120129585266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7298 , TFLOPS: 69.77422807546438, Tokens per sec: 172908.22343674165, Loss: 3.508669853210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7299 , TFLOPS: 69.55625241837471, Tokens per sec: 172368.0557464726, Loss: 3.503361701965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7300 , TFLOPS: 68.67647572441744, Tokens per sec: 170187.87218344302, Loss: 3.497246265411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7301 , TFLOPS: 68.73596094931779, Tokens per sec: 170335.28312358458, Loss: 3.5098845958709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7302 , TFLOPS: 70.5924709241825, Tokens per sec: 174935.91935275594, Loss: 3.489255428314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7303 , TFLOPS: 68.66621869832225, Tokens per sec: 170162.45414287449, Loss: 3.5079030990600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7304 , TFLOPS: 68.87396389128803, Tokens per sec: 170677.26961606587, Loss: 3.534463882446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7305 , TFLOPS: 70.00946985250957, Tokens per sec: 173491.17847428063, Loss: 3.506173610687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7306 , TFLOPS: 69.54148332313468, Tokens per sec: 172331.45630151764, Loss: 3.4933104515075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7307 , TFLOPS: 68.6618399769676, Tokens per sec: 170151.60318900167, Loss: 3.5076804161071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7308 , TFLOPS: 68.68651380421825, Tokens per sec: 170212.74765097522, Loss: 3.5088534355163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7309 , TFLOPS: 69.64495566325405, Tokens per sec: 172587.8721587527, Loss: 3.512144088745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7310 , TFLOPS: 69.78337417542575, Tokens per sec: 172930.88847997203, Loss: 3.497556686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7311 , TFLOPS: 67.4440579105223, Tokens per sec: 167133.8050785812, Loss: 3.4841628074645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7312 , TFLOPS: 70.60018969114554, Tokens per sec: 174955.04730758275, Loss: 3.513106346130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7313 , TFLOPS: 69.55964022998687, Tokens per sec: 172376.45111684577, Loss: 3.4735355377197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7314 , TFLOPS: 67.82428570674423, Tokens per sec: 168076.0514431687, Loss: 3.466254472732544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7315 , TFLOPS: 68.60883522403459, Tokens per sec: 170020.25157227812, Loss: 3.490086078643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7316 , TFLOPS: 69.63921719565353, Tokens per sec: 172573.6516038942, Loss: 3.4835262298583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7317 , TFLOPS: 68.14767454053815, Tokens per sec: 168877.4445975309, Loss: 3.496291160583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7318 , TFLOPS: 68.800111577104, Tokens per sec: 170494.25544601382, Loss: 3.491241931915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7319 , TFLOPS: 69.5747393294774, Tokens per sec: 172413.8683486864, Loss: 3.4901034832000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7320 , TFLOPS: 69.67200057282497, Tokens per sec: 172654.89242391213, Loss: 3.5252296924591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7321 , TFLOPS: 69.58329497132081, Tokens per sec: 172435.07017165615, Loss: 3.522310256958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7322 , TFLOPS: 68.85986050674317, Tokens per sec: 170642.31987554766, Loss: 3.5140304565429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7323 , TFLOPS: 68.89485807138767, Tokens per sec: 170729.04769603672, Loss: 3.515979290008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7324 , TFLOPS: 68.8505346609397, Tokens per sec: 170619.20940231992, Loss: 3.480433702468872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7325 , TFLOPS: 69.68403822708846, Tokens per sec: 172684.72305723414, Loss: 3.4753990173339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7326 , TFLOPS: 68.58613901493958, Tokens per sec: 169964.00786594683, Loss: 3.501758337020874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7327 , TFLOPS: 69.66672531023794, Tokens per sec: 172641.81974210328, Loss: 3.5185794830322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7328 , TFLOPS: 68.80105503441685, Tokens per sec: 170496.59343716546, Loss: 3.4954092502593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7329 , TFLOPS: 69.53050209988601, Tokens per sec: 172304.2436206258, Loss: 3.5170836448669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7330 , TFLOPS: 69.13495971722656, Tokens per sec: 171324.04602380525, Loss: 3.49605131149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7331 , TFLOPS: 67.92054338627972, Tokens per sec: 168314.58857671637, Loss: 3.493197441101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7332 , TFLOPS: 70.56345671182012, Tokens per sec: 174864.0189383426, Loss: 3.5040383338928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7333 , TFLOPS: 68.47921568229377, Tokens per sec: 169699.04006907306, Loss: 3.5031096935272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7334 , TFLOPS: 67.98701059256064, Tokens per sec: 168479.3016358477, Loss: 3.505831241607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7335 , TFLOPS: 69.59610603659631, Tokens per sec: 172466.81740266416, Loss: 3.4967658519744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7336 , TFLOPS: 69.63065212888166, Tokens per sec: 172552.42642491352, Loss: 3.488588571548462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7337 , TFLOPS: 68.03840906524715, Tokens per sec: 168606.67271317408, Loss: 3.4789183139801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7338 , TFLOPS: 68.5722599056125, Tokens per sec: 169929.6139041242, Loss: 3.4608092308044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7339 , TFLOPS: 69.56957042047068, Tokens per sec: 172401.05922276544, Loss: 3.4841837882995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7340 , TFLOPS: 68.62877829256026, Tokens per sec: 170069.67269299505, Loss: 3.5175278186798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7341 , TFLOPS: 69.7340246363859, Tokens per sec: 172808.59488593074, Loss: 3.525944948196411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7342 , TFLOPS: 69.66676780948805, Tokens per sec: 172641.92505992678, Loss: 3.4886474609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7343 , TFLOPS: 69.56666914071445, Tokens per sec: 172393.86953192714, Loss: 3.482099771499634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7344 , TFLOPS: 68.17358543426732, Tokens per sec: 168941.65464651235, Loss: 3.4828059673309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7345 , TFLOPS: 69.61526223432267, Tokens per sec: 172514.28857085947, Loss: 3.5208263397216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7346 , TFLOPS: 69.58310989889655, Tokens per sec: 172434.61154180142, Loss: 3.4806101322174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7347 , TFLOPS: 69.65554402480701, Tokens per sec: 172614.1112850852, Loss: 3.5066564083099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7348 , TFLOPS: 67.7448274847338, Tokens per sec: 167879.1452159811, Loss: 3.4813029766082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7349 , TFLOPS: 70.44548525133138, Tokens per sec: 174571.6726636223, Loss: 3.480050802230835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7350 , TFLOPS: 68.68779571191504, Tokens per sec: 170215.92435946167, Loss: 3.5028786659240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7351 , TFLOPS: 68.77434194864652, Tokens per sec: 170430.39546793728, Loss: 3.5262866020202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7352 , TFLOPS: 68.9873035976036, Tokens per sec: 170958.1378937152, Loss: 3.4952423572540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7353 , TFLOPS: 69.61420590656724, Tokens per sec: 172511.6708743174, Loss: 3.4913723468780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7354 , TFLOPS: 68.75969857581603, Tokens per sec: 170394.10757696265, Loss: 3.490380048751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7355 , TFLOPS: 67.85729845463145, Tokens per sec: 168157.8606690876, Loss: 3.4965574741363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7356 , TFLOPS: 69.53318516474975, Tokens per sec: 172310.89255092217, Loss: 3.5081052780151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7357 , TFLOPS: 68.87154860974688, Tokens per sec: 170671.28428234166, Loss: 3.4913196563720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7358 , TFLOPS: 68.85099534865714, Tokens per sec: 170620.3510372327, Loss: 3.5088844299316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7359 , TFLOPS: 68.7221765177034, Tokens per sec: 170301.1237835636, Loss: 3.499563455581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7360 , TFLOPS: 70.55339608842161, Tokens per sec: 174839.08760529192, Loss: 3.4875259399414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7361 , TFLOPS: 69.03777750364749, Tokens per sec: 171083.21779305174, Loss: 3.4831526279449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7362 , TFLOPS: 69.65260677834748, Tokens per sec: 172606.8324647941, Loss: 3.5119705200195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7363 , TFLOPS: 68.5650434419175, Tokens per sec: 169911.7307121289, Loss: 3.4703476428985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7364 , TFLOPS: 68.13994146882509, Tokens per sec: 168858.28119395775, Loss: 3.5004241466522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7365 , TFLOPS: 69.67587268133732, Tokens per sec: 172664.4879353537, Loss: 3.511334180831909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7366 , TFLOPS: 69.63814706993647, Tokens per sec: 172570.99971448325, Loss: 3.491691827774048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7367 , TFLOPS: 68.78541533418195, Tokens per sec: 170457.8365371282, Loss: 3.496212959289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7368 , TFLOPS: 68.74793820744159, Tokens per sec: 170364.96408861925, Loss: 3.4819185733795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7369 , TFLOPS: 70.56569990593015, Tokens per sec: 174869.5778204555, Loss: 3.5009496212005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7370 , TFLOPS: 67.75999860901811, Tokens per sec: 167916.74093319493, Loss: 3.487058639526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7371 , TFLOPS: 68.88138291515695, Tokens per sec: 170695.654774486, Loss: 3.4882652759552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7372 , TFLOPS: 69.62101782981054, Tokens per sec: 172528.5515705096, Loss: 3.5061745643615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7373 , TFLOPS: 69.62833370764683, Tokens per sec: 172546.6811216984, Loss: 3.4894909858703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7374 , TFLOPS: 67.94097918593886, Tokens per sec: 168365.2307394609, Loss: 3.4668569564819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7375 , TFLOPS: 68.72438063963014, Tokens per sec: 170306.58584050217, Loss: 3.480306386947632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7376 , TFLOPS: 69.5848782647346, Tokens per sec: 172438.99374714968, Loss: 3.5030357837677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7377 , TFLOPS: 68.64228408912564, Tokens per sec: 170103.1415446704, Loss: 3.5162105560302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7378 , TFLOPS: 68.31164968760329, Tokens per sec: 169283.79307531193, Loss: 3.506923198699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7379 , TFLOPS: 70.55310031997038, Tokens per sec: 174838.3546584882, Loss: 3.4893229007720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7380 , TFLOPS: 69.87313570851998, Tokens per sec: 173153.32744702869, Loss: 3.5241754055023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7381 , TFLOPS: 68.64094025883547, Tokens per sec: 170099.81138517623, Loss: 3.4875988960266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7382 , TFLOPS: 69.61568220779029, Tokens per sec: 172515.32931137612, Loss: 3.4962124824523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7383 , TFLOPS: 69.59804791757347, Tokens per sec: 172471.6295976414, Loss: 3.4886648654937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7384 , TFLOPS: 68.11133076741767, Tokens per sec: 168787.38072414123, Loss: 3.5018012523651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7385 , TFLOPS: 68.71291614216476, Tokens per sec: 170278.1755528643, Loss: 3.5010411739349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7386 , TFLOPS: 69.56323693696625, Tokens per sec: 172385.36415295064, Loss: 3.487321615219116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7387 , TFLOPS: 69.73687154146707, Tokens per sec: 172815.64983033438, Loss: 3.508589744567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7388 , TFLOPS: 69.67660743295352, Tokens per sec: 172666.3087307982, Loss: 3.480198860168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7389 , TFLOPS: 68.87725885262319, Tokens per sec: 170685.43489322436, Loss: 3.492061138153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7390 , TFLOPS: 69.61167620891575, Tokens per sec: 172505.40200486782, Loss: 3.4978082180023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7391 , TFLOPS: 68.2467338553195, Tokens per sec: 169122.92449184344, Loss: 3.494243621826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7392 , TFLOPS: 68.72175059276621, Tokens per sec: 170300.06829464977, Loss: 3.481985330581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7393 , TFLOPS: 69.56204516480469, Tokens per sec: 172382.41081024977, Loss: 3.476346254348755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7394 , TFLOPS: 68.9031411626378, Tokens per sec: 170749.57410861237, Loss: 3.4762840270996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7395 , TFLOPS: 68.81010031083571, Tokens per sec: 170519.00862855086, Loss: 3.489309310913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7396 , TFLOPS: 69.49603152340136, Tokens per sec: 172218.8217348494, Loss: 3.493915319442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7397 , TFLOPS: 68.92275198356246, Tokens per sec: 170798.17188317396, Loss: 3.4920175075531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7398 , TFLOPS: 68.04587349468432, Tokens per sec: 168625.17039159412, Loss: 3.5144028663635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7399 , TFLOPS: 70.61662352231536, Tokens per sec: 174995.7721515575, Loss: 3.5027475357055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7400 , TFLOPS: 68.62027017229398, Tokens per sec: 170048.58863372845, Loss: 3.4955756664276123 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/model.pt +[rank0]:[2024-08-25 12:16:39,119] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006381821993272752, 'preprocessing_with_comm': 0.0012684370012721047, 'state_converting': 0.9848340090102283, : 0.9938748059939826}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005400 + eval ppl=23.447528839111328, eval loss=3.1547651290893555 +------------------------------------------------------------------ +iteration: 7401 , TFLOPS: 62.546807339941196, Tokens per sec: 154997.8786879959, Loss: 3.5100789070129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7402 , TFLOPS: 69.66622124518003, Tokens per sec: 172640.57061336248, Loss: 3.492441415786743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7403 , TFLOPS: 68.1997820807985, Tokens per sec: 169006.57282241544, Loss: 3.478520393371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7404 , TFLOPS: 67.98946728440919, Tokens per sec: 168485.38958887535, Loss: 3.4996540546417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7405 , TFLOPS: 69.68344323168915, Tokens per sec: 172683.2485930905, Loss: 3.5001354217529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7406 , TFLOPS: 69.55815979812003, Tokens per sec: 172372.7824435967, Loss: 3.4770023822784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7407 , TFLOPS: 67.44871570534208, Tokens per sec: 167145.34760724308, Loss: 3.5077247619628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7408 , TFLOPS: 68.85486720580157, Tokens per sec: 170629.94592575685, Loss: 3.5152547359466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7409 , TFLOPS: 70.58224876012933, Tokens per sec: 174910.58770416785, Loss: 3.4982666969299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7410 , TFLOPS: 67.7330923280382, Tokens per sec: 167850.06420495515, Loss: 3.4916467666625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7411 , TFLOPS: 68.45103658027264, Tokens per sec: 169629.2091500804, Loss: 3.5034046173095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7412 , TFLOPS: 69.89431146953052, Tokens per sec: 173205.80331551543, Loss: 3.5002541542053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7413 , TFLOPS: 69.58210419454396, Tokens per sec: 172432.11929562822, Loss: 3.505779266357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7414 , TFLOPS: 67.41226896346431, Tokens per sec: 167055.02856592994, Loss: 3.485870361328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7415 , TFLOPS: 69.14340837415753, Tokens per sec: 171344.9827263766, Loss: 3.5081264972686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7416 , TFLOPS: 69.50940194288903, Tokens per sec: 172251.95510721314, Loss: 3.51853609085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7417 , TFLOPS: 68.73478312137279, Tokens per sec: 170332.3643361884, Loss: 3.529494524002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7418 , TFLOPS: 67.69903213990531, Tokens per sec: 167765.65930672237, Loss: 3.5093960762023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7419 , TFLOPS: 69.6537716164628, Tokens per sec: 172609.71906196137, Loss: 3.4939327239990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7420 , TFLOPS: 70.55368886944711, Tokens per sec: 174839.81314892563, Loss: 3.522508144378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7421 , TFLOPS: 69.17506898542553, Tokens per sec: 171423.44120880292, Loss: 3.4925546646118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7422 , TFLOPS: 69.40356237643907, Tokens per sec: 171989.67300235937, Loss: 3.4771664142608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7423 , TFLOPS: 69.60649011470345, Tokens per sec: 172492.5502921143, Loss: 3.492178440093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7424 , TFLOPS: 67.91082449450658, Tokens per sec: 168290.50409227825, Loss: 3.488431215286255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7425 , TFLOPS: 68.56893860086542, Tokens per sec: 169921.38334508904, Loss: 3.4951696395874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7426 , TFLOPS: 69.57184213488192, Tokens per sec: 172406.6887813265, Loss: 3.4802730083465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7427 , TFLOPS: 69.74698263773945, Tokens per sec: 172840.70624072655, Loss: 3.5129599571228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7428 , TFLOPS: 68.8464776489717, Tokens per sec: 170609.15567974644, Loss: 3.480449676513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7429 , TFLOPS: 69.49862904575551, Tokens per sec: 172225.25868138368, Loss: 3.4685072898864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7430 , TFLOPS: 68.6566962256149, Tokens per sec: 170138.85640651867, Loss: 3.494256019592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7431 , TFLOPS: 68.71669926038781, Tokens per sec: 170287.55053656586, Loss: 3.4977521896362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7432 , TFLOPS: 69.63399613278925, Tokens per sec: 172560.71323496895, Loss: 3.4983456134796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7433 , TFLOPS: 68.59810876640421, Tokens per sec: 169993.67022865315, Loss: 3.488276481628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7434 , TFLOPS: 68.88582191426255, Tokens per sec: 170706.6551032658, Loss: 3.4855213165283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7435 , TFLOPS: 68.88347101910058, Tokens per sec: 170700.82932608196, Loss: 3.4872000217437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7436 , TFLOPS: 69.53695915819881, Tokens per sec: 172320.24492242862, Loss: 3.524724245071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7437 , TFLOPS: 68.66927236653186, Tokens per sec: 170170.0214690872, Loss: 3.5107645988464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7438 , TFLOPS: 68.13091411720328, Tokens per sec: 168835.91042219405, Loss: 3.471923351287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7439 , TFLOPS: 70.58453110118097, Tokens per sec: 174916.24359671446, Loss: 3.4887421131134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7440 , TFLOPS: 68.37227692002993, Tokens per sec: 169434.0340359061, Loss: 3.4686663150787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7441 , TFLOPS: 68.89288384565174, Tokens per sec: 170724.15534718416, Loss: 3.528623342514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7442 , TFLOPS: 69.66281165048979, Tokens per sec: 172632.12126212218, Loss: 3.4869534969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7443 , TFLOPS: 69.56770842232837, Tokens per sec: 172396.44498625275, Loss: 3.4784979820251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7444 , TFLOPS: 67.03651673870169, Tokens per sec: 166123.87316044353, Loss: 3.496814250946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7445 , TFLOPS: 69.67257175415607, Tokens per sec: 172656.3078741714, Loss: 3.480894088745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7446 , TFLOPS: 69.59437670258882, Tokens per sec: 172462.53192249173, Loss: 3.4777848720550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7447 , TFLOPS: 69.40792499431598, Tokens per sec: 172000.4840500404, Loss: 3.4840097427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7448 , TFLOPS: 69.78801484620101, Tokens per sec: 172942.3885733652, Loss: 3.4966659545898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7449 , TFLOPS: 70.00894158770842, Tokens per sec: 173489.86937591023, Loss: 3.495703935623169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7450 , TFLOPS: 69.60451726279904, Tokens per sec: 172487.66134776757, Loss: 3.484645366668701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7451 , TFLOPS: 67.21648382763635, Tokens per sec: 166569.8514318351, Loss: 3.503645896911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7452 , TFLOPS: 69.61859419590688, Tokens per sec: 172522.545538711, Loss: 3.4991579055786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7453 , TFLOPS: 69.60710496442607, Tokens per sec: 172494.07395745823, Loss: 3.4681646823883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7454 , TFLOPS: 69.04038261193823, Tokens per sec: 171089.67353837253, Loss: 3.4917330741882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7455 , TFLOPS: 68.602556530691, Tokens per sec: 170004.69227851756, Loss: 3.5104949474334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7456 , TFLOPS: 70.5589424277834, Tokens per sec: 174852.8320452102, Loss: 3.501075506210327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7457 , TFLOPS: 68.89989655537674, Tokens per sec: 170741.53361439647, Loss: 3.4963088035583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7458 , TFLOPS: 67.89415174744083, Tokens per sec: 168249.18718839457, Loss: 3.5219366550445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7459 , TFLOPS: 69.34235139919925, Tokens per sec: 171837.9854578133, Loss: 3.4994239807128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7460 , TFLOPS: 69.57482288778463, Tokens per sec: 172414.07541537646, Loss: 3.4941744804382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7461 , TFLOPS: 68.93429245761168, Tokens per sec: 170826.77044915606, Loss: 3.5008420944213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7462 , TFLOPS: 68.82790689212759, Tokens per sec: 170563.13529854896, Loss: 3.4717419147491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7463 , TFLOPS: 69.55278294789802, Tokens per sec: 172359.45801643762, Loss: 3.5100653171539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7464 , TFLOPS: 68.76244869082412, Tokens per sec: 170400.92266490284, Loss: 3.503459930419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7465 , TFLOPS: 68.93312605161266, Tokens per sec: 170823.8799666003, Loss: 3.5107169151306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7466 , TFLOPS: 68.40387962664718, Tokens per sec: 169512.34902422922, Loss: 3.5013275146484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7467 , TFLOPS: 69.80476350902697, Tokens per sec: 172983.89360486544, Loss: 3.4965083599090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7468 , TFLOPS: 69.68420905178327, Tokens per sec: 172685.1463796458, Loss: 3.4853079319000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7469 , TFLOPS: 69.70847325627443, Tokens per sec: 172745.27575129765, Loss: 3.498284339904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7470 , TFLOPS: 68.61758262492116, Tokens per sec: 170041.92859528077, Loss: 3.4946365356445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7471 , TFLOPS: 67.90426407808539, Tokens per sec: 168274.24665769696, Loss: 3.5232388973236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7472 , TFLOPS: 69.60624427323145, Tokens per sec: 172491.94106986505, Loss: 3.5245707035064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7473 , TFLOPS: 68.6148564100385, Tokens per sec: 170035.1727344842, Loss: 3.486856698989868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7474 , TFLOPS: 68.9462897642612, Tokens per sec: 170856.50109664598, Loss: 3.48665714263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7475 , TFLOPS: 68.7574265651872, Tokens per sec: 170388.47728434176, Loss: 3.521455764770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7476 , TFLOPS: 70.49771759246163, Tokens per sec: 174701.11015881042, Loss: 3.481255054473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7477 , TFLOPS: 67.9624483236333, Tokens per sec: 168418.43362768844, Loss: 3.4946014881134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7478 , TFLOPS: 67.85983316816606, Tokens per sec: 168164.1419684481, Loss: 3.5227622985839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7479 , TFLOPS: 70.59822445390786, Tokens per sec: 174950.177233219, Loss: 3.507457971572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7480 , TFLOPS: 69.57614237349095, Tokens per sec: 172417.34524631008, Loss: 3.4920458793640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7481 , TFLOPS: 68.65654010604383, Tokens per sec: 170138.4695250236, Loss: 3.502624988555908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7482 , TFLOPS: 69.08439312539996, Tokens per sec: 171198.73643888897, Loss: 3.517061710357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7483 , TFLOPS: 69.71277768861593, Tokens per sec: 172755.94260881224, Loss: 3.491804599761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7484 , TFLOPS: 67.85872864498525, Tokens per sec: 168161.40483833276, Loss: 3.4747884273529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7485 , TFLOPS: 68.78581993880402, Tokens per sec: 170458.83919195257, Loss: 3.492443561553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7486 , TFLOPS: 69.5588740198058, Tokens per sec: 172374.55236361205, Loss: 3.509544610977173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7487 , TFLOPS: 70.52229818799255, Tokens per sec: 174762.0235823122, Loss: 3.4871551990509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7488 , TFLOPS: 68.71230082006072, Tokens per sec: 170276.65071690717, Loss: 3.4723615646362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7489 , TFLOPS: 69.71761862221432, Tokens per sec: 172767.93897554194, Loss: 3.4942569732666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7490 , TFLOPS: 69.6242972096372, Tokens per sec: 172536.6782349736, Loss: 3.5020241737365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7491 , TFLOPS: 68.08392843064155, Tokens per sec: 168719.47471499344, Loss: 3.5217695236206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7492 , TFLOPS: 68.48212471369052, Tokens per sec: 169706.2489693305, Loss: 3.4948129653930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7493 , TFLOPS: 69.60937831629815, Tokens per sec: 172499.70757382762, Loss: 3.506028890609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7494 , TFLOPS: 68.7831935249679, Tokens per sec: 170452.3306491433, Loss: 3.49566650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7495 , TFLOPS: 68.82390290749152, Tokens per sec: 170553.21298354553, Loss: 3.4992475509643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7496 , TFLOPS: 69.61800515096799, Tokens per sec: 172521.08582046378, Loss: 3.482577323913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7497 , TFLOPS: 68.8597040977186, Tokens per sec: 170641.93227675493, Loss: 3.5147945880889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7498 , TFLOPS: 68.78781316506229, Tokens per sec: 170463.77862619256, Loss: 3.505695343017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7499 , TFLOPS: 68.9517053532243, Tokens per sec: 170869.92152267296, Loss: 3.489215850830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7500 , TFLOPS: 69.62868632256287, Tokens per sec: 172547.55494030408, Loss: 3.4921250343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7501 , TFLOPS: 69.6403343281171, Tokens per sec: 172576.4199812012, Loss: 3.491572380065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7502 , TFLOPS: 68.80580465011658, Tokens per sec: 170508.36350808275, Loss: 3.4984183311462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7503 , TFLOPS: 68.71425075238436, Tokens per sec: 170281.4828640079, Loss: 3.493382692337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7504 , TFLOPS: 69.72796575811077, Tokens per sec: 172793.58031812444, Loss: 3.4998345375061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7505 , TFLOPS: 68.17964654398456, Tokens per sec: 168956.67474407802, Loss: 3.515338659286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7506 , TFLOPS: 70.56335197150747, Tokens per sec: 174863.75938030978, Loss: 3.505009651184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7507 , TFLOPS: 69.50235929824866, Tokens per sec: 172234.50265798296, Loss: 3.4832777976989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7508 , TFLOPS: 68.77248763974166, Tokens per sec: 170425.80028620202, Loss: 3.517000198364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7509 , TFLOPS: 69.6832697600214, Tokens per sec: 172682.81871118833, Loss: 3.502229928970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7510 , TFLOPS: 69.6051450812142, Tokens per sec: 172489.2171509603, Loss: 3.5021190643310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7511 , TFLOPS: 67.56824465229569, Tokens per sec: 167441.55350499463, Loss: 3.494518756866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7512 , TFLOPS: 69.0458144878651, Tokens per sec: 171103.13432528984, Loss: 3.5001704692840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7513 , TFLOPS: 69.59736136278931, Tokens per sec: 172469.92823925603, Loss: 3.503253698348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7514 , TFLOPS: 67.78233371216169, Tokens per sec: 167972.0897791994, Loss: 3.507596731185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7515 , TFLOPS: 69.5741338379045, Tokens per sec: 172412.36787386838, Loss: 3.4958276748657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7516 , TFLOPS: 69.98399622650065, Tokens per sec: 173428.05202287898, Loss: 3.4901328086853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7517 , TFLOPS: 68.71629978517167, Tokens per sec: 170286.56059297448, Loss: 3.4938435554504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7518 , TFLOPS: 67.93279298113553, Tokens per sec: 168344.9444221733, Loss: 3.4985992908477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7519 , TFLOPS: 69.67759411028983, Tokens per sec: 172668.75382592907, Loss: 3.5108654499053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7520 , TFLOPS: 69.68348689760107, Tokens per sec: 172683.35680203044, Loss: 3.4847002029418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7521 , TFLOPS: 68.69637613587169, Tokens per sec: 170237.18759523795, Loss: 3.5123026371002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7522 , TFLOPS: 68.76226629808059, Tokens per sec: 170400.4706755922, Loss: 3.499177932739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7523 , TFLOPS: 70.56821643896987, Tokens per sec: 174875.8140665469, Loss: 3.4792861938476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7524 , TFLOPS: 69.05869084035118, Tokens per sec: 171135.04334519766, Loss: 3.5105042457580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7525 , TFLOPS: 67.87606265188647, Tokens per sec: 168204.3604169325, Loss: 3.502105712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7526 , TFLOPS: 70.5763065183884, Tokens per sec: 174895.8621745419, Loss: 3.491387367248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7527 , TFLOPS: 69.67317357821537, Tokens per sec: 172657.79926048676, Loss: 3.4891579151153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7528 , TFLOPS: 68.76143509793735, Tokens per sec: 170398.41087007296, Loss: 3.511902093887329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7529 , TFLOPS: 67.9494603570163, Tokens per sec: 168386.24801566894, Loss: 3.49328351020813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7530 , TFLOPS: 69.57766291666115, Tokens per sec: 172421.1133197875, Loss: 3.5079827308654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7531 , TFLOPS: 68.93571155423898, Tokens per sec: 170830.2871269249, Loss: 3.498904228210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7532 , TFLOPS: 68.83252174808797, Tokens per sec: 170574.57141998716, Loss: 3.4898016452789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7533 , TFLOPS: 68.72538563783162, Tokens per sec: 170309.07633675513, Loss: 3.5209710597991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7534 , TFLOPS: 69.67779333073634, Tokens per sec: 172669.24751614145, Loss: 3.5170934200286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7535 , TFLOPS: 69.0240957731979, Tokens per sec: 171049.31295783058, Loss: 3.50666880607605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7536 , TFLOPS: 68.56312082765079, Tokens per sec: 169906.96626218257, Loss: 3.5060386657714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7537 , TFLOPS: 69.47080041873245, Tokens per sec: 172156.2962780442, Loss: 3.499418020248413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7538 , TFLOPS: 68.30621876822282, Tokens per sec: 169270.334658822, Loss: 3.487727165222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7539 , TFLOPS: 69.62838484108626, Tokens per sec: 172546.8078359941, Loss: 3.4881486892700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7540 , TFLOPS: 68.68983244604688, Tokens per sec: 170220.97161100514, Loss: 3.5016884803771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7541 , TFLOPS: 69.66654481725071, Tokens per sec: 172641.372460598, Loss: 3.499890089035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7542 , TFLOPS: 68.77222306938664, Tokens per sec: 170425.14465171748, Loss: 3.538945436477661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7543 , TFLOPS: 70.57497043480745, Tokens per sec: 174892.55121224653, Loss: 3.479250192642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7544 , TFLOPS: 67.83098549788815, Tokens per sec: 168092.65426366622, Loss: 3.5198452472686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7545 , TFLOPS: 67.80908420445358, Tokens per sec: 168038.38044590873, Loss: 3.514402151107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7546 , TFLOPS: 70.47478743394865, Tokens per sec: 174644.2867000498, Loss: 3.51997709274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7547 , TFLOPS: 68.68971844478958, Tokens per sec: 170220.68910333127, Loss: 3.4985384941101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7548 , TFLOPS: 69.72361206968012, Tokens per sec: 172782.7913985949, Loss: 3.491142511367798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7549 , TFLOPS: 69.06584232042194, Tokens per sec: 171152.76550061358, Loss: 3.5053181648254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7550 , TFLOPS: 69.55222488542479, Tokens per sec: 172358.0750761533, Loss: 3.478482484817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7551 , TFLOPS: 68.01464383548785, Tokens per sec: 168547.7798558743, Loss: 3.474126100540161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7552 , TFLOPS: 68.7667141089993, Tokens per sec: 170411.49284101554, Loss: 3.5101699829101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7553 , TFLOPS: 70.57280767535977, Tokens per sec: 174887.19165608753, Loss: 3.505072832107544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7554 , TFLOPS: 69.94229196957146, Tokens per sec: 173324.70428010527, Loss: 3.4924652576446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7555 , TFLOPS: 67.2299521630696, Tokens per sec: 166603.2274506983, Loss: 3.510812997817993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7556 , TFLOPS: 69.62204432516604, Tokens per sec: 172531.09533907834, Loss: 3.4988999366760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7557 , TFLOPS: 69.49761099398377, Tokens per sec: 172222.73583694565, Loss: 3.5050415992736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7558 , TFLOPS: 67.83783361532889, Tokens per sec: 168109.6246530668, Loss: 3.4968605041503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7559 , TFLOPS: 69.62077454973473, Tokens per sec: 172527.94869568222, Loss: 3.5025951862335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7560 , TFLOPS: 69.69500207568957, Tokens per sec: 172711.89268183534, Loss: 3.4972729682922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7561 , TFLOPS: 69.6499935920087, Tokens per sec: 172600.35670118037, Loss: 3.50905179977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7562 , TFLOPS: 67.80880308541504, Tokens per sec: 168037.68380196337, Loss: 3.4926939010620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7563 , TFLOPS: 70.58026901506403, Tokens per sec: 174905.68167780916, Loss: 3.486391544342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7564 , TFLOPS: 68.77583465206514, Tokens per sec: 170434.0945514435, Loss: 3.4980735778808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7565 , TFLOPS: 68.68664081386854, Tokens per sec: 170213.0623948799, Loss: 3.506068468093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7566 , TFLOPS: 69.01053685339453, Tokens per sec: 171015.71246091955, Loss: 3.4647367000579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7567 , TFLOPS: 69.66659443003927, Tokens per sec: 172641.49540655338, Loss: 3.5049855709075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7568 , TFLOPS: 69.68469429951978, Tokens per sec: 172686.3488769913, Loss: 3.4770514965057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7569 , TFLOPS: 68.83358037636255, Tokens per sec: 170577.1948174691, Loss: 3.4889512062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7570 , TFLOPS: 68.55013837895649, Tokens per sec: 169874.7943241833, Loss: 3.476229190826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7571 , TFLOPS: 69.66452615494714, Tokens per sec: 172636.36999303577, Loss: 3.5044991970062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7572 , TFLOPS: 68.09909532949352, Tokens per sec: 168757.05996112042, Loss: 3.53226900100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7573 , TFLOPS: 69.64184496343267, Tokens per sec: 172580.1635019234, Loss: 3.4706478118896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7574 , TFLOPS: 69.62392174936701, Tokens per sec: 172535.7478030637, Loss: 3.5083460807800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7575 , TFLOPS: 68.1508321148102, Tokens per sec: 168885.26941441256, Loss: 3.511552095413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7576 , TFLOPS: 69.62885609763791, Tokens per sec: 172547.9756616423, Loss: 3.4954142570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7577 , TFLOPS: 68.4700918591967, Tokens per sec: 169676.43023036644, Loss: 3.513902187347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7578 , TFLOPS: 67.67835430073693, Tokens per sec: 167714.41734341113, Loss: 3.4934048652648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7579 , TFLOPS: 69.63948079341688, Tokens per sec: 172574.30482818946, Loss: 3.4866130352020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7580 , TFLOPS: 69.72119655658592, Tokens per sec: 172776.8054910568, Loss: 3.5227928161621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7581 , TFLOPS: 68.84921276550229, Tokens per sec: 170615.9335998072, Loss: 3.4899165630340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7582 , TFLOPS: 69.67702719301735, Tokens per sec: 172667.3489424768, Loss: 3.5206093788146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7583 , TFLOPS: 69.96780551341132, Tokens per sec: 173387.92965228934, Loss: 3.505488872528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7584 , TFLOPS: 68.65297953499825, Tokens per sec: 170129.64603774334, Loss: 3.497459888458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7585 , TFLOPS: 67.77159654040989, Tokens per sec: 167945.48188480135, Loss: 3.5124611854553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7586 , TFLOPS: 69.64751895000158, Tokens per sec: 172594.22426568237, Loss: 3.5289852619171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7587 , TFLOPS: 69.67035050112825, Tokens per sec: 172650.80336447724, Loss: 3.5038304328918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7588 , TFLOPS: 68.74267546711565, Tokens per sec: 170351.92243835286, Loss: 3.489755153656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7589 , TFLOPS: 68.71936729279636, Tokens per sec: 170294.16221478232, Loss: 3.4991893768310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7590 , TFLOPS: 70.54546994962612, Tokens per sec: 174819.44576022046, Loss: 3.5090088844299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7591 , TFLOPS: 68.57181670658217, Tokens per sec: 169928.51560810467, Loss: 3.482830047607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7592 , TFLOPS: 66.92236816790145, Tokens per sec: 165841.0004274974, Loss: 3.5065090656280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7593 , TFLOPS: 70.58128544132724, Tokens per sec: 174908.20049405796, Loss: 3.4933078289031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7594 , TFLOPS: 69.59289545950162, Tokens per sec: 172458.86123894926, Loss: 3.5113258361816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7595 , TFLOPS: 66.25427374224711, Tokens per sec: 164185.38884404942, Loss: 3.518897771835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7596 , TFLOPS: 69.55059860012176, Tokens per sec: 172354.04496202228, Loss: 3.5221314430236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7597 , TFLOPS: 69.6406614705655, Tokens per sec: 172577.23067622606, Loss: 3.4638426303863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7598 , TFLOPS: 68.7805006527921, Tokens per sec: 170445.6574152473, Loss: 3.5062921047210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7599 , TFLOPS: 68.062210306789, Tokens per sec: 168665.65481163797, Loss: 3.492788314819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7600 , TFLOPS: 69.50597919248777, Tokens per sec: 172243.47315467172, Loss: 3.505993604660034 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/optimizer.pt +[rank0]:[2024-08-25 12:42:19,735] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006468702995334752, 'preprocessing_with_comm': 0.0012672239972744137, 'state_converting': 0.9932253690058133, : 1.0023380730126519}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600, took 4.31s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005600 + eval ppl=23.8265380859375, eval loss=3.170799970626831 +------------------------------------------------------------------ +iteration: 7601 , TFLOPS: 64.6420042372126, Tokens per sec: 160190.00740442614, Loss: 3.4907162189483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7602 , TFLOPS: 68.75607834773686, Tokens per sec: 170385.1362529814, Loss: 3.4721596240997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7603 , TFLOPS: 69.15900641646161, Tokens per sec: 171383.6363934723, Loss: 3.497101068496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7604 , TFLOPS: 68.7917879954131, Tokens per sec: 170473.62869366558, Loss: 3.5071773529052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7605 , TFLOPS: 68.73680289405202, Tokens per sec: 170337.36955538386, Loss: 3.506540060043335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7606 , TFLOPS: 69.67337668863844, Tokens per sec: 172658.3025904896, Loss: 3.4954090118408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7607 , TFLOPS: 68.61821320246138, Tokens per sec: 170043.49123588926, Loss: 3.503380537033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7608 , TFLOPS: 68.96881189869487, Tokens per sec: 170912.31342679058, Loss: 3.5140960216522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7609 , TFLOPS: 68.69972409660281, Tokens per sec: 170245.48421073798, Loss: 3.5046494007110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7610 , TFLOPS: 68.7879846199546, Tokens per sec: 170464.203510303, Loss: 3.5091872215270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7611 , TFLOPS: 69.63180305442381, Tokens per sec: 172555.27854519186, Loss: 3.480415105819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7612 , TFLOPS: 67.82863895360576, Tokens per sec: 168086.83926844125, Loss: 3.4981186389923096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7613 , TFLOPS: 70.5099320372483, Tokens per sec: 174731.378898524, Loss: 3.5066652297973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7614 , TFLOPS: 68.76322291798847, Tokens per sec: 170402.8412851043, Loss: 3.49847412109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7615 , TFLOPS: 68.32124963423647, Tokens per sec: 169307.58280059037, Loss: 3.4862847328186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7616 , TFLOPS: 69.26310896901543, Tokens per sec: 171641.61398654355, Loss: 3.512220859527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7617 , TFLOPS: 69.60342523531034, Tokens per sec: 172484.95518335383, Loss: 3.523451089859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7618 , TFLOPS: 67.81223816040007, Tokens per sec: 168046.19629618103, Loss: 3.5030276775360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7619 , TFLOPS: 69.393465036873, Tokens per sec: 171964.6506826009, Loss: 3.492832899093628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7620 , TFLOPS: 69.63766722407227, Tokens per sec: 172569.81060357235, Loss: 3.4970853328704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7621 , TFLOPS: 68.9417701710668, Tokens per sec: 170845.30104683602, Loss: 3.4790022373199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7622 , TFLOPS: 67.74726147749728, Tokens per sec: 167885.17691817542, Loss: 3.500805377960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7623 , TFLOPS: 70.5785259162914, Tokens per sec: 174901.36208703235, Loss: 3.5205869674682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7624 , TFLOPS: 68.78874805364593, Tokens per sec: 170466.0953830893, Loss: 3.4894957542419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7625 , TFLOPS: 68.22715445226397, Tokens per sec: 169074.40457422257, Loss: 3.4843544960021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7626 , TFLOPS: 69.93996404002459, Tokens per sec: 173318.93541424553, Loss: 3.5033862590789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7627 , TFLOPS: 69.51555391132642, Tokens per sec: 172267.20036269628, Loss: 3.478127956390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7628 , TFLOPS: 68.88623188666325, Tokens per sec: 170707.67106003693, Loss: 3.500185489654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7629 , TFLOPS: 68.37080871057648, Tokens per sec: 169430.39565114386, Loss: 3.5022683143615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7630 , TFLOPS: 69.69930252313648, Tokens per sec: 172722.54966434216, Loss: 3.496826410293579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7631 , TFLOPS: 69.82829332725447, Tokens per sec: 173042.20308645663, Loss: 3.489704132080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7632 , TFLOPS: 67.91884606150073, Tokens per sec: 168310.38241893824, Loss: 3.512550115585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7633 , TFLOPS: 69.28874142911202, Tokens per sec: 171705.13404631146, Loss: 3.4866950511932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7634 , TFLOPS: 69.6022973180627, Tokens per sec: 172482.16008016383, Loss: 3.5023365020751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7635 , TFLOPS: 67.83798843286465, Tokens per sec: 168110.00830797496, Loss: 3.4895992279052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7636 , TFLOPS: 68.6537434824937, Tokens per sec: 170131.53918379283, Loss: 3.5094757080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7637 , TFLOPS: 69.6245033772242, Tokens per sec: 172537.18914096543, Loss: 3.4834401607513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7638 , TFLOPS: 68.8884329023609, Tokens per sec: 170713.1254193976, Loss: 3.508247137069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7639 , TFLOPS: 69.61154272551036, Tokens per sec: 172505.07121828487, Loss: 3.5055415630340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7640 , TFLOPS: 68.222149790204, Tokens per sec: 169062.00247032888, Loss: 3.5043606758117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7641 , TFLOPS: 69.67626167741695, Tokens per sec: 172665.45191049046, Loss: 3.5046379566192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7642 , TFLOPS: 69.08737845861347, Tokens per sec: 171206.13442345368, Loss: 3.4796597957611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7643 , TFLOPS: 69.87976777120079, Tokens per sec: 173169.76242893373, Loss: 3.472276449203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7644 , TFLOPS: 67.43399088898761, Tokens per sec: 167108.85789024428, Loss: 3.480741024017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7645 , TFLOPS: 69.65112503482936, Tokens per sec: 172603.16054112866, Loss: 3.494345188140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7646 , TFLOPS: 69.66799633154663, Tokens per sec: 172644.9694729208, Loss: 3.4940600395202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7647 , TFLOPS: 68.49844620389233, Tokens per sec: 169746.69542001112, Loss: 3.5026960372924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7648 , TFLOPS: 68.76013912555995, Tokens per sec: 170395.19930775862, Loss: 3.5260169506073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7649 , TFLOPS: 68.89101666698524, Tokens per sec: 170719.52827276188, Loss: 3.5024311542510986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7650 , TFLOPS: 70.60077829226027, Tokens per sec: 174956.50592598482, Loss: 3.499626398086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7651 , TFLOPS: 66.64059967419469, Tokens per sec: 165142.7470607301, Loss: 3.528775215148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7652 , TFLOPS: 67.86183403910893, Tokens per sec: 168169.10034705867, Loss: 3.494757652282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7653 , TFLOPS: 70.59050606952192, Tokens per sec: 174931.05022646044, Loss: 3.5079991817474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7654 , TFLOPS: 69.66415661287067, Tokens per sec: 172635.45422706185, Loss: 3.487325429916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7655 , TFLOPS: 67.7987386477772, Tokens per sec: 168012.7430168075, Loss: 3.504753828048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7656 , TFLOPS: 69.747402538847, Tokens per sec: 172841.746801927, Loss: 3.48796010017395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7657 , TFLOPS: 69.53523549306362, Tokens per sec: 172315.97349034625, Loss: 3.5092430114746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7658 , TFLOPS: 69.28842508382186, Tokens per sec: 171704.35010783412, Loss: 3.482638359069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7659 , TFLOPS: 68.15532107723837, Tokens per sec: 168896.3935578096, Loss: 3.4690005779266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7660 , TFLOPS: 69.66969320427722, Tokens per sec: 172649.17451047318, Loss: 3.4868760108947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7661 , TFLOPS: 69.71519930157416, Tokens per sec: 172761.94363248543, Loss: 3.5102100372314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7662 , TFLOPS: 68.60234400596066, Tokens per sec: 170004.1656188242, Loss: 3.491426467895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7663 , TFLOPS: 69.71387641807986, Tokens per sec: 172758.6653814589, Loss: 3.5083107948303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7664 , TFLOPS: 69.54971687580137, Tokens per sec: 172351.8599520264, Loss: 3.528860330581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7665 , TFLOPS: 69.18020367206312, Tokens per sec: 171436.16552792332, Loss: 3.4734954833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7666 , TFLOPS: 68.71994313771772, Tokens per sec: 170295.589221932, Loss: 3.4844188690185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7667 , TFLOPS: 69.61796921995625, Tokens per sec: 172520.99677945857, Loss: 3.505714178085327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7668 , TFLOPS: 68.80804280995909, Tokens per sec: 170513.9099147262, Loss: 3.4799740314483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7669 , TFLOPS: 68.85502811513689, Tokens per sec: 170630.3446768154, Loss: 3.4876513481140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7670 , TFLOPS: 69.54048078555148, Tokens per sec: 172328.97190294781, Loss: 3.4995429515838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7671 , TFLOPS: 68.7519863637342, Tokens per sec: 170374.9958658536, Loss: 3.50325870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7672 , TFLOPS: 68.75518815210971, Tokens per sec: 170382.9302501542, Loss: 3.4915921688079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7673 , TFLOPS: 69.83840735076767, Tokens per sec: 173067.26675087403, Loss: 3.5173404216766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7674 , TFLOPS: 68.65547595666165, Tokens per sec: 170135.8324456288, Loss: 3.491328239440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7675 , TFLOPS: 68.8090959725126, Tokens per sec: 170516.51976754918, Loss: 3.511082172393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7676 , TFLOPS: 68.89321998741791, Tokens per sec: 170724.9883435098, Loss: 3.503140687942505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7677 , TFLOPS: 68.69788002225911, Tokens per sec: 170240.91439137276, Loss: 3.509429454803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7678 , TFLOPS: 69.75591951925358, Tokens per sec: 172862.85281759733, Loss: 3.4749763011932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7679 , TFLOPS: 68.76469335521342, Tokens per sec: 170406.48519052975, Loss: 3.4839651584625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7680 , TFLOPS: 69.21176688395921, Tokens per sec: 171514.38264397666, Loss: 3.4979186058044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7681 , TFLOPS: 69.6321939670076, Tokens per sec: 172556.24726963704, Loss: 3.474682331085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7682 , TFLOPS: 68.1350173648109, Tokens per sec: 168846.0787217758, Loss: 3.5117568969726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7683 , TFLOPS: 69.7200311975276, Tokens per sec: 172773.91760293534, Loss: 3.4847350120544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7684 , TFLOPS: 68.63043850387264, Tokens per sec: 170073.78687951455, Loss: 3.498685121536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7685 , TFLOPS: 68.83598839527707, Tokens per sec: 170583.1621536045, Loss: 3.513253688812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7686 , TFLOPS: 69.65085365193333, Tokens per sec: 172602.48802441702, Loss: 3.4869742393493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7687 , TFLOPS: 69.61546128158194, Tokens per sec: 172514.78183189483, Loss: 3.497135639190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7688 , TFLOPS: 68.67440788304036, Tokens per sec: 170182.7478447191, Loss: 3.487509250640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7689 , TFLOPS: 68.29595377668501, Tokens per sec: 169244.89687900912, Loss: 3.5035178661346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7690 , TFLOPS: 70.58020616157822, Tokens per sec: 174905.52591994737, Loss: 3.5070269107818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7691 , TFLOPS: 68.73640134661316, Tokens per sec: 170336.37447659628, Loss: 3.4682631492614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7692 , TFLOPS: 67.99995630671557, Tokens per sec: 168511.38254161176, Loss: 3.510465383529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7693 , TFLOPS: 69.65209656737187, Tokens per sec: 172605.56810579216, Loss: 3.4902894496917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7694 , TFLOPS: 69.59942437092204, Tokens per sec: 172475.04060066803, Loss: 3.5022177696228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7695 , TFLOPS: 68.79084543580925, Tokens per sec: 170471.2929271357, Loss: 3.5126736164093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7696 , TFLOPS: 68.71480301164966, Tokens per sec: 170282.8514233037, Loss: 3.4749863147735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7697 , TFLOPS: 69.53755211149338, Tokens per sec: 172321.7143260118, Loss: 3.476006507873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7698 , TFLOPS: 69.72155982672021, Tokens per sec: 172777.70571446704, Loss: 3.500613212585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7699 , TFLOPS: 67.35779975386122, Tokens per sec: 166920.04786425468, Loss: 3.518821954727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7700 , TFLOPS: 70.58301575652808, Tokens per sec: 174912.48840573558, Loss: 3.5137929916381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7701 , TFLOPS: 69.92565923861494, Tokens per sec: 173283.48654054847, Loss: 3.4796926975250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7702 , TFLOPS: 67.97398755318585, Tokens per sec: 168447.02910967072, Loss: 3.5019936561584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7703 , TFLOPS: 68.79195070160893, Tokens per sec: 170474.0318975424, Loss: 3.50301456451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7704 , TFLOPS: 69.52281108602229, Tokens per sec: 172285.18444103637, Loss: 3.4899399280548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7705 , TFLOPS: 68.87545772361655, Tokens per sec: 170680.97149713515, Loss: 3.5185840129852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7706 , TFLOPS: 69.67210521423665, Tokens per sec: 172655.15173685752, Loss: 3.5136008262634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7707 , TFLOPS: 68.64255354319768, Tokens per sec: 170103.8092815437, Loss: 3.5045042037963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7708 , TFLOPS: 69.68493992506761, Tokens per sec: 172686.95756415674, Loss: 3.5019114017486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7709 , TFLOPS: 68.82998580437078, Tokens per sec: 170568.28707209873, Loss: 3.516263008117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7710 , TFLOPS: 68.86958724942333, Tokens per sec: 170666.42381539824, Loss: 3.502384901046753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7711 , TFLOPS: 69.52049350512715, Tokens per sec: 172279.44122027562, Loss: 3.4680027961730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7712 , TFLOPS: 68.17541896449008, Tokens per sec: 168946.19833638438, Loss: 3.5146989822387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7713 , TFLOPS: 69.62212908655772, Tokens per sec: 172531.30538714415, Loss: 3.5126447677612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7714 , TFLOPS: 68.6710123405181, Tokens per sec: 170174.33331629907, Loss: 3.502371072769165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7715 , TFLOPS: 68.82358214103219, Tokens per sec: 170552.41808892394, Loss: 3.4926934242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7716 , TFLOPS: 68.78249083638929, Tokens per sec: 170450.58930943796, Loss: 3.483431339263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7717 , TFLOPS: 70.53585899440321, Tokens per sec: 174795.62875444398, Loss: 3.5072760581970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7718 , TFLOPS: 68.09677147865497, Tokens per sec: 168751.30120274922, Loss: 3.477130889892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7719 , TFLOPS: 68.73049163048661, Tokens per sec: 170321.72954902568, Loss: 3.5151517391204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7720 , TFLOPS: 70.61025493530948, Tokens per sec: 174979.99009139914, Loss: 3.4940459728240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7721 , TFLOPS: 68.5698102947568, Tokens per sec: 169923.5434985769, Loss: 3.4776980876922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7722 , TFLOPS: 68.06924129947352, Tokens per sec: 168683.078386038, Loss: 3.4849634170532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7723 , TFLOPS: 69.62919197824499, Tokens per sec: 172548.80801078695, Loss: 3.477797746658325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7724 , TFLOPS: 69.5726884355536, Tokens per sec: 172408.78600761865, Loss: 3.514420747756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7725 , TFLOPS: 68.64668678732467, Tokens per sec: 170114.05191580483, Loss: 3.509403705596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7726 , TFLOPS: 68.97740255819234, Tokens per sec: 170933.60202736457, Loss: 3.4882092475891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7727 , TFLOPS: 69.62810141076164, Tokens per sec: 172546.1054644273, Loss: 3.5116100311279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7728 , TFLOPS: 69.73827672288627, Tokens per sec: 172819.13202468466, Loss: 3.503906488418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7729 , TFLOPS: 68.78792018130547, Tokens per sec: 170464.04382423195, Loss: 3.502906560897827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7730 , TFLOPS: 69.69983767783843, Tokens per sec: 172723.87583664572, Loss: 3.494868278503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7731 , TFLOPS: 69.63616007649028, Tokens per sec: 172566.07572583816, Loss: 3.4865612983703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7732 , TFLOPS: 67.9367534645072, Tokens per sec: 168354.7589362516, Loss: 3.4928925037384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7733 , TFLOPS: 68.6778712456524, Tokens per sec: 170191.33043879262, Loss: 3.5223731994628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7734 , TFLOPS: 69.58922771501057, Tokens per sec: 172449.772164065, Loss: 3.4990241527557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7735 , TFLOPS: 68.81140998153516, Tokens per sec: 170522.25413681625, Loss: 3.4820396900177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7736 , TFLOPS: 67.78543644858605, Tokens per sec: 167979.77870185362, Loss: 3.4900801181793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7737 , TFLOPS: 70.59296773660726, Tokens per sec: 174937.15050867293, Loss: 3.4767091274261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7738 , TFLOPS: 68.79650391666819, Tokens per sec: 170485.3152660373, Loss: 3.494568109512329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7739 , TFLOPS: 68.7812982242835, Tokens per sec: 170447.63388526277, Loss: 3.5146536827087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7740 , TFLOPS: 69.1151811487322, Tokens per sec: 171275.03255228704, Loss: 3.522340774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7741 , TFLOPS: 69.58493705317902, Tokens per sec: 172439.13943139103, Loss: 3.5026979446411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7742 , TFLOPS: 68.82566174950938, Tokens per sec: 170557.57158781824, Loss: 3.515805959701538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7743 , TFLOPS: 69.04854328625497, Tokens per sec: 171109.8965883019, Loss: 3.508929967880249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7744 , TFLOPS: 68.62838688443806, Tokens per sec: 170068.7027405511, Loss: 3.4749062061309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7745 , TFLOPS: 69.69622703371627, Tokens per sec: 172714.92826276575, Loss: 3.475876808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7746 , TFLOPS: 68.75439854329163, Tokens per sec: 170380.9735125204, Loss: 3.518218994140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7747 , TFLOPS: 69.63223853970877, Tokens per sec: 172556.35772570065, Loss: 3.4929895401000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7748 , TFLOPS: 69.53816861169508, Tokens per sec: 172323.24208142472, Loss: 3.5130105018615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7749 , TFLOPS: 68.33307677381077, Tokens per sec: 169336.89175532246, Loss: 3.507575511932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7750 , TFLOPS: 69.68791343492695, Tokens per sec: 172694.32624914733, Loss: 3.493056058883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7751 , TFLOPS: 68.71770974214509, Tokens per sec: 170290.05462167418, Loss: 3.493178606033325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7752 , TFLOPS: 67.93635116768034, Tokens per sec: 168353.76200039798, Loss: 3.505643606185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7753 , TFLOPS: 69.59775295930784, Tokens per sec: 172470.89865856688, Loss: 3.5122504234313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7754 , TFLOPS: 69.6657431942506, Tokens per sec: 172639.38595049767, Loss: 3.4770267009735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7755 , TFLOPS: 68.09540548983571, Tokens per sec: 168747.916131979, Loss: 3.5090384483337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7756 , TFLOPS: 68.69879966702749, Tokens per sec: 170243.19337241605, Loss: 3.5074222087860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7757 , TFLOPS: 70.63138899931288, Tokens per sec: 175032.36263010892, Loss: 3.5017330646514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7758 , TFLOPS: 68.64615199071869, Tokens per sec: 170112.72663090253, Loss: 3.491664171218872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7759 , TFLOPS: 69.66086211172994, Tokens per sec: 172627.2900903159, Loss: 3.494805097579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7760 , TFLOPS: 69.91438453360963, Tokens per sec: 173255.546579534, Loss: 3.484442710876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7761 , TFLOPS: 69.53859748308626, Tokens per sec: 172324.30487197614, Loss: 3.4934985637664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7762 , TFLOPS: 67.84841782709475, Tokens per sec: 168135.85349576976, Loss: 3.5033066272735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7763 , TFLOPS: 68.7320667701961, Tokens per sec: 170325.6329187418, Loss: 3.4923551082611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7764 , TFLOPS: 69.55540932294942, Tokens per sec: 172365.96646313407, Loss: 3.50032377243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7765 , TFLOPS: 69.72589791442886, Tokens per sec: 172788.45597368886, Loss: 3.5009007453918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7766 , TFLOPS: 68.6430464279488, Tokens per sec: 170105.03070425277, Loss: 3.500843048095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7767 , TFLOPS: 70.55188490082008, Tokens per sec: 174835.34271594434, Loss: 3.4985833168029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7768 , TFLOPS: 69.89317624574373, Tokens per sec: 173202.99010591654, Loss: 3.5313899517059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7769 , TFLOPS: 68.13075176994825, Tokens per sec: 168835.50810781203, Loss: 3.4645986557006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7770 , TFLOPS: 68.75388851346997, Tokens per sec: 170379.7096024379, Loss: 3.5051846504211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7771 , TFLOPS: 69.61372835864377, Tokens per sec: 172510.48745795683, Loss: 3.50315523147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7772 , TFLOPS: 68.81097231595673, Tokens per sec: 170521.16955330005, Loss: 3.4809787273406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7773 , TFLOPS: 67.99727426640536, Tokens per sec: 168504.73615027193, Loss: 3.4728786945343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7774 , TFLOPS: 69.05522584704488, Tokens per sec: 171126.45671008417, Loss: 3.5014212131500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7775 , TFLOPS: 69.79135551770425, Tokens per sec: 172950.66712535554, Loss: 3.485220432281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7776 , TFLOPS: 69.10489703088837, Tokens per sec: 171249.54737538093, Loss: 3.5154778957366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7777 , TFLOPS: 68.83822496586836, Tokens per sec: 170588.7046219083, Loss: 3.5104880332946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7778 , TFLOPS: 68.14682461614949, Tokens per sec: 168875.3383912689, Loss: 3.5097827911376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7779 , TFLOPS: 68.09197800258526, Tokens per sec: 168739.4224410035, Loss: 3.5117650032043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7780 , TFLOPS: 69.7277036941564, Tokens per sec: 172792.9308947768, Loss: 3.5110878944396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7781 , TFLOPS: 68.46058089096074, Tokens per sec: 169652.86100335812, Loss: 3.4885871410369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7782 , TFLOPS: 68.80872623031112, Tokens per sec: 170515.6035056435, Loss: 3.5035722255706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7783 , TFLOPS: 68.79831919210396, Tokens per sec: 170489.81371853623, Loss: 3.489372730255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7784 , TFLOPS: 69.63110384573869, Tokens per sec: 172553.54582904594, Loss: 3.498074769973755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7785 , TFLOPS: 69.00549155613125, Tokens per sec: 171003.20965851576, Loss: 3.4819278717041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7786 , TFLOPS: 68.81417867082148, Tokens per sec: 170529.11525386406, Loss: 3.4998607635498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7787 , TFLOPS: 70.53745525036048, Tokens per sec: 174799.58445254818, Loss: 3.460758686065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7788 , TFLOPS: 68.52550976696095, Tokens per sec: 169813.7619105928, Loss: 3.495805263519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7789 , TFLOPS: 67.92376799600024, Tokens per sec: 168322.5795148239, Loss: 3.4835245609283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7790 , TFLOPS: 69.61654510948513, Tokens per sec: 172517.46767683243, Loss: 3.499326229095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7791 , TFLOPS: 69.4932393112381, Tokens per sec: 172211.90232552125, Loss: 3.488314151763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7792 , TFLOPS: 69.55648412645283, Tokens per sec: 172368.6299446149, Loss: 3.4880757331848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7793 , TFLOPS: 68.13175408793168, Tokens per sec: 168837.99196218947, Loss: 3.5272111892700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7794 , TFLOPS: 69.55592866371786, Tokens per sec: 172367.25344676842, Loss: 3.498185157775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7795 , TFLOPS: 68.61913066203478, Tokens per sec: 170045.76480177869, Loss: 3.5079522132873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7796 , TFLOPS: 68.64356421294465, Tokens per sec: 170106.31383251116, Loss: 3.5191991329193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7797 , TFLOPS: 69.3962292343817, Tokens per sec: 171971.50066852898, Loss: 3.4898881912231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7798 , TFLOPS: 69.32231439539378, Tokens per sec: 171788.3315551252, Loss: 3.509126663208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7799 , TFLOPS: 68.00896331281653, Tokens per sec: 168533.70289493358, Loss: 3.506554365158081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7800 , TFLOPS: 69.49283685866124, Tokens per sec: 172210.90500370198, Loss: 3.503211498260498 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/optimizer.pt +[rank0]:[2024-08-25 13:08:00,097] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006388940004399046, 'preprocessing_with_comm': 0.001893694992759265, 'state_converting': 0.9813758340023924, : 0.9910832329915138}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800, took 4.28s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0005800 + eval ppl=23.981840133666992, eval loss=3.1772968769073486 +------------------------------------------------------------------ +iteration: 7801 , TFLOPS: 64.29117416361863, Tokens per sec: 159320.61183493727, Loss: 3.4791383743286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7802 , TFLOPS: 69.5916104371302, Tokens per sec: 172455.676811956, Loss: 3.50966477394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7803 , TFLOPS: 67.57434947941654, Tokens per sec: 167456.68193910233, Loss: 3.510443687438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7804 , TFLOPS: 69.56164931798558, Tokens per sec: 172381.4298582259, Loss: 3.4697725772857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7805 , TFLOPS: 69.6165636920726, Tokens per sec: 172517.5137265312, Loss: 3.4836153984069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7806 , TFLOPS: 67.83773115174408, Tokens per sec: 168109.37073701635, Loss: 3.5223686695098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7807 , TFLOPS: 70.52584008757378, Tokens per sec: 174770.80079964697, Loss: 3.499070167541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7808 , TFLOPS: 69.48152364689297, Tokens per sec: 172182.86961868705, Loss: 3.5071513652801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7809 , TFLOPS: 66.91829028599663, Tokens per sec: 165830.89498692148, Loss: 3.46791934967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7810 , TFLOPS: 68.57021292649841, Tokens per sec: 169924.54126438618, Loss: 3.5087924003601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7811 , TFLOPS: 69.58895985098574, Tokens per sec: 172449.10836750385, Loss: 3.4885919094085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7812 , TFLOPS: 68.79590877499253, Tokens per sec: 170483.8404394047, Loss: 3.49735689163208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7813 , TFLOPS: 68.01020308851567, Tokens per sec: 168536.7751956886, Loss: 3.4997518062591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7814 , TFLOPS: 69.53478860804756, Tokens per sec: 172314.866060048, Loss: 3.504925489425659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7815 , TFLOPS: 68.8878876816001, Tokens per sec: 170711.77430229133, Loss: 3.5138967037200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7816 , TFLOPS: 67.12481235836489, Tokens per sec: 166342.6794325344, Loss: 3.5017073154449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7817 , TFLOPS: 70.52682654575582, Tokens per sec: 174773.24535168937, Loss: 3.502443313598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7818 , TFLOPS: 68.66267210140879, Tokens per sec: 170153.66528503297, Loss: 3.5138039588928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7819 , TFLOPS: 67.80110956284193, Tokens per sec: 168018.61840551574, Loss: 3.486023187637329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7820 , TFLOPS: 69.16646537437836, Tokens per sec: 171402.12051286205, Loss: 3.4927845001220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7821 , TFLOPS: 68.35033721142615, Tokens per sec: 169379.66502112756, Loss: 3.4875569343566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7822 , TFLOPS: 69.7899595948279, Tokens per sec: 172947.2078746944, Loss: 3.487525463104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7823 , TFLOPS: 68.17391790301619, Tokens per sec: 168942.47854069652, Loss: 3.472888469696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7824 , TFLOPS: 69.64522699099193, Tokens per sec: 172588.54453877633, Loss: 3.504343032836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7825 , TFLOPS: 68.67284895112522, Tokens per sec: 170178.88463970285, Loss: 3.480894088745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7826 , TFLOPS: 67.1669454883778, Tokens per sec: 166447.08997005384, Loss: 3.504767894744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7827 , TFLOPS: 70.60301903934888, Tokens per sec: 174962.05874410903, Loss: 3.4780261516571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7828 , TFLOPS: 68.55309151043078, Tokens per sec: 169882.1125092909, Loss: 3.4856200218200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7829 , TFLOPS: 69.11295012011735, Tokens per sec: 171269.50381761132, Loss: 3.5025177001953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7830 , TFLOPS: 69.01257108362387, Tokens per sec: 171020.75350751678, Loss: 3.499382495880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7831 , TFLOPS: 69.63446194256562, Tokens per sec: 172561.86756290798, Loss: 3.509068727493286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7832 , TFLOPS: 67.82096338761232, Tokens per sec: 168067.8183703761, Loss: 3.5124504566192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7833 , TFLOPS: 67.96391327653421, Tokens per sec: 168422.06394235478, Loss: 3.5153892040252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7834 , TFLOPS: 70.4793062112283, Tokens per sec: 174655.48472793322, Loss: 3.4888083934783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7835 , TFLOPS: 68.34494648455205, Tokens per sec: 169366.30620609596, Loss: 3.5060250759124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7836 , TFLOPS: 66.81514290535358, Tokens per sec: 165575.28441506604, Loss: 3.472722053527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7837 , TFLOPS: 69.87175729720813, Tokens per sec: 173149.91159195337, Loss: 3.5204005241394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7838 , TFLOPS: 69.67185232597073, Tokens per sec: 172654.52505187658, Loss: 3.5277791023254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7839 , TFLOPS: 68.79013884592713, Tokens per sec: 170469.5419195705, Loss: 3.5117595195770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7840 , TFLOPS: 68.05807394329517, Tokens per sec: 168655.4044472422, Loss: 3.5081896781921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7841 , TFLOPS: 69.50073197761287, Tokens per sec: 172230.4699781833, Loss: 3.4778027534484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7842 , TFLOPS: 69.80446260835188, Tokens per sec: 172983.14793984563, Loss: 3.500216484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7843 , TFLOPS: 66.85200898563747, Tokens per sec: 165666.6426231435, Loss: 3.5017268657684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7844 , TFLOPS: 69.61171664564243, Tokens per sec: 172505.502211531, Loss: 3.500192403793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7845 , TFLOPS: 70.5499263473041, Tokens per sec: 174830.48920457924, Loss: 3.4783496856689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7846 , TFLOPS: 67.32171280665679, Tokens per sec: 166830.6204337767, Loss: 3.4690356254577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7847 , TFLOPS: 68.86808057476256, Tokens per sec: 170662.69010961495, Loss: 3.5146303176879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7848 , TFLOPS: 69.49215074527467, Tokens per sec: 172209.20473914847, Loss: 3.4866700172424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7849 , TFLOPS: 68.66113132411122, Tokens per sec: 170149.847069159, Loss: 3.494940757751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7850 , TFLOPS: 68.46283273256643, Tokens per sec: 169658.44131491784, Loss: 3.4996495246887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7851 , TFLOPS: 68.57725535472538, Tokens per sec: 169941.99317732008, Loss: 3.4863784313201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7852 , TFLOPS: 69.70376182209121, Tokens per sec: 172733.60029838432, Loss: 3.4955854415893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7853 , TFLOPS: 65.11622290141443, Tokens per sec: 161365.17349381663, Loss: 3.487987518310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7854 , TFLOPS: 68.7934495329141, Tokens per sec: 170477.74616662652, Loss: 3.4911019802093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7855 , TFLOPS: 67.5420486220604, Tokens per sec: 167376.63685634063, Loss: 3.4882614612579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7856 , TFLOPS: 66.94814664923425, Tokens per sec: 165904.88234397426, Loss: 3.5054361820220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7857 , TFLOPS: 69.01805398037074, Tokens per sec: 171034.34072964647, Loss: 3.5001909732818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7858 , TFLOPS: 68.9917383146611, Tokens per sec: 170969.12761110812, Loss: 3.5079970359802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7859 , TFLOPS: 68.09598269643908, Tokens per sec: 168749.3465135267, Loss: 3.505032777786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7860 , TFLOPS: 66.99473992004143, Tokens per sec: 166020.3456017069, Loss: 3.4712021350860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7861 , TFLOPS: 69.00250860099636, Tokens per sec: 170995.8175670919, Loss: 3.490119218826294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7862 , TFLOPS: 68.16707865136104, Tokens per sec: 168925.53012168928, Loss: 3.4959816932678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7863 , TFLOPS: 66.95764477329764, Tokens per sec: 165928.4197417081, Loss: 3.4889369010925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7864 , TFLOPS: 70.15357403315035, Tokens per sec: 173848.28450829443, Loss: 3.4960367679595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7865 , TFLOPS: 68.30292909721138, Tokens per sec: 169262.18249166844, Loss: 3.478229284286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7866 , TFLOPS: 66.16148666202982, Tokens per sec: 163955.4522379303, Loss: 3.4671125411987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7867 , TFLOPS: 68.90687825433983, Tokens per sec: 170758.83503352464, Loss: 3.5109004974365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7868 , TFLOPS: 69.05953082797609, Tokens per sec: 171137.12492706446, Loss: 3.4884297847747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7869 , TFLOPS: 66.89951116990528, Tokens per sec: 165784.3582685567, Loss: 3.5210254192352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7870 , TFLOPS: 68.12725535146747, Tokens per sec: 168826.84359765437, Loss: 3.4895992279052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7871 , TFLOPS: 68.97886916553858, Tokens per sec: 170937.2364419287, Loss: 3.4664864540100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7872 , TFLOPS: 69.12079104309684, Tokens per sec: 171288.9344885035, Loss: 3.4936161041259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7873 , TFLOPS: 64.69917487039393, Tokens per sec: 160331.6825313146, Loss: 3.4996213912963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7874 , TFLOPS: 70.17616411995319, Tokens per sec: 173904.26523189535, Loss: 3.504368782043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7875 , TFLOPS: 68.99904623066776, Tokens per sec: 170987.23743200602, Loss: 3.4717750549316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7876 , TFLOPS: 66.1917749870389, Tokens per sec: 164030.51004383722, Loss: 3.4964866638183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7877 , TFLOPS: 68.9770034519408, Tokens per sec: 170932.6129981081, Loss: 3.488232135772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7878 , TFLOPS: 68.98767883042727, Tokens per sec: 170959.06776198756, Loss: 3.4751856327056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7879 , TFLOPS: 68.11648870908849, Tokens per sec: 168800.1626717957, Loss: 3.4976110458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7880 , TFLOPS: 65.73242324316855, Tokens per sec: 162892.18582075465, Loss: 3.4979989528656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7881 , TFLOPS: 70.26743562613134, Tokens per sec: 174130.44607859128, Loss: 3.50217604637146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7882 , TFLOPS: 68.10876132865243, Tokens per sec: 168781.01337183395, Loss: 3.490800380706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7883 , TFLOPS: 66.94691538367455, Tokens per sec: 165901.8311322822, Loss: 3.5141468048095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7884 , TFLOPS: 69.05022458940051, Tokens per sec: 171114.0630427074, Loss: 3.5056073665618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7885 , TFLOPS: 68.9754537834874, Tokens per sec: 170928.77274897046, Loss: 3.501833915710449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7886 , TFLOPS: 67.19949361385221, Tokens per sec: 166527.74781045143, Loss: 3.4958455562591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7887 , TFLOPS: 68.17328813497593, Tokens per sec: 168940.91790611806, Loss: 3.501769781112671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7888 , TFLOPS: 67.62993250018422, Tokens per sec: 167594.42278753984, Loss: 3.4910428524017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7889 , TFLOPS: 69.27683755473491, Tokens per sec: 171675.63493428496, Loss: 3.5231781005859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7890 , TFLOPS: 67.2543149459992, Tokens per sec: 166663.6011105799, Loss: 3.488697052001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7891 , TFLOPS: 69.09778297533475, Tokens per sec: 171231.91796204116, Loss: 3.4915528297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7892 , TFLOPS: 67.86162904506578, Tokens per sec: 168168.59234923776, Loss: 3.4938766956329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7893 , TFLOPS: 66.1855571203955, Tokens per sec: 164015.1014853403, Loss: 3.4964380264282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7894 , TFLOPS: 70.23101796686088, Tokens per sec: 174040.1991071827, Loss: 3.490386486053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7895 , TFLOPS: 68.9366147782302, Tokens per sec: 170832.5254154735, Loss: 3.4957962036132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7896 , TFLOPS: 66.9387195438572, Tokens per sec: 165881.52093836854, Loss: 3.4602527618408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7897 , TFLOPS: 68.54879886415917, Tokens per sec: 169871.4748589558, Loss: 3.4853014945983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7898 , TFLOPS: 68.97386187059408, Tokens per sec: 170924.82781345714, Loss: 3.487548351287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7899 , TFLOPS: 67.155982651729, Tokens per sec: 166419.92285318195, Loss: 3.4966354370117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7900 , TFLOPS: 66.98907988181855, Tokens per sec: 166006.31940348598, Loss: 3.5058705806732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7901 , TFLOPS: 70.32565866041114, Tokens per sec: 174274.7291713333, Loss: 3.504857063293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7902 , TFLOPS: 66.52944103480387, Tokens per sec: 164867.28370718466, Loss: 3.494635581970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7903 , TFLOPS: 67.95840450371779, Tokens per sec: 168408.41259647426, Loss: 3.5024256706237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7904 , TFLOPS: 69.53485354032422, Tokens per sec: 172315.02696938245, Loss: 3.5020411014556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7905 , TFLOPS: 69.1193546608872, Tokens per sec: 171285.37497515747, Loss: 3.494506359100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7906 , TFLOPS: 67.0157339995491, Tokens per sec: 166072.37124341572, Loss: 3.5030922889709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7907 , TFLOPS: 67.20352206343875, Tokens per sec: 166537.7307522972, Loss: 3.5027873516082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7908 , TFLOPS: 69.23485965417062, Tokens per sec: 171571.60907243914, Loss: 3.484090566635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7909 , TFLOPS: 69.16309261271692, Tokens per sec: 171393.76243792463, Loss: 3.487806797027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7910 , TFLOPS: 65.87616711244252, Tokens per sec: 163248.39896350988, Loss: 3.4973175525665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7911 , TFLOPS: 69.07617118936517, Tokens per sec: 171178.36157567374, Loss: 3.4704012870788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7912 , TFLOPS: 70.10208600231064, Tokens per sec: 173720.69149599882, Loss: 3.5092661380767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7913 , TFLOPS: 66.21864189925174, Tokens per sec: 164097.08921193535, Loss: 3.5006661415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7914 , TFLOPS: 68.167771999116, Tokens per sec: 168927.24831380465, Loss: 3.5015764236450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7915 , TFLOPS: 68.96682798795372, Tokens per sec: 170907.3970774282, Loss: 3.5080180168151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7916 , TFLOPS: 66.92945360091223, Tokens per sec: 165858.558911023, Loss: 3.488269805908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7917 , TFLOPS: 67.47555507877088, Tokens per sec: 167211.8585312007, Loss: 3.5024757385253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7918 , TFLOPS: 69.57795897566331, Tokens per sec: 172421.84698660838, Loss: 3.496717929840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7919 , TFLOPS: 68.84012705710443, Tokens per sec: 170593.4182135239, Loss: 3.5336410999298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7920 , TFLOPS: 67.82557349066629, Tokens per sec: 168079.24271358864, Loss: 3.4861345291137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7921 , TFLOPS: 69.46063712105209, Tokens per sec: 172131.1105068126, Loss: 3.4921021461486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7922 , TFLOPS: 68.50320459857055, Tokens per sec: 169758.48724620423, Loss: 3.51328182220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7923 , TFLOPS: 68.41107386310536, Tokens per sec: 169530.17713468312, Loss: 3.4957103729248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7924 , TFLOPS: 67.97268149967687, Tokens per sec: 168443.79256520182, Loss: 3.481363534927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7925 , TFLOPS: 67.8381776638863, Tokens per sec: 168110.47724329212, Loss: 3.475430965423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7926 , TFLOPS: 67.673140997132, Tokens per sec: 167701.4982027853, Loss: 3.4764907360076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7927 , TFLOPS: 66.94204185567561, Tokens per sec: 165889.7539930374, Loss: 3.508890390396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7928 , TFLOPS: 69.01660544547006, Tokens per sec: 171030.75110059333, Loss: 3.520596504211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7929 , TFLOPS: 69.25078528673562, Tokens per sec: 171611.07454428848, Loss: 3.508002281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7930 , TFLOPS: 67.77123775987398, Tokens per sec: 167944.59278711045, Loss: 3.506373167037964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7931 , TFLOPS: 70.56081835967947, Tokens per sec: 174857.4808110996, Loss: 3.4854447841644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7932 , TFLOPS: 68.76849813030422, Tokens per sec: 170415.91384233534, Loss: 3.4916019439697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7933 , TFLOPS: 67.88809802530866, Tokens per sec: 168234.18539807852, Loss: 3.4980359077453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7934 , TFLOPS: 68.88028379781291, Tokens per sec: 170692.9310406297, Loss: 3.4824399948120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7935 , TFLOPS: 69.18534027838957, Tokens per sec: 171448.89460424418, Loss: 3.510634422302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7936 , TFLOPS: 66.41736128942756, Tokens per sec: 164589.537751539, Loss: 3.503861904144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7937 , TFLOPS: 67.28197874927984, Tokens per sec: 166732.15506252888, Loss: 3.474088430404663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7938 , TFLOPS: 68.9140788505015, Tokens per sec: 170776.6789040244, Loss: 3.4816207885742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7939 , TFLOPS: 68.85400366011677, Tokens per sec: 170627.80596442253, Loss: 3.4820542335510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7940 , TFLOPS: 66.32727820841004, Tokens per sec: 164366.30195330697, Loss: 3.531430244445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7941 , TFLOPS: 70.21995814557145, Tokens per sec: 174012.7916516876, Loss: 3.4792277812957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7942 , TFLOPS: 67.7554144717088, Tokens per sec: 167905.38093595803, Loss: 3.5150513648986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7943 , TFLOPS: 67.84862692348253, Tokens per sec: 168136.37165965256, Loss: 3.531320095062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7944 , TFLOPS: 68.34214244927874, Tokens per sec: 169359.3574977987, Loss: 3.498051643371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7945 , TFLOPS: 69.07248664218147, Tokens per sec: 171169.23086186618, Loss: 3.511822462081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7946 , TFLOPS: 68.15665264566616, Tokens per sec: 168899.69333105802, Loss: 3.502284526824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7947 , TFLOPS: 66.67360541023702, Tokens per sec: 165224.53891052498, Loss: 3.481701374053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7948 , TFLOPS: 70.26536544783657, Tokens per sec: 174125.31594872786, Loss: 3.4743828773498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7949 , TFLOPS: 68.89360042805261, Tokens per sec: 170725.9311173116, Loss: 3.5049147605895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7950 , TFLOPS: 67.57599828521285, Tokens per sec: 167460.7678614965, Loss: 3.4989571571350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7951 , TFLOPS: 68.37714889664198, Tokens per sec: 169446.1073306433, Loss: 3.4874491691589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7952 , TFLOPS: 69.05037204891127, Tokens per sec: 171114.42846361906, Loss: 3.478769063949585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7953 , TFLOPS: 67.20830514199952, Tokens per sec: 166549.58374787043, Loss: 3.5065908432006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7954 , TFLOPS: 66.91047165467822, Tokens per sec: 165811.51955722112, Loss: 3.523662805557251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7955 , TFLOPS: 68.86552560900529, Tokens per sec: 170656.3586230157, Loss: 3.5026142597198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7956 , TFLOPS: 69.20266267920194, Tokens per sec: 171491.8214216772, Loss: 3.488769054412842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7957 , TFLOPS: 66.11701458988493, Tokens per sec: 163845.24554415207, Loss: 3.488004207611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7958 , TFLOPS: 67.85021872594382, Tokens per sec: 168140.31632150314, Loss: 3.4890081882476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7959 , TFLOPS: 69.1895331367872, Tokens per sec: 171459.28496923496, Loss: 3.477085590362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7960 , TFLOPS: 66.05247769249131, Tokens per sec: 163685.31600308453, Loss: 3.5155222415924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7961 , TFLOPS: 70.48952791497067, Tokens per sec: 174680.8152358204, Loss: 3.488349199295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7962 , TFLOPS: 68.59447540574378, Tokens per sec: 169984.66636068892, Loss: 3.5074286460876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7963 , TFLOPS: 69.01414533382335, Tokens per sec: 171024.6546729291, Loss: 3.4745700359344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7964 , TFLOPS: 68.70232162012711, Tokens per sec: 170251.92116017206, Loss: 3.4828569889068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7965 , TFLOPS: 69.51687554023933, Tokens per sec: 172270.47550473188, Loss: 3.4956774711608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7966 , TFLOPS: 67.53578777057311, Tokens per sec: 167361.1217766654, Loss: 3.4756388664245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7967 , TFLOPS: 67.70828147154052, Tokens per sec: 167788.58016941242, Loss: 3.4956469535827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7968 , TFLOPS: 70.6041696808702, Tokens per sec: 174964.91016055248, Loss: 3.494640588760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7969 , TFLOPS: 67.6920001957334, Tokens per sec: 167748.2333744318, Loss: 3.501919984817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7970 , TFLOPS: 68.71717777160526, Tokens per sec: 170288.73634007512, Loss: 3.5098559856414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7971 , TFLOPS: 69.93580922151389, Tokens per sec: 173308.63931628497, Loss: 3.4848146438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7972 , TFLOPS: 69.54122689707935, Tokens per sec: 172330.82084950487, Loss: 3.49104380607605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7973 , TFLOPS: 68.19058164531744, Tokens per sec: 168983.77312978174, Loss: 3.5093488693237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7974 , TFLOPS: 67.92887725219056, Tokens per sec: 168335.24081449452, Loss: 3.500002145767212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7975 , TFLOPS: 68.50838414989754, Tokens per sec: 169771.32274496718, Loss: 3.5110151767730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7976 , TFLOPS: 67.54092869120086, Tokens per sec: 167373.86154430013, Loss: 3.5004448890686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7977 , TFLOPS: 64.63906420513389, Tokens per sec: 160182.72168106382, Loss: 3.4614782333374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7978 , TFLOPS: 70.08110132190154, Tokens per sec: 173668.6891462917, Loss: 3.525578498840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7979 , TFLOPS: 69.30456763703172, Tokens per sec: 171744.3531329372, Loss: 3.5042877197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7980 , TFLOPS: 66.19607415438267, Tokens per sec: 164041.16385410746, Loss: 3.4973361492156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7981 , TFLOPS: 69.0235848478118, Tokens per sec: 171048.0468284408, Loss: 3.503652334213257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7982 , TFLOPS: 68.75498010958736, Tokens per sec: 170382.41469786604, Loss: 3.5098578929901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7983 , TFLOPS: 68.0822981880469, Tokens per sec: 168715.43479425783, Loss: 3.503901958465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7984 , TFLOPS: 67.6617863429519, Tokens per sec: 167673.3601189086, Loss: 3.5079638957977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7985 , TFLOPS: 68.92059168703604, Tokens per sec: 170792.81843040517, Loss: 3.4873178005218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7986 , TFLOPS: 68.09194036291089, Tokens per sec: 168739.32916574404, Loss: 3.499128818511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7987 , TFLOPS: 66.80039918945374, Tokens per sec: 165538.74786291184, Loss: 3.5111377239227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7988 , TFLOPS: 68.90206766023569, Tokens per sec: 170746.91385140386, Loss: 3.5013201236724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7989 , TFLOPS: 67.78787153767504, Tokens per sec: 167985.81312086334, Loss: 3.491158962249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7990 , TFLOPS: 67.79815967112185, Tokens per sec: 168011.30824887598, Loss: 3.4745965003967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7991 , TFLOPS: 68.09548678493235, Tokens per sec: 168748.11759018386, Loss: 3.522705554962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7992 , TFLOPS: 68.86361052351694, Tokens per sec: 170651.6128302115, Loss: 3.5025582313537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7993 , TFLOPS: 68.34094445995729, Tokens per sec: 169356.38874829072, Loss: 3.5115838050842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7994 , TFLOPS: 67.75793423442255, Tokens per sec: 167911.62518553747, Loss: 3.496727705001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7995 , TFLOPS: 68.52547424533617, Tokens per sec: 169813.67388409353, Loss: 3.489969491958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7996 , TFLOPS: 69.5739401284669, Tokens per sec: 172411.88784054315, Loss: 3.476767063140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7997 , TFLOPS: 67.02313019031186, Tokens per sec: 166090.69981888353, Loss: 3.4883065223693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7998 , TFLOPS: 70.57245389978205, Tokens per sec: 174886.31496123434, Loss: 3.4862914085388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7999 , TFLOPS: 68.20764508455866, Tokens per sec: 169026.05821191453, Loss: 3.4960172176361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8000 , TFLOPS: 66.77101121836664, Tokens per sec: 165465.92123320577, Loss: 3.491941213607788 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/optimizer.pt +[rank0]:[2024-08-25 13:33:56,414] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006431093992432579, 'preprocessing_with_comm': 0.0019200090027879924, 'state_converting': 1.0148011230048724, : 1.024650245992234}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000, took 4.70s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006000 + eval ppl=23.669631958007812, eval loss=3.1641929149627686 +------------------------------------------------------------------ +iteration: 8001 , TFLOPS: 65.69450624073104, Tokens per sec: 162798.22331181186, Loss: 3.505859136581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8002 , TFLOPS: 68.45050700484794, Tokens per sec: 169627.8968038404, Loss: 3.4999852180480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8003 , TFLOPS: 68.11834205289836, Tokens per sec: 168804.75546191924, Loss: 3.499321699142456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8004 , TFLOPS: 67.42569949357312, Tokens per sec: 167088.31089903458, Loss: 3.4980990886688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8005 , TFLOPS: 69.47468739140336, Tokens per sec: 172165.92862450812, Loss: 3.5097429752349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8006 , TFLOPS: 68.68801520931574, Tokens per sec: 170216.4682982002, Loss: 3.518357038497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8007 , TFLOPS: 67.07160966964211, Tokens per sec: 166210.8373091194, Loss: 3.5158133506774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8008 , TFLOPS: 70.51706470857665, Tokens per sec: 174749.05444380426, Loss: 3.5107245445251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8009 , TFLOPS: 68.33386476980671, Tokens per sec: 169338.84449620533, Loss: 3.5118701457977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8010 , TFLOPS: 67.65649824581921, Tokens per sec: 167660.2556316818, Loss: 3.497831106185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8011 , TFLOPS: 69.16542210755057, Tokens per sec: 171399.53518273745, Loss: 3.512165069580078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8012 , TFLOPS: 67.9647398427267, Tokens per sec: 168424.11226443356, Loss: 3.5133113861083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8013 , TFLOPS: 68.00822725822403, Tokens per sec: 168531.87887057025, Loss: 3.484391212463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8014 , TFLOPS: 66.75043813887338, Tokens per sec: 165414.93887590963, Loss: 3.4980082511901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8015 , TFLOPS: 70.20924172425912, Tokens per sec: 173986.2351791644, Loss: 3.489450454711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8016 , TFLOPS: 67.97086078651652, Tokens per sec: 168439.2806374224, Loss: 3.498511791229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8017 , TFLOPS: 65.91804457933829, Tokens per sec: 163352.17593966142, Loss: 3.484858989715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8018 , TFLOPS: 70.26229566537904, Tokens per sec: 174117.70868963367, Loss: 3.520312786102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8019 , TFLOPS: 69.06119874824753, Tokens per sec: 171141.25821723373, Loss: 3.513237714767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8020 , TFLOPS: 66.46205388102877, Tokens per sec: 164700.29091682215, Loss: 3.5045950412750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8021 , TFLOPS: 68.5980657922146, Tokens per sec: 169993.5637338773, Loss: 3.4938583374023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8022 , TFLOPS: 69.02737121319522, Tokens per sec: 171057.4298589635, Loss: 3.497617244720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8023 , TFLOPS: 68.07509901968595, Tokens per sec: 168697.59446200554, Loss: 3.4983041286468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8024 , TFLOPS: 67.18217915205466, Tokens per sec: 166484.84066677166, Loss: 3.4931387901306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8025 , TFLOPS: 70.35511917707079, Tokens per sec: 174347.73557695973, Loss: 3.4737730026245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8026 , TFLOPS: 68.28601075095897, Tokens per sec: 169220.25696594527, Loss: 3.4915027618408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8027 , TFLOPS: 66.94063803393931, Tokens per sec: 165886.2751681311, Loss: 3.4871697425842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8028 , TFLOPS: 68.22448516032122, Tokens per sec: 169067.78977474163, Loss: 3.4718456268310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8029 , TFLOPS: 68.86186020271299, Tokens per sec: 170647.27534244553, Loss: 3.476969003677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8030 , TFLOPS: 67.19885547549862, Tokens per sec: 166526.16643331447, Loss: 3.501108407974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8031 , TFLOPS: 69.5207439903695, Tokens per sec: 172280.0619502994, Loss: 3.490987539291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8032 , TFLOPS: 67.87397209953996, Tokens per sec: 168199.1797979245, Loss: 3.4856956005096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8033 , TFLOPS: 68.12111007417076, Tokens per sec: 168811.61492355494, Loss: 3.5154547691345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8034 , TFLOPS: 66.92510249657359, Tokens per sec: 165847.77639515814, Loss: 3.5044119358062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8035 , TFLOPS: 70.10530584998719, Tokens per sec: 173728.67063323682, Loss: 3.5132524967193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8036 , TFLOPS: 68.08153087515258, Tokens per sec: 168713.53330837924, Loss: 3.5047333240509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8037 , TFLOPS: 67.13242594582803, Tokens per sec: 166361.54674097395, Loss: 3.4874422550201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8038 , TFLOPS: 70.17949109609863, Tokens per sec: 173912.50984528044, Loss: 3.5122084617614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8039 , TFLOPS: 67.8292242383495, Tokens per sec: 168088.28966851067, Loss: 3.485558032989502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8040 , TFLOPS: 67.18713759605659, Tokens per sec: 166497.12823722616, Loss: 3.514939308166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8041 , TFLOPS: 69.0480856873627, Tokens per sec: 171108.7626078382, Loss: 3.4882760047912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8042 , TFLOPS: 69.50957328147186, Tokens per sec: 172252.3797030959, Loss: 3.4836056232452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8043 , TFLOPS: 68.6245917352859, Tokens per sec: 170059.29794288185, Loss: 3.497519016265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8044 , TFLOPS: 68.13316562133261, Tokens per sec: 168841.48989745017, Loss: 3.4950344562530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8045 , TFLOPS: 70.268090829881, Tokens per sec: 174132.06974565957, Loss: 3.4824607372283936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8046 , TFLOPS: 67.9521415833104, Tokens per sec: 168392.89238978713, Loss: 3.4981372356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8047 , TFLOPS: 65.88596743864035, Tokens per sec: 163272.68525142322, Loss: 3.482788562774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8048 , TFLOPS: 69.20155061661671, Tokens per sec: 171489.06560808717, Loss: 3.515906572341919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8049 , TFLOPS: 68.79280472714012, Tokens per sec: 170476.14826688738, Loss: 3.4727871417999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8050 , TFLOPS: 67.09524926973846, Tokens per sec: 166269.41884227507, Loss: 3.519653558731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8051 , TFLOPS: 67.71402745672488, Tokens per sec: 167802.8193536718, Loss: 3.5140745639801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8052 , TFLOPS: 69.0450135344239, Tokens per sec: 171101.14947443025, Loss: 3.512293577194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8053 , TFLOPS: 68.23544640870006, Tokens per sec: 169094.95295570555, Loss: 3.4866831302642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8054 , TFLOPS: 66.77166866765893, Tokens per sec: 165467.55046497606, Loss: 3.5048820972442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8055 , TFLOPS: 70.54033085101351, Tokens per sec: 174806.71050773998, Loss: 3.4728152751922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8056 , TFLOPS: 68.72026647097334, Tokens per sec: 170296.39047735752, Loss: 3.490250587463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8057 , TFLOPS: 68.74968748207098, Tokens per sec: 170369.29898384938, Loss: 3.4951870441436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8058 , TFLOPS: 68.9429543940884, Tokens per sec: 170848.2356819365, Loss: 3.494703769683838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8059 , TFLOPS: 69.54419441226281, Tokens per sec: 172338.17467902825, Loss: 3.4899966716766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8060 , TFLOPS: 68.85297307527495, Tokens per sec: 170625.25206165615, Loss: 3.488764524459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8061 , TFLOPS: 66.59065485975344, Tokens per sec: 165018.97830866923, Loss: 3.4891326427459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8062 , TFLOPS: 70.5351973311432, Tokens per sec: 174793.9890799974, Loss: 3.495821952819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8063 , TFLOPS: 68.82091550198345, Tokens per sec: 170545.8098636066, Loss: 3.485804557800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8064 , TFLOPS: 67.70527692252708, Tokens per sec: 167781.13456597872, Loss: 3.5189878940582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8065 , TFLOPS: 69.60373555991818, Tokens per sec: 172485.72420191712, Loss: 3.5094196796417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8066 , TFLOPS: 69.67629273531738, Tokens per sec: 172665.52887538902, Loss: 3.496920585632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8067 , TFLOPS: 67.22712822152837, Tokens per sec: 166596.2294124763, Loss: 3.494325637817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8068 , TFLOPS: 69.77156030540975, Tokens per sec: 172901.61240866713, Loss: 3.502899646759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8069 , TFLOPS: 68.53508022592992, Tokens per sec: 169837.47856219095, Loss: 3.4946439266204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8070 , TFLOPS: 67.97251034993499, Tokens per sec: 168443.36843728778, Loss: 3.5012035369873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8071 , TFLOPS: 69.50015458717321, Tokens per sec: 172229.03914106902, Loss: 3.494683027267456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8072 , TFLOPS: 69.61021344009815, Tokens per sec: 172501.77710260052, Loss: 3.4827880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8073 , TFLOPS: 67.842278525624, Tokens per sec: 168120.63963042534, Loss: 3.4826784133911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8074 , TFLOPS: 67.92327764822402, Tokens per sec: 168321.36437901814, Loss: 3.504051446914673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8075 , TFLOPS: 70.65470913755482, Tokens per sec: 175090.1525015977, Loss: 3.4741744995117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8076 , TFLOPS: 67.68802432164703, Tokens per sec: 167738.3807204671, Loss: 3.4967689514160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8077 , TFLOPS: 69.62364401362714, Tokens per sec: 172535.05954330516, Loss: 3.496882677078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8078 , TFLOPS: 69.01169986242382, Tokens per sec: 171018.59452541015, Loss: 3.4767634868621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8079 , TFLOPS: 69.59799507219873, Tokens per sec: 172471.4986409814, Loss: 3.4828617572784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8080 , TFLOPS: 68.77018215221621, Tokens per sec: 170420.08703414357, Loss: 3.4911723136901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8081 , TFLOPS: 67.76370888508691, Tokens per sec: 167925.93540601403, Loss: 3.5054197311401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8082 , TFLOPS: 70.61308852787185, Tokens per sec: 174987.0120459147, Loss: 3.5121426582336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8083 , TFLOPS: 68.73153230210596, Tokens per sec: 170324.3084479379, Loss: 3.499845027923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8084 , TFLOPS: 66.24898895613843, Tokens per sec: 164172.29256190488, Loss: 3.484127998352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8085 , TFLOPS: 70.65074698982889, Tokens per sec: 175080.33386306628, Loss: 3.4979934692382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8086 , TFLOPS: 69.55705662293718, Tokens per sec: 172370.04865396884, Loss: 3.49223256111145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8087 , TFLOPS: 68.06672293812143, Tokens per sec: 168676.83760918712, Loss: 3.503185272216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8088 , TFLOPS: 68.21321914209078, Tokens per sec: 169039.8713405122, Loss: 3.483741283416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8089 , TFLOPS: 69.58043532491486, Tokens per sec: 172427.98365284613, Loss: 3.521714687347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8090 , TFLOPS: 68.82721351674584, Tokens per sec: 170561.41703797143, Loss: 3.496777057647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8091 , TFLOPS: 68.81835709316667, Tokens per sec: 170539.46984472664, Loss: 3.5066750049591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8092 , TFLOPS: 69.68365862769164, Tokens per sec: 172683.7823681125, Loss: 3.4936373233795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8093 , TFLOPS: 69.77686835959472, Tokens per sec: 172914.7663516679, Loss: 3.5179033279418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8094 , TFLOPS: 68.21664453688679, Tokens per sec: 169048.35984615513, Loss: 3.4881274700164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8095 , TFLOPS: 68.99331052667253, Tokens per sec: 170973.02372566584, Loss: 3.5166726112365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8096 , TFLOPS: 68.68199349046769, Tokens per sec: 170201.54581554767, Loss: 3.486417055130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8097 , TFLOPS: 69.58004456344338, Tokens per sec: 172427.0153028739, Loss: 3.5168285369873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8098 , TFLOPS: 67.90323954589081, Tokens per sec: 168271.70775405728, Loss: 3.4823572635650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8099 , TFLOPS: 69.65164729029645, Tokens per sec: 172604.45474770732, Loss: 3.500669240951538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8100 , TFLOPS: 68.93677797273892, Tokens per sec: 170832.92982944346, Loss: 3.510117530822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8101 , TFLOPS: 67.98334831397591, Tokens per sec: 168470.22610608142, Loss: 3.4950573444366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8102 , TFLOPS: 69.53489260938778, Tokens per sec: 172315.12378682577, Loss: 3.4978818893432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8103 , TFLOPS: 68.71544606165472, Tokens per sec: 170284.44497205404, Loss: 3.501309394836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8104 , TFLOPS: 68.00820235209495, Tokens per sec: 168531.8171504388, Loss: 3.4786553382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8105 , TFLOPS: 70.58211948920543, Tokens per sec: 174910.26735657806, Loss: 3.506334066390991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8106 , TFLOPS: 67.29057570176789, Tokens per sec: 166753.45925782676, Loss: 3.520031213760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8107 , TFLOPS: 67.10558544022118, Tokens per sec: 166295.03301135442, Loss: 3.506492853164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8108 , TFLOPS: 68.19169288734119, Tokens per sec: 168986.52690993, Loss: 3.4927902221679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8109 , TFLOPS: 69.02965977615315, Tokens per sec: 171063.10117007923, Loss: 3.4998457431793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8110 , TFLOPS: 67.87263194095259, Tokens per sec: 168195.85873731432, Loss: 3.509955883026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8111 , TFLOPS: 67.34603852925352, Tokens per sec: 166890.90225407097, Loss: 3.5043768882751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8112 , TFLOPS: 70.18904827704388, Tokens per sec: 173936.1935924732, Loss: 3.4993021488189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8113 , TFLOPS: 67.1397852784419, Tokens per sec: 166379.78397788855, Loss: 3.463197708129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8114 , TFLOPS: 67.00984557251579, Tokens per sec: 166057.77907256337, Loss: 3.492678642272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8115 , TFLOPS: 69.5116833919046, Tokens per sec: 172257.60878919603, Loss: 3.4876809120178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8116 , TFLOPS: 69.51553519986109, Tokens per sec: 172267.15399362394, Loss: 3.4944067001342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8117 , TFLOPS: 68.76275019537393, Tokens per sec: 170401.66982639075, Loss: 3.481884479522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8118 , TFLOPS: 68.77968611382984, Tokens per sec: 170443.63889796994, Loss: 3.499009132385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8119 , TFLOPS: 69.4932887139985, Tokens per sec: 172212.0247510037, Loss: 3.496349811553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8120 , TFLOPS: 68.80762188298708, Tokens per sec: 170512.8668113206, Loss: 3.524907112121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8121 , TFLOPS: 66.86926501575675, Tokens per sec: 165709.40496668773, Loss: 3.503553867340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8122 , TFLOPS: 70.56822534654438, Tokens per sec: 174875.8361404978, Loss: 3.492257595062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8123 , TFLOPS: 68.71224474524388, Tokens per sec: 170276.51175733388, Loss: 3.500001907348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8124 , TFLOPS: 68.75258833476246, Tokens per sec: 170376.48761637422, Loss: 3.4812710285186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8125 , TFLOPS: 69.19824935568123, Tokens per sec: 171480.88471982512, Loss: 3.4961373805999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8126 , TFLOPS: 68.97785698286147, Tokens per sec: 170934.72814175367, Loss: 3.4906108379364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8127 , TFLOPS: 68.0552162573928, Tokens per sec: 168648.32278677603, Loss: 3.4728846549987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8128 , TFLOPS: 67.33042925346781, Tokens per sec: 166852.22074917032, Loss: 3.4922444820404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8129 , TFLOPS: 70.2585080756839, Tokens per sec: 174108.32262513164, Loss: 3.5100255012512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8130 , TFLOPS: 68.28964043292474, Tokens per sec: 169229.25171769893, Loss: 3.4899141788482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8131 , TFLOPS: 66.80751903438635, Tokens per sec: 165556.39162297052, Loss: 3.4967410564422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8132 , TFLOPS: 68.745946336277, Tokens per sec: 170360.02801245067, Loss: 3.4912450313568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8133 , TFLOPS: 69.04881768632075, Tokens per sec: 171110.57658189256, Loss: 3.516162157058716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8134 , TFLOPS: 66.36101347105095, Tokens per sec: 164449.90165037714, Loss: 3.491447925567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8135 , TFLOPS: 67.75257140277974, Tokens per sec: 167898.3354979619, Loss: 3.492788791656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8136 , TFLOPS: 68.9229265433447, Tokens per sec: 170798.60446154367, Loss: 3.504868507385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8137 , TFLOPS: 68.12416491567056, Tokens per sec: 168819.18515731, Loss: 3.4923062324523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8138 , TFLOPS: 68.22380411799372, Tokens per sec: 169066.1020768311, Loss: 3.4802584648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8139 , TFLOPS: 68.8578113040143, Tokens per sec: 170637.2417254472, Loss: 3.491140604019165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8140 , TFLOPS: 67.9287717155719, Tokens per sec: 168334.97928312767, Loss: 3.505030632019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8141 , TFLOPS: 66.8817640010756, Tokens per sec: 165740.37882918472, Loss: 3.4979538917541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8142 , TFLOPS: 70.19918166416406, Tokens per sec: 173961.3052420417, Loss: 3.518651008605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8143 , TFLOPS: 66.93859170537623, Tokens per sec: 165881.2041405301, Loss: 3.5088298320770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8144 , TFLOPS: 68.7736333391895, Tokens per sec: 170428.639455643, Loss: 3.474057674407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8145 , TFLOPS: 68.30142742041177, Tokens per sec: 169258.46117113566, Loss: 3.482564926147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8146 , TFLOPS: 69.08123547191951, Tokens per sec: 171190.91142572017, Loss: 3.476442337036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8147 , TFLOPS: 66.34088882744214, Tokens per sec: 164400.030566602, Loss: 3.4804248809814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8148 , TFLOPS: 67.09185430163406, Tokens per sec: 166261.00573732573, Loss: 3.4771993160247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8149 , TFLOPS: 70.56532853839899, Tokens per sec: 174868.6575308038, Loss: 3.4840855598449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8150 , TFLOPS: 67.43000341516272, Tokens per sec: 167098.9764908497, Loss: 3.4950942993164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8151 , TFLOPS: 67.6985434215093, Tokens per sec: 167764.44820870028, Loss: 3.515880584716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8152 , TFLOPS: 70.56819277350526, Tokens per sec: 174875.75542091863, Loss: 3.5037753582000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8153 , TFLOPS: 69.56404399013343, Tokens per sec: 172387.36411960318, Loss: 3.4950404167175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8154 , TFLOPS: 65.89884940898484, Tokens per sec: 163304.60819300305, Loss: 3.498748540878296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8155 , TFLOPS: 69.68264618119167, Tokens per sec: 172681.2734141554, Loss: 3.496169090270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8156 , TFLOPS: 69.5479768091273, Tokens per sec: 172347.5478751239, Loss: 3.475372314453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8157 , TFLOPS: 68.74202388321909, Tokens per sec: 170350.3077416793, Loss: 3.500608205795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8158 , TFLOPS: 65.57352268745447, Tokens per sec: 162498.4127393539, Loss: 3.4920589923858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8159 , TFLOPS: 69.57443593769251, Tokens per sec: 172413.1165104221, Loss: 3.5038440227508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8160 , TFLOPS: 69.67954180688085, Tokens per sec: 172673.5804326381, Loss: 3.4957990646362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8161 , TFLOPS: 68.08987385767237, Tokens per sec: 168734.20813811864, Loss: 3.5234670639038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8162 , TFLOPS: 68.91862716451578, Tokens per sec: 170787.95012718995, Loss: 3.505354881286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8163 , TFLOPS: 68.75256580890219, Tokens per sec: 170376.43179481098, Loss: 3.4928431510925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8164 , TFLOPS: 69.55565899991339, Tokens per sec: 172366.585190155, Loss: 3.502376079559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8165 , TFLOPS: 67.81490771675632, Tokens per sec: 168052.81175090774, Loss: 3.5019431114196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8166 , TFLOPS: 69.56415331646284, Tokens per sec: 172387.63504229076, Loss: 3.4844212532043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8167 , TFLOPS: 68.87392104575787, Tokens per sec: 170677.1634401223, Loss: 3.483365058898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8168 , TFLOPS: 67.92942473357748, Tokens per sec: 168336.59753368137, Loss: 3.4995880126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8169 , TFLOPS: 69.58485527244407, Tokens per sec: 172438.93676972095, Loss: 3.505544424057007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8170 , TFLOPS: 68.22695404169227, Tokens per sec: 169073.9079347488, Loss: 3.487945795059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8171 , TFLOPS: 67.87618602925066, Tokens per sec: 168204.6661596321, Loss: 3.5077197551727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8172 , TFLOPS: 69.70196478864274, Tokens per sec: 172729.14705153942, Loss: 3.487381935119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8173 , TFLOPS: 69.56958120764222, Tokens per sec: 172401.0859545648, Loss: 3.4735217094421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8174 , TFLOPS: 67.65877327769303, Tokens per sec: 167665.8934112806, Loss: 3.508668899536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8175 , TFLOPS: 69.08464885275663, Tokens per sec: 171199.37015944952, Loss: 3.475604295730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8176 , TFLOPS: 69.59183643510606, Tokens per sec: 172456.2368598359, Loss: 3.5009777545928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8177 , TFLOPS: 69.72587402080858, Tokens per sec: 172788.39676266574, Loss: 3.488365411758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8178 , TFLOPS: 68.10743992788917, Tokens per sec: 168777.73879517827, Loss: 3.5017526149749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8179 , TFLOPS: 70.57737414427122, Tokens per sec: 174898.5078690882, Loss: 3.5246293544769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8180 , TFLOPS: 68.02755066403834, Tokens per sec: 168579.76439882698, Loss: 3.506326198577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8181 , TFLOPS: 68.12719429241555, Tokens per sec: 168826.69228659754, Loss: 3.5141243934631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8182 , TFLOPS: 69.60235861594035, Tokens per sec: 172482.31198305718, Loss: 3.5012192726135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8183 , TFLOPS: 69.55428425886382, Tokens per sec: 172363.17843039392, Loss: 3.5024774074554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8184 , TFLOPS: 68.16505880118206, Tokens per sec: 168920.5247104409, Loss: 3.5003297328948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8185 , TFLOPS: 68.78791900169513, Tokens per sec: 170464.04090102762, Loss: 3.499271869659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8186 , TFLOPS: 69.65473031077298, Tokens per sec: 172612.09481207086, Loss: 3.5010764598846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8187 , TFLOPS: 67.97888907208228, Tokens per sec: 168459.175613443, Loss: 3.494201898574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8188 , TFLOPS: 68.05985966275851, Tokens per sec: 168659.8296567856, Loss: 3.530071973800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8189 , TFLOPS: 70.54255709458339, Tokens per sec: 174812.2273845471, Loss: 3.5033373832702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8190 , TFLOPS: 68.76812247338121, Tokens per sec: 170414.98292309808, Loss: 3.5056304931640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8191 , TFLOPS: 67.78088841280892, Tokens per sec: 167968.5081681904, Loss: 3.491641044616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8192 , TFLOPS: 69.86185272038813, Tokens per sec: 173125.36695951418, Loss: 3.511927604675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8193 , TFLOPS: 69.71724524854339, Tokens per sec: 172767.01371445484, Loss: 3.4921910762786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8194 , TFLOPS: 68.93546348975285, Tokens per sec: 170829.67239580172, Loss: 3.488553524017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8195 , TFLOPS: 68.1206636424545, Tokens per sec: 168810.50861658357, Loss: 3.5010993480682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8196 , TFLOPS: 70.59773507035625, Tokens per sec: 174948.96448686806, Loss: 3.4707913398742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8197 , TFLOPS: 69.11299428314281, Tokens per sec: 171269.6132584534, Loss: 3.4994709491729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8198 , TFLOPS: 67.93684766693862, Tokens per sec: 168354.99238025388, Loss: 3.507819890975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8199 , TFLOPS: 69.55054197092099, Tokens per sec: 172353.90462862453, Loss: 3.4968714714050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8200 , TFLOPS: 69.54053488648972, Tokens per sec: 172329.1059710325, Loss: 3.507877826690674 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/optimizer.pt +[rank0]:[2024-08-25 13:59:47,624] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006337747006909922, 'preprocessing_with_comm': 0.001377235006657429, 'state_converting': 0.9643843630037736, : 0.9734761510044336}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200, took 4.24s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006200 + eval ppl=24.352449417114258, eval loss=3.1926324367523193 +------------------------------------------------------------------ +iteration: 8201 , TFLOPS: 63.56581831073188, Tokens per sec: 157523.1001269404, Loss: 3.4979512691497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8202 , TFLOPS: 68.20133154809959, Tokens per sec: 169010.41257307547, Loss: 3.5075526237487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8203 , TFLOPS: 69.68042314860233, Tokens per sec: 172675.76449451174, Loss: 3.502209186553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8204 , TFLOPS: 67.8622928117722, Tokens per sec: 168170.23723625625, Loss: 3.5050551891326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8205 , TFLOPS: 67.77440020351902, Tokens per sec: 167952.42967083474, Loss: 3.5193872451782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8206 , TFLOPS: 70.5106128812311, Tokens per sec: 174733.0661049145, Loss: 3.478262186050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8207 , TFLOPS: 68.77406599134741, Tokens per sec: 170429.71161535088, Loss: 3.4808852672576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8208 , TFLOPS: 68.93047137907948, Tokens per sec: 170817.30139562674, Loss: 3.503351926803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8209 , TFLOPS: 68.91249611087001, Tokens per sec: 170772.75670086444, Loss: 3.4965877532958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8210 , TFLOPS: 68.62400336229626, Tokens per sec: 170057.83988979892, Loss: 3.5016372203826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8211 , TFLOPS: 69.07199510462537, Tokens per sec: 171168.01277765483, Loss: 3.4805972576141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8212 , TFLOPS: 68.71074646298513, Tokens per sec: 170272.79884884728, Loss: 3.4962105751037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8213 , TFLOPS: 68.54888949073764, Tokens per sec: 169871.6994416006, Loss: 3.4872024059295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8214 , TFLOPS: 69.69612850376517, Tokens per sec: 172714.68409469296, Loss: 3.500673294067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8215 , TFLOPS: 67.62353224578649, Tokens per sec: 167578.56225209602, Loss: 3.500041961669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8216 , TFLOPS: 70.15981718675539, Tokens per sec: 173863.75573066788, Loss: 3.520796298980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8217 , TFLOPS: 66.83223727128845, Tokens per sec: 165617.6461369525, Loss: 3.4955084323883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8218 , TFLOPS: 66.80945931614042, Tokens per sec: 165561.19985489678, Loss: 3.4782447814941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8219 , TFLOPS: 70.2837682336675, Tokens per sec: 174170.9200792511, Loss: 3.496457576751709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8220 , TFLOPS: 67.86135231805243, Tokens per sec: 168167.9065892147, Loss: 3.490959644317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8221 , TFLOPS: 66.09831293904185, Tokens per sec: 163798.900793206, Loss: 3.4820356369018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8222 , TFLOPS: 69.13784819399353, Tokens per sec: 171331.2039874258, Loss: 3.5110952854156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8223 , TFLOPS: 69.1044016315014, Tokens per sec: 171248.31972112725, Loss: 3.484518051147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8224 , TFLOPS: 66.85034950191299, Tokens per sec: 165662.53023966702, Loss: 3.487070083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8225 , TFLOPS: 68.69644706168643, Tokens per sec: 170237.36335722, Loss: 3.4943037033081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8226 , TFLOPS: 70.66230845819675, Tokens per sec: 175108.9844552827, Loss: 3.4798057079315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8227 , TFLOPS: 68.94860693122061, Tokens per sec: 170862.24329162843, Loss: 3.4855568408966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8228 , TFLOPS: 67.46475367070535, Tokens per sec: 167185.091452141, Loss: 3.4782071113586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8229 , TFLOPS: 69.12530807451235, Tokens per sec: 171300.1281899432, Loss: 3.4825849533081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8230 , TFLOPS: 69.09150989696688, Tokens per sec: 171216.37258280895, Loss: 3.5054001808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8231 , TFLOPS: 68.39769712314555, Tokens per sec: 169497.02809949318, Loss: 3.49955677986145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8232 , TFLOPS: 67.90862297282278, Tokens per sec: 168285.04847902813, Loss: 3.5215933322906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8233 , TFLOPS: 68.95314500463299, Tokens per sec: 170873.48913745437, Loss: 3.5059146881103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8234 , TFLOPS: 69.26166123833084, Tokens per sec: 171638.0263504264, Loss: 3.4778454303741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8235 , TFLOPS: 66.25119565631086, Tokens per sec: 164177.76100802023, Loss: 3.4994688034057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8236 , TFLOPS: 70.34444589123086, Tokens per sec: 174321.2859988889, Loss: 3.5100014209747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8237 , TFLOPS: 68.14037845381074, Tokens per sec: 168859.3640908901, Loss: 3.5088160037994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8238 , TFLOPS: 68.66926530354444, Tokens per sec: 170170.00396622633, Loss: 3.4821360111236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8239 , TFLOPS: 68.7889014680223, Tokens per sec: 170466.47556081394, Loss: 3.492180585861206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8240 , TFLOPS: 69.54066448034176, Tokens per sec: 172329.42711887378, Loss: 3.522179126739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8241 , TFLOPS: 67.7683362937783, Tokens per sec: 167937.40263450245, Loss: 3.5079846382141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8242 , TFLOPS: 67.27264943957762, Tokens per sec: 166709.03600537844, Loss: 3.481983184814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8243 , TFLOPS: 70.17159356978188, Tokens per sec: 173892.9389050832, Loss: 3.499800443649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8244 , TFLOPS: 67.23395688346963, Tokens per sec: 166613.15158900595, Loss: 3.4889135360717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8245 , TFLOPS: 67.1487046092902, Tokens per sec: 166401.88706227604, Loss: 3.4863128662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8246 , TFLOPS: 68.8811855611667, Tokens per sec: 170695.16570955783, Loss: 3.5185775756835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8247 , TFLOPS: 68.83922317316296, Tokens per sec: 170591.1782895459, Loss: 3.5079753398895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8248 , TFLOPS: 67.35322377439604, Tokens per sec: 166908.70808305423, Loss: 3.500115394592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8249 , TFLOPS: 69.25886180638872, Tokens per sec: 171631.08904969096, Loss: 3.4982070922851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8250 , TFLOPS: 67.97115613094071, Tokens per sec: 168440.01253344037, Loss: 3.4900124073028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8251 , TFLOPS: 68.72425992228172, Tokens per sec: 170306.28668961467, Loss: 3.47330904006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8252 , TFLOPS: 68.76611523988251, Tokens per sec: 170410.00877737327, Loss: 3.527369499206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8253 , TFLOPS: 69.53775715376572, Tokens per sec: 172322.2224433499, Loss: 3.4823708534240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8254 , TFLOPS: 68.64671877661186, Tokens per sec: 170114.1311887824, Loss: 3.484466791152954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8255 , TFLOPS: 67.8471260716927, Tokens per sec: 168132.65238358488, Loss: 3.513725996017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8256 , TFLOPS: 70.50866552031108, Tokens per sec: 174728.24033003522, Loss: 3.5070884227752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8257 , TFLOPS: 67.78960799582619, Tokens per sec: 167990.11625544855, Loss: 3.47721266746521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8258 , TFLOPS: 68.83412101477165, Tokens per sec: 170578.53457900314, Loss: 3.480583667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8259 , TFLOPS: 69.67170762790971, Tokens per sec: 172654.16647414156, Loss: 3.5075364112854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8260 , TFLOPS: 69.56489404061107, Tokens per sec: 172389.47063832753, Loss: 3.4990789890289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8261 , TFLOPS: 66.96796802321703, Tokens per sec: 165954.00189220818, Loss: 3.472996234893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8262 , TFLOPS: 68.72750267551936, Tokens per sec: 170314.32258935628, Loss: 3.502333641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8263 , TFLOPS: 70.57894274530787, Tokens per sec: 174902.39503525267, Loss: 3.4939510822296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8264 , TFLOPS: 68.78235788693732, Tokens per sec: 170450.25984605038, Loss: 3.4864020347595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8265 , TFLOPS: 67.19310677079599, Tokens per sec: 166511.92050978926, Loss: 3.4974143505096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8266 , TFLOPS: 70.65288206612449, Tokens per sec: 175085.62481732486, Loss: 3.477993965148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8267 , TFLOPS: 69.61504722424866, Tokens per sec: 172513.75575221074, Loss: 3.499971389770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8268 , TFLOPS: 67.85943660041794, Tokens per sec: 168163.15922988273, Loss: 3.496711492538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8269 , TFLOPS: 68.66324308721741, Tokens per sec: 170155.08025076604, Loss: 3.4972801208496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8270 , TFLOPS: 69.50306270290694, Tokens per sec: 172236.2457722126, Loss: 3.511185646057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8271 , TFLOPS: 68.88607252544774, Tokens per sec: 170707.2761453898, Loss: 3.488342761993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8272 , TFLOPS: 67.90159069554657, Tokens per sec: 168267.6217212684, Loss: 3.4729597568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8273 , TFLOPS: 70.64324861017933, Tokens per sec: 175061.75205227998, Loss: 3.4725284576416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8274 , TFLOPS: 68.86378062785786, Tokens per sec: 170652.03436750683, Loss: 3.4901483058929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8275 , TFLOPS: 68.87266435469436, Tokens per sec: 170674.049221231, Loss: 3.5035054683685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8276 , TFLOPS: 69.105877117072, Tokens per sec: 171251.97613690936, Loss: 3.509484052658081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8277 , TFLOPS: 69.56765500127452, Tokens per sec: 172396.3126029965, Loss: 3.4969279766082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8278 , TFLOPS: 69.0635596911434, Tokens per sec: 171147.108893809, Loss: 3.4819908142089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8279 , TFLOPS: 67.98694099317981, Tokens per sec: 168479.12916091504, Loss: 3.5110132694244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8280 , TFLOPS: 69.58955280417365, Tokens per sec: 172450.57777082268, Loss: 3.514324903488159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8281 , TFLOPS: 68.88694185922688, Tokens per sec: 170709.43045025953, Loss: 3.4942569732666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8282 , TFLOPS: 68.23694923986481, Tokens per sec: 169098.67713688235, Loss: 3.501279830932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8283 , TFLOPS: 69.61668348193035, Tokens per sec: 172517.81057899466, Loss: 3.506460189819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8284 , TFLOPS: 68.63428791610671, Tokens per sec: 170083.32614707833, Loss: 3.4981234073638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8285 , TFLOPS: 66.96845964275008, Tokens per sec: 165955.22017956743, Loss: 3.5242061614990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8286 , TFLOPS: 70.63192737789633, Tokens per sec: 175033.696791546, Loss: 3.4994215965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8287 , TFLOPS: 69.57067679922723, Tokens per sec: 172403.80095120185, Loss: 3.491285800933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8288 , TFLOPS: 68.00242153316324, Tokens per sec: 168517.49164431618, Loss: 3.5020663738250732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8289 , TFLOPS: 69.62608619328113, Tokens per sec: 172541.11153351603, Loss: 3.5022084712982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8290 , TFLOPS: 69.5490519677881, Tokens per sec: 172350.21223672386, Loss: 3.5014758110046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8291 , TFLOPS: 68.7122804162542, Tokens per sec: 170276.60015402694, Loss: 3.480013608932495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8292 , TFLOPS: 68.7845360678221, Tokens per sec: 170455.6576182293, Loss: 3.5042824745178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8293 , TFLOPS: 70.58830442077276, Tokens per sec: 174925.5942983284, Loss: 3.509169340133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8294 , TFLOPS: 67.69600582842503, Tokens per sec: 167758.15977349854, Loss: 3.4690942764282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8295 , TFLOPS: 68.85637291727392, Tokens per sec: 170633.6772446531, Loss: 3.5030834674835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8296 , TFLOPS: 69.58926591213994, Tokens per sec: 172449.86682075937, Loss: 3.5066845417022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8297 , TFLOPS: 69.56736318910028, Tokens per sec: 172395.58946028296, Loss: 3.5259101390838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8298 , TFLOPS: 63.60226672412987, Tokens per sec: 157613.4233104648, Loss: 3.4940364360809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8299 , TFLOPS: 68.79832652286132, Tokens per sec: 170489.8318849605, Loss: 3.5080745220184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8300 , TFLOPS: 69.59275099789916, Tokens per sec: 172458.50324718462, Loss: 3.4940197467803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8301 , TFLOPS: 68.8144208453927, Tokens per sec: 170529.71538912927, Loss: 3.5234315395355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8302 , TFLOPS: 67.67839891598905, Tokens per sec: 167714.52790492066, Loss: 3.506599187850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8303 , TFLOPS: 70.59347559710206, Tokens per sec: 174938.40904292447, Loss: 3.5140490531921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8304 , TFLOPS: 68.68671725341383, Tokens per sec: 170213.25182049364, Loss: 3.5152852535247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8305 , TFLOPS: 67.10850293955437, Tokens per sec: 166302.26289609115, Loss: 3.4762825965881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8306 , TFLOPS: 69.56199979534519, Tokens per sec: 172382.29837973125, Loss: 3.49507999420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8307 , TFLOPS: 69.6246867379306, Tokens per sec: 172537.6435289947, Loss: 3.5090479850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8308 , TFLOPS: 68.03836046779801, Tokens per sec: 168606.55228334165, Loss: 3.50502347946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8309 , TFLOPS: 69.51787727869997, Tokens per sec: 172272.95792298796, Loss: 3.5231120586395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8310 , TFLOPS: 70.56717147582714, Tokens per sec: 174873.22453276702, Loss: 3.492086410522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8311 , TFLOPS: 68.20100815959769, Tokens per sec: 169009.6111807436, Loss: 3.4956836700439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8312 , TFLOPS: 68.74717079648093, Tokens per sec: 170363.06235972143, Loss: 3.498075485229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8313 , TFLOPS: 69.66271187674114, Tokens per sec: 172631.87401178206, Loss: 3.499502182006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8314 , TFLOPS: 69.71945192945269, Tokens per sec: 172772.48211283408, Loss: 3.496807098388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8315 , TFLOPS: 68.22036344710807, Tokens per sec: 169057.57571535525, Loss: 3.519989490509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8316 , TFLOPS: 68.50197307243288, Tokens per sec: 169755.43538877126, Loss: 3.4814865589141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8317 , TFLOPS: 69.55097942818846, Tokens per sec: 172354.9886959232, Loss: 3.4765777587890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8318 , TFLOPS: 68.92891591243033, Tokens per sec: 170813.44677792134, Loss: 3.4777278900146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8319 , TFLOPS: 68.63781252988527, Tokens per sec: 170092.06052828982, Loss: 3.4936256408691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8320 , TFLOPS: 68.79198031474199, Tokens per sec: 170474.10528214826, Loss: 3.480210781097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8321 , TFLOPS: 68.7935533130651, Tokens per sec: 170478.003345273, Loss: 3.4749202728271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8322 , TFLOPS: 67.96812356500875, Tokens per sec: 168432.49750099628, Loss: 3.5095109939575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8323 , TFLOPS: 70.5917707281891, Tokens per sec: 174934.18418995134, Loss: 3.510882616043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8324 , TFLOPS: 67.41105899417212, Tokens per sec: 167052.03012873477, Loss: 3.493170738220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8325 , TFLOPS: 67.8867705487459, Tokens per sec: 168230.8957649224, Loss: 3.5003304481506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8326 , TFLOPS: 69.90319029045817, Tokens per sec: 173227.80601185703, Loss: 3.517698049545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8327 , TFLOPS: 69.63255890223012, Tokens per sec: 172557.15161931937, Loss: 3.49368953704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8328 , TFLOPS: 67.86797969944848, Tokens per sec: 168184.32997038044, Loss: 3.511111259460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8329 , TFLOPS: 69.07689110317652, Tokens per sec: 171180.14560140338, Loss: 3.499040126800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8330 , TFLOPS: 70.56702736174336, Tokens per sec: 174872.86740219194, Loss: 3.496720314025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8331 , TFLOPS: 67.95242568859959, Tokens per sec: 168393.59643399052, Loss: 3.492316722869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8332 , TFLOPS: 68.0157975925855, Tokens per sec: 168550.63899305914, Loss: 3.482766628265381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8333 , TFLOPS: 70.57783260191383, Tokens per sec: 174899.64397762984, Loss: 3.476138114929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8334 , TFLOPS: 69.68373881568799, Tokens per sec: 172683.9810828009, Loss: 3.481419801712036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8335 , TFLOPS: 67.8583145038312, Tokens per sec: 168160.3785509316, Loss: 3.494722843170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8336 , TFLOPS: 69.6847361236514, Tokens per sec: 172686.45252179677, Loss: 3.504189968109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8337 , TFLOPS: 69.53103452860009, Tokens per sec: 172305.56303763116, Loss: 3.49680495262146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8338 , TFLOPS: 68.9431025359228, Tokens per sec: 170848.6027937213, Loss: 3.502908229827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8339 , TFLOPS: 67.8247187829466, Tokens per sec: 168077.12465370545, Loss: 3.468529224395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8340 , TFLOPS: 69.705729511298, Tokens per sec: 172738.47644899762, Loss: 3.4782915115356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8341 , TFLOPS: 68.78551051075195, Tokens per sec: 170458.07239515323, Loss: 3.4939019680023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8342 , TFLOPS: 68.93056438174419, Tokens per sec: 170817.53186647498, Loss: 3.5129318237304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8343 , TFLOPS: 69.58195410789703, Tokens per sec: 172431.74736438374, Loss: 3.4927990436553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8344 , TFLOPS: 68.86692639629484, Tokens per sec: 170659.82992822866, Loss: 3.502922534942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8345 , TFLOPS: 68.74826359852139, Tokens per sec: 170365.77044355977, Loss: 3.5134332180023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8346 , TFLOPS: 68.10500893174205, Tokens per sec: 168771.71451892937, Loss: 3.492128849029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8347 , TFLOPS: 69.54816220642195, Tokens per sec: 172348.0073100432, Loss: 3.498058795928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8348 , TFLOPS: 69.67530223447419, Tokens per sec: 172663.07430518695, Loss: 3.496983051300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8349 , TFLOPS: 68.26862461053763, Tokens per sec: 169177.17219473972, Loss: 3.4855966567993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8350 , TFLOPS: 69.57094595782299, Tokens per sec: 172404.4679558524, Loss: 3.495098114013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8351 , TFLOPS: 68.3276646865888, Tokens per sec: 169323.48000699584, Loss: 3.4787821769714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8352 , TFLOPS: 67.86130876587853, Tokens per sec: 168167.7986621301, Loss: 3.5104329586029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8353 , TFLOPS: 69.67692554454993, Tokens per sec: 172667.09704637702, Loss: 3.5181963443756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8354 , TFLOPS: 69.63359630687953, Tokens per sec: 172559.7224223202, Loss: 3.487628698348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8355 , TFLOPS: 68.11252678383593, Tokens per sec: 168790.34458457778, Loss: 3.5078392028808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8356 , TFLOPS: 69.56956113253631, Tokens per sec: 172401.03620624088, Loss: 3.5104150772094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8357 , TFLOPS: 68.54137861542752, Tokens per sec: 169853.08666519087, Loss: 3.514777898788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8358 , TFLOPS: 68.71338514765478, Tokens per sec: 170279.3378001338, Loss: 3.498609781265259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8359 , TFLOPS: 68.7547711307423, Tokens per sec: 170381.89682526703, Loss: 3.4794743061065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8360 , TFLOPS: 70.56459010374758, Tokens per sec: 174866.82760839225, Loss: 3.526047945022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8361 , TFLOPS: 68.43285029273355, Tokens per sec: 169584.14152617354, Loss: 3.489471912384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8362 , TFLOPS: 68.31927914065092, Tokens per sec: 169302.6997004176, Loss: 3.489752769470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8363 , TFLOPS: 69.58112651128287, Tokens per sec: 172429.6964888065, Loss: 3.4857935905456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8364 , TFLOPS: 69.5672098477278, Tokens per sec: 172395.20946346998, Loss: 3.4884190559387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8365 , TFLOPS: 67.11265430755691, Tokens per sec: 166312.55044331236, Loss: 3.5049190521240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8366 , TFLOPS: 68.73818849874007, Tokens per sec: 170340.80323643782, Loss: 3.5141263008117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8367 , TFLOPS: 69.67070277166431, Tokens per sec: 172651.67632967143, Loss: 3.495344877243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8368 , TFLOPS: 68.90333446368041, Tokens per sec: 170750.05312988983, Loss: 3.4846248626708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8369 , TFLOPS: 67.93932471860535, Tokens per sec: 168361.13078715382, Loss: 3.4811818599700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8370 , TFLOPS: 70.60089946881403, Tokens per sec: 174956.80621483372, Loss: 3.4942307472229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8371 , TFLOPS: 68.71206895914773, Tokens per sec: 170276.0761400232, Loss: 3.4851653575897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8372 , TFLOPS: 67.00156857024353, Tokens per sec: 166037.26774914569, Loss: 3.4946980476379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8373 , TFLOPS: 70.55681301575514, Tokens per sec: 174847.55512763007, Loss: 3.4897093772888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8374 , TFLOPS: 69.6278108860304, Tokens per sec: 172545.38551213933, Loss: 3.5068130493164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8375 , TFLOPS: 67.96412501353973, Tokens per sec: 168422.58864997924, Loss: 3.4966092109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8376 , TFLOPS: 68.67227262155048, Tokens per sec: 170177.4564315287, Loss: 3.499448299407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8377 , TFLOPS: 70.61333595756854, Tokens per sec: 174987.62520395973, Loss: 3.5111818313598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8378 , TFLOPS: 68.29477779192659, Tokens per sec: 169241.98265923234, Loss: 3.494840383529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8379 , TFLOPS: 68.8016760598743, Tokens per sec: 170498.13240666065, Loss: 3.4916489124298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8380 , TFLOPS: 69.6866962446307, Tokens per sec: 172691.3099175086, Loss: 3.491482734680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8381 , TFLOPS: 69.67183087147274, Tokens per sec: 172654.47188526715, Loss: 3.5160295963287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8382 , TFLOPS: 67.40350769128798, Tokens per sec: 167033.31716834317, Loss: 3.523203134536743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8383 , TFLOPS: 68.58307383016057, Tokens per sec: 169956.41200040648, Loss: 3.4938831329345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8384 , TFLOPS: 69.54358251875514, Tokens per sec: 172336.6583395106, Loss: 3.509662389755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8385 , TFLOPS: 68.10975617413013, Tokens per sec: 168783.47870851893, Loss: 3.488921880722046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8386 , TFLOPS: 68.73356202936726, Tokens per sec: 170329.33833568433, Loss: 3.505099058151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8387 , TFLOPS: 68.81302175489562, Tokens per sec: 170526.24828875496, Loss: 3.494370937347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8388 , TFLOPS: 69.39330267452615, Tokens per sec: 171964.24833081974, Loss: 3.505990982055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8389 , TFLOPS: 67.89761904643215, Tokens per sec: 168257.77953724956, Loss: 3.468635082244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8390 , TFLOPS: 69.62533018366119, Tokens per sec: 172539.2380583965, Loss: 3.497952938079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8391 , TFLOPS: 68.8019987947386, Tokens per sec: 170498.93217920652, Loss: 3.490375280380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8392 , TFLOPS: 68.78422198631353, Tokens per sec: 170454.87928965213, Loss: 3.4886298179626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8393 , TFLOPS: 69.85228970645325, Tokens per sec: 173101.6687575306, Loss: 3.511197328567505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8394 , TFLOPS: 68.65492937815097, Tokens per sec: 170134.47796386873, Loss: 3.4975948333740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8395 , TFLOPS: 68.81762788820708, Tokens per sec: 170537.66279450827, Loss: 3.4783191680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8396 , TFLOPS: 69.09990351109995, Tokens per sec: 171237.17288326359, Loss: 3.5220537185668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8397 , TFLOPS: 70.61625857907445, Tokens per sec: 174994.86778200473, Loss: 3.5025408267974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8398 , TFLOPS: 67.77180058299, Tokens per sec: 167945.987524792, Loss: 3.474456548690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8399 , TFLOPS: 68.11040365155233, Tokens per sec: 168785.0832288967, Loss: 3.5029919147491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8400 , TFLOPS: 70.61642345572301, Tokens per sec: 174995.2763645025, Loss: 3.5062172412872314 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/model.pt +[rank0]:[2024-08-25 14:25:33,235] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006289223994826898, 'preprocessing_with_comm': 0.0013479759945767, 'state_converting': 0.996921988989925, : 1.0059490080020623}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006400 + eval ppl=24.4470157623291, eval loss=3.1965081691741943 +------------------------------------------------------------------ +iteration: 8401 , TFLOPS: 64.34350940641987, Tokens per sec: 159450.30433180166, Loss: 3.5123562812805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8402 , TFLOPS: 65.86507066935211, Tokens per sec: 163220.90075515775, Loss: 3.5151963233947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8403 , TFLOPS: 69.62617734656615, Tokens per sec: 172541.3374213977, Loss: 3.5057144165039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8404 , TFLOPS: 69.57052149591819, Tokens per sec: 172403.41609249395, Loss: 3.481248617172241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8405 , TFLOPS: 67.69015749027793, Tokens per sec: 167743.66694732237, Loss: 3.5089311599731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8406 , TFLOPS: 68.66693594358155, Tokens per sec: 170164.23155563808, Loss: 3.5052075386047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8407 , TFLOPS: 69.63777915435995, Tokens per sec: 172570.08797915536, Loss: 3.5013537406921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8408 , TFLOPS: 69.77398244530345, Tokens per sec: 172907.61473814445, Loss: 3.4793107509613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8409 , TFLOPS: 67.1883756247472, Tokens per sec: 166500.1962087016, Loss: 3.487835168838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8410 , TFLOPS: 70.5832918359536, Tokens per sec: 174913.17256096468, Loss: 3.473796844482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8411 , TFLOPS: 68.73897319263513, Tokens per sec: 170342.7477943509, Loss: 3.5003390312194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8412 , TFLOPS: 68.69535886545658, Tokens per sec: 170234.66668708774, Loss: 3.4903318881988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8413 , TFLOPS: 68.37039940257982, Tokens per sec: 169429.38134083914, Loss: 3.512706756591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8414 , TFLOPS: 69.6125972516526, Tokens per sec: 172507.68445023097, Loss: 3.484928607940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8415 , TFLOPS: 68.96852017424546, Tokens per sec: 170911.59050146883, Loss: 3.4897074699401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8416 , TFLOPS: 66.85888107830023, Tokens per sec: 165683.6724257853, Loss: 3.5088179111480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8417 , TFLOPS: 69.53014892630483, Tokens per sec: 172303.36841758637, Loss: 3.5064830780029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8418 , TFLOPS: 69.70267746993397, Tokens per sec: 172730.91315428744, Loss: 3.4734232425689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8419 , TFLOPS: 67.8960570441151, Tokens per sec: 168253.90872343915, Loss: 3.486804962158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8420 , TFLOPS: 69.55454382387305, Tokens per sec: 172363.82166108023, Loss: 3.485316514968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8421 , TFLOPS: 69.91225783375224, Tokens per sec: 173250.27638301204, Loss: 3.4704229831695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8422 , TFLOPS: 67.3255245842108, Tokens per sec: 166840.0664384575, Loss: 3.486079692840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8423 , TFLOPS: 69.63465155634368, Tokens per sec: 172562.33744673769, Loss: 3.507964849472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8424 , TFLOPS: 68.65125696099817, Tokens per sec: 170125.37730961395, Loss: 3.4835448265075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8425 , TFLOPS: 68.87892047988132, Tokens per sec: 170689.55258861333, Loss: 3.512683629989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8426 , TFLOPS: 68.77274475253519, Tokens per sec: 170426.43744002772, Loss: 3.496922254562378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8427 , TFLOPS: 69.63233033200241, Tokens per sec: 172556.58519711607, Loss: 3.4896585941314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8428 , TFLOPS: 68.9365583457118, Tokens per sec: 170832.38556947635, Loss: 3.50791597366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8429 , TFLOPS: 68.09201784035378, Tokens per sec: 168739.52116338236, Loss: 3.4947896003723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8430 , TFLOPS: 69.88244123061496, Tokens per sec: 173176.38755586787, Loss: 3.502859115600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8431 , TFLOPS: 68.62006613829962, Tokens per sec: 170048.08301501418, Loss: 3.492361307144165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8432 , TFLOPS: 68.31649656926096, Tokens per sec: 169295.80418198774, Loss: 3.4957096576690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8433 , TFLOPS: 69.58708149482479, Tokens per sec: 172444.453594019, Loss: 3.461421251296997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8434 , TFLOPS: 69.58825086508168, Tokens per sec: 172447.35142233234, Loss: 3.5017731189727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8435 , TFLOPS: 68.15100279535628, Tokens per sec: 168885.69237960773, Loss: 3.5180139541625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8436 , TFLOPS: 67.7847985281681, Tokens per sec: 167978.19786478518, Loss: 3.489306926727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8437 , TFLOPS: 70.59222320779755, Tokens per sec: 174935.3054842658, Loss: 3.4990687370300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8438 , TFLOPS: 69.18539389143798, Tokens per sec: 171449.02746328406, Loss: 3.466606616973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8439 , TFLOPS: 66.865817659084, Tokens per sec: 165700.86203709498, Loss: 3.505892038345337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8440 , TFLOPS: 70.64394572743025, Tokens per sec: 175063.47958562154, Loss: 3.4996020793914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8441 , TFLOPS: 69.62823399599031, Tokens per sec: 172546.4340252294, Loss: 3.495880126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8442 , TFLOPS: 67.87687652675736, Tokens per sec: 168206.37728851827, Loss: 3.5071043968200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8443 , TFLOPS: 68.59692962344457, Tokens per sec: 169990.74818250586, Loss: 3.472087860107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8444 , TFLOPS: 69.58586089155742, Tokens per sec: 172441.4288046619, Loss: 3.499302387237549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8445 , TFLOPS: 69.79222229247674, Tokens per sec: 172952.81508872492, Loss: 3.4719796180725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8446 , TFLOPS: 67.38870732605503, Tokens per sec: 166996.6402329011, Loss: 3.503692626953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8447 , TFLOPS: 69.6265220557025, Tokens per sec: 172542.19164861037, Loss: 3.498993396759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8448 , TFLOPS: 69.74670324865862, Tokens per sec: 172840.01388380703, Loss: 3.510237455368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8449 , TFLOPS: 67.95479790134712, Tokens per sec: 168399.47503849972, Loss: 3.487804651260376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8450 , TFLOPS: 67.8768870477723, Tokens per sec: 168206.4033607523, Loss: 3.4827334880828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8451 , TFLOPS: 69.61594679902953, Tokens per sec: 172515.9849976141, Loss: 3.493906259536743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8452 , TFLOPS: 68.14128781947868, Tokens per sec: 168861.61759919013, Loss: 3.4727652072906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8453 , TFLOPS: 68.61075637517048, Tokens per sec: 170025.0123964251, Loss: 3.4906387329101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8454 , TFLOPS: 69.5434204488155, Tokens per sec: 172336.2567123467, Loss: 3.506547451019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8455 , TFLOPS: 68.86985825810328, Tokens per sec: 170667.09540476123, Loss: 3.502614736557007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8456 , TFLOPS: 67.78930862866665, Tokens per sec: 167989.37439064917, Loss: 3.5073511600494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8457 , TFLOPS: 69.65725481723193, Tokens per sec: 172618.35081717352, Loss: 3.4890761375427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8458 , TFLOPS: 68.81486533464908, Tokens per sec: 170530.81688247112, Loss: 3.5167579650878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8459 , TFLOPS: 68.75879549005823, Tokens per sec: 170391.8696309717, Loss: 3.491466999053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8460 , TFLOPS: 70.07220491641986, Tokens per sec: 173646.64287348813, Loss: 3.4860191345214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8461 , TFLOPS: 69.50371776251063, Tokens per sec: 172237.86908207132, Loss: 3.4938507080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8462 , TFLOPS: 68.93707725591706, Tokens per sec: 170833.67148612769, Loss: 3.4814822673797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8463 , TFLOPS: 68.16036013003898, Tokens per sec: 168908.880885747, Loss: 3.4898948669433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8464 , TFLOPS: 69.61752958293778, Tokens per sec: 172519.90731049675, Loss: 3.4946212768554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8465 , TFLOPS: 68.88009527710707, Tokens per sec: 170692.46386555338, Loss: 3.498234272003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8466 , TFLOPS: 67.95067362320002, Tokens per sec: 168389.25462292446, Loss: 3.482719659805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8467 , TFLOPS: 69.5912066888928, Tokens per sec: 172454.67627934716, Loss: 3.484010934829712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8468 , TFLOPS: 69.56140251345296, Tokens per sec: 172380.8182494063, Loss: 3.5096371173858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8469 , TFLOPS: 67.54685717438649, Tokens per sec: 167388.55297871045, Loss: 3.5165719985961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8470 , TFLOPS: 69.6878749735203, Tokens per sec: 172694.23093754475, Loss: 3.504734754562378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8471 , TFLOPS: 69.64634600207711, Tokens per sec: 172591.31757151466, Loss: 3.498555898666382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8472 , TFLOPS: 67.16658667344299, Tokens per sec: 166446.20078711864, Loss: 3.5045900344848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8473 , TFLOPS: 69.53261753573648, Tokens per sec: 172309.48590369776, Loss: 3.503293514251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8474 , TFLOPS: 69.55710124577615, Tokens per sec: 172370.1592342794, Loss: 3.5078396797180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8475 , TFLOPS: 68.94285487614417, Tokens per sec: 170847.98906550786, Loss: 3.4884722232818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8476 , TFLOPS: 66.92259948555727, Tokens per sec: 165841.57365813028, Loss: 3.5052480697631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8477 , TFLOPS: 70.62742567889127, Tokens per sec: 175022.5410855084, Loss: 3.498966932296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8478 , TFLOPS: 68.77243675521423, Tokens per sec: 170425.67418873767, Loss: 3.4914376735687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8479 , TFLOPS: 68.81105675065855, Tokens per sec: 170521.3787917924, Loss: 3.5149803161621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8480 , TFLOPS: 69.00670658935144, Tokens per sec: 171006.22064468235, Loss: 3.5148048400878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8481 , TFLOPS: 69.5848536811201, Tokens per sec: 172438.93282624485, Loss: 3.4882607460021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8482 , TFLOPS: 68.92392681379776, Tokens per sec: 170801.08324191524, Loss: 3.4852564334869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8483 , TFLOPS: 66.94805254438856, Tokens per sec: 165904.64914180012, Loss: 3.4945437908172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8484 , TFLOPS: 69.63923350039961, Tokens per sec: 172573.6920088511, Loss: 3.4815378189086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8485 , TFLOPS: 69.7233045960952, Tokens per sec: 172782.0294451805, Loss: 3.484571695327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8486 , TFLOPS: 67.41943904695026, Tokens per sec: 167072.796822658, Loss: 3.4844377040863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8487 , TFLOPS: 69.27093789693525, Tokens per sec: 171661.01493235645, Loss: 3.49691104888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8488 , TFLOPS: 69.84630855201105, Tokens per sec: 173086.84679794428, Loss: 3.5075771808624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8489 , TFLOPS: 68.0090722526738, Tokens per sec: 168533.97285990062, Loss: 3.4791512489318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8490 , TFLOPS: 68.14292898372842, Tokens per sec: 168865.68458499014, Loss: 3.514228105545044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8491 , TFLOPS: 69.64387315848678, Tokens per sec: 172585.18959269163, Loss: 3.5082693099975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8492 , TFLOPS: 68.50614124204021, Tokens per sec: 169765.76457222004, Loss: 3.495044231414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8493 , TFLOPS: 68.51845098700389, Tokens per sec: 169796.26947627036, Loss: 3.485447406768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8494 , TFLOPS: 68.63917463141503, Tokens per sec: 170095.43596592848, Loss: 3.4788014888763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8495 , TFLOPS: 69.18090921307434, Tokens per sec: 171437.9139362709, Loss: 3.4946632385253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8496 , TFLOPS: 68.32436914035749, Tokens per sec: 169315.31328040024, Loss: 3.4987382888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8497 , TFLOPS: 69.81985933153285, Tokens per sec: 173021.30271597716, Loss: 3.5184481143951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8498 , TFLOPS: 67.93422353145756, Tokens per sec: 168348.48948345947, Loss: 3.4940176010131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8499 , TFLOPS: 69.7249577280336, Tokens per sec: 172786.12608823137, Loss: 3.4944260120391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8500 , TFLOPS: 68.7811113082809, Tokens per sec: 170447.1706868178, Loss: 3.478050947189331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8501 , TFLOPS: 69.67566537504766, Tokens per sec: 172663.97420753122, Loss: 3.487346649169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8502 , TFLOPS: 67.86892267360744, Tokens per sec: 168186.66676422354, Loss: 3.511009693145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8503 , TFLOPS: 68.01789932006778, Tokens per sec: 168555.8473052846, Loss: 3.4943253993988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8504 , TFLOPS: 70.63757804514627, Tokens per sec: 175047.69976746375, Loss: 3.484905481338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8505 , TFLOPS: 68.76204188733944, Tokens per sec: 170399.9145610547, Loss: 3.491713523864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8506 , TFLOPS: 68.05302346270008, Tokens per sec: 168642.8887999712, Loss: 3.496260643005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8507 , TFLOPS: 70.57577675836899, Tokens per sec: 174894.54937085585, Loss: 3.484113931655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8508 , TFLOPS: 69.59396216412446, Tokens per sec: 172461.5046505119, Loss: 3.501880645751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8509 , TFLOPS: 67.90611163081667, Tokens per sec: 168278.8250968898, Loss: 3.509194850921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8510 , TFLOPS: 68.82362546073372, Tokens per sec: 170552.5254399164, Loss: 3.5134103298187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8511 , TFLOPS: 69.60447275314503, Tokens per sec: 172487.55104794182, Loss: 3.5204529762268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8512 , TFLOPS: 69.8630195341708, Tokens per sec: 173128.2584526028, Loss: 3.506049156188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8513 , TFLOPS: 67.86089765494089, Tokens per sec: 168166.77988393902, Loss: 3.4575271606445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8514 , TFLOPS: 69.55515957464239, Tokens per sec: 172365.34755931722, Loss: 3.4709932804107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8515 , TFLOPS: 69.71752106005327, Tokens per sec: 172767.6972057595, Loss: 3.47721529006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8516 , TFLOPS: 68.00513160883926, Tokens per sec: 168524.20751038086, Loss: 3.4885261058807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8517 , TFLOPS: 68.48898227973741, Tokens per sec: 169723.24277341797, Loss: 3.4893665313720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8518 , TFLOPS: 69.60627108656553, Tokens per sec: 172492.00751626058, Loss: 3.5209426879882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8519 , TFLOPS: 68.76983584764655, Tokens per sec: 170419.22885327134, Loss: 3.4831652641296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8520 , TFLOPS: 67.96612314857553, Tokens per sec: 168427.5402487107, Loss: 3.5082459449768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8521 , TFLOPS: 69.61405706347587, Tokens per sec: 172511.30202474055, Loss: 3.484377384185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8522 , TFLOPS: 68.78102188068581, Tokens per sec: 170446.9490753859, Loss: 3.4803459644317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8523 , TFLOPS: 67.90945465796187, Tokens per sec: 168287.10948642046, Loss: 3.49630069732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8524 , TFLOPS: 69.59823436863321, Tokens per sec: 172472.09164390675, Loss: 3.4985649585723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8525 , TFLOPS: 68.87615886327379, Tokens per sec: 170682.70899844271, Loss: 3.4942634105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8526 , TFLOPS: 68.72483373486052, Tokens per sec: 170307.7086603956, Loss: 3.4945950508117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8527 , TFLOPS: 69.14987341098599, Tokens per sec: 171361.0037998201, Loss: 3.5144386291503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8528 , TFLOPS: 69.60696853348026, Tokens per sec: 172493.7358665454, Loss: 3.4934277534484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8529 , TFLOPS: 68.75244228727756, Tokens per sec: 170376.12569461812, Loss: 3.4896957874298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8530 , TFLOPS: 68.16012539763196, Tokens per sec: 168908.2991929845, Loss: 3.543473243713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8531 , TFLOPS: 69.56876940933223, Tokens per sec: 172399.0742289257, Loss: 3.487973690032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8532 , TFLOPS: 69.79596537675938, Tokens per sec: 172962.0908639121, Loss: 3.507819890975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8533 , TFLOPS: 67.75636194261581, Tokens per sec: 167907.72887323814, Loss: 3.511086940765381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8534 , TFLOPS: 70.59928322508793, Tokens per sec: 174952.80098483647, Loss: 3.504786968231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8535 , TFLOPS: 69.01934480203488, Tokens per sec: 171037.53952792598, Loss: 3.511878490447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8536 , TFLOPS: 67.43492285552865, Tokens per sec: 167111.16740599758, Loss: 3.4753148555755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8537 , TFLOPS: 69.71207642159978, Tokens per sec: 172754.2047918953, Loss: 3.478970527648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8538 , TFLOPS: 69.57427349547329, Tokens per sec: 172412.71396070838, Loss: 3.5050690174102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8539 , TFLOPS: 68.66180501722229, Tokens per sec: 170151.51655490167, Loss: 3.507702350616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8540 , TFLOPS: 68.71960024654315, Tokens per sec: 170294.73949982884, Loss: 3.470583200454712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8541 , TFLOPS: 69.65012053393315, Tokens per sec: 172600.67127725296, Loss: 3.498094320297241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8542 , TFLOPS: 67.88233945075052, Tokens per sec: 168219.91501596407, Loss: 3.4954843521118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8543 , TFLOPS: 68.02314474619153, Tokens per sec: 168568.84604905068, Loss: 3.4840524196624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8544 , TFLOPS: 70.60757338252661, Tokens per sec: 174973.34490820576, Loss: 3.509216070175171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8545 , TFLOPS: 68.82205154332685, Tokens per sec: 170548.62509920084, Loss: 3.4933431148529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8546 , TFLOPS: 68.36241092146344, Tokens per sec: 169409.5850046289, Loss: 3.505509376525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8547 , TFLOPS: 69.01595923860769, Tokens per sec: 171029.1497288028, Loss: 3.4999308586120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8548 , TFLOPS: 69.71980836920812, Tokens per sec: 172773.36540981312, Loss: 3.490341901779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8549 , TFLOPS: 68.93083726153101, Tokens per sec: 170818.2080926469, Loss: 3.5179882049560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8550 , TFLOPS: 67.01108762715982, Tokens per sec: 166060.8570207949, Loss: 3.5277936458587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8551 , TFLOPS: 69.55559354434517, Tokens per sec: 172366.42298404642, Loss: 3.4929068088531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8552 , TFLOPS: 69.63631516930907, Tokens per sec: 172566.46006292803, Loss: 3.4904630184173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8553 , TFLOPS: 66.96196540331161, Tokens per sec: 165939.12673883938, Loss: 3.510791063308716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8554 , TFLOPS: 70.60148628364915, Tokens per sec: 174958.26040663812, Loss: 3.5107617378234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8555 , TFLOPS: 69.63814968037727, Tokens per sec: 172571.0061834431, Loss: 3.5087456703186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8556 , TFLOPS: 67.31903024524246, Tokens per sec: 166823.97275108346, Loss: 3.5046305656433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8557 , TFLOPS: 68.53936421547179, Tokens per sec: 169848.09476019614, Loss: 3.492044448852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8558 , TFLOPS: 69.4734419928498, Tokens per sec: 172162.84238970038, Loss: 3.4925689697265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8559 , TFLOPS: 68.82697478638231, Tokens per sec: 170560.82543783227, Loss: 3.4981765747070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8560 , TFLOPS: 68.92120362140398, Tokens per sec: 170794.33487117907, Loss: 3.4980363845825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8561 , TFLOPS: 68.84584450747418, Tokens per sec: 170607.58668536932, Loss: 3.502310276031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8562 , TFLOPS: 69.89070258004666, Tokens per sec: 173196.86008982244, Loss: 3.5060532093048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8563 , TFLOPS: 68.30679542996026, Tokens per sec: 169271.76369013186, Loss: 3.508218288421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8564 , TFLOPS: 68.96702798920698, Tokens per sec: 170907.89270256573, Loss: 3.500946283340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8565 , TFLOPS: 68.68167121221435, Tokens per sec: 170200.74717453404, Loss: 3.499882936477661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8566 , TFLOPS: 69.65613521726287, Tokens per sec: 172615.57632511604, Loss: 3.4717369079589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8567 , TFLOPS: 68.86843021665196, Tokens per sec: 170663.5565607331, Loss: 3.4881229400634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8568 , TFLOPS: 68.836825363639, Tokens per sec: 170585.236253405, Loss: 3.4872920513153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8569 , TFLOPS: 68.95209078641147, Tokens per sec: 170870.87666856975, Loss: 3.4901442527770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8570 , TFLOPS: 67.9694991551468, Tokens per sec: 168435.90636489214, Loss: 3.503969192504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8571 , TFLOPS: 70.60322775788083, Tokens per sec: 174962.57597162345, Loss: 3.4914162158966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8572 , TFLOPS: 67.89112275781105, Tokens per sec: 168241.68101842122, Loss: 3.497293472290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8573 , TFLOPS: 67.88395703270338, Tokens per sec: 168223.92356223465, Loss: 3.4800689220428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8574 , TFLOPS: 70.62449772404207, Tokens per sec: 175015.28529085123, Loss: 3.486406087875366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8575 , TFLOPS: 69.62750112502438, Tokens per sec: 172544.61789024342, Loss: 3.50540828704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8576 , TFLOPS: 67.53687518056361, Tokens per sec: 167363.81649840996, Loss: 3.498643398284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8577 , TFLOPS: 69.06958090802965, Tokens per sec: 171162.03013254493, Loss: 3.5020971298217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8578 , TFLOPS: 69.58763013818164, Tokens per sec: 172445.8131926953, Loss: 3.5063133239746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8579 , TFLOPS: 68.84065976027317, Tokens per sec: 170594.73831065817, Loss: 3.5020341873168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8580 , TFLOPS: 68.3304071774209, Tokens per sec: 169330.27620138804, Loss: 3.5145647525787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8581 , TFLOPS: 69.60446073804766, Tokens per sec: 172487.52127320698, Loss: 3.489560604095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8582 , TFLOPS: 69.78238040713698, Tokens per sec: 172928.42581267984, Loss: 3.502169132232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8583 , TFLOPS: 67.00974295716757, Tokens per sec: 166057.52478042652, Loss: 3.5196778774261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8584 , TFLOPS: 69.69126572078734, Tokens per sec: 172702.63358279414, Loss: 3.4900434017181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8585 , TFLOPS: 69.52194218969274, Tokens per sec: 172283.03122021497, Loss: 3.47082257270813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8586 , TFLOPS: 69.15065565534336, Tokens per sec: 171362.942287509, Loss: 3.48134446144104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8587 , TFLOPS: 68.00765377481706, Tokens per sec: 168530.45771551336, Loss: 3.500408411026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8588 , TFLOPS: 69.59657931739062, Tokens per sec: 172467.99024460863, Loss: 3.5150411128997803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8589 , TFLOPS: 68.81244121865909, Tokens per sec: 170524.8096560095, Loss: 3.5019679069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8590 , TFLOPS: 67.85320297738215, Tokens per sec: 168147.71162530978, Loss: 3.5088837146759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8591 , TFLOPS: 69.50110546276854, Tokens per sec: 172231.39551554178, Loss: 3.4985601902008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8592 , TFLOPS: 68.83660631051566, Tokens per sec: 170584.6934156347, Loss: 3.494072675704956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8593 , TFLOPS: 68.5987045349745, Tokens per sec: 169995.14660879967, Loss: 3.522430658340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8594 , TFLOPS: 68.93287991739918, Tokens per sec: 170823.27001890534, Loss: 3.5094497203826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8595 , TFLOPS: 69.64424960359666, Tokens per sec: 172586.12246514275, Loss: 3.507540464401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8596 , TFLOPS: 68.85751865517065, Tokens per sec: 170636.51650937472, Loss: 3.470548152923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8597 , TFLOPS: 68.1381598140943, Tokens per sec: 168853.86605726936, Loss: 3.495143413543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8598 , TFLOPS: 68.54801407729106, Tokens per sec: 169869.5300706452, Loss: 3.4932916164398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8599 , TFLOPS: 69.82682232479459, Tokens per sec: 173038.55778031677, Loss: 3.4939537048339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8600 , TFLOPS: 68.02888678149141, Tokens per sec: 168583.07544506117, Loss: 3.497627019882202 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/optimizer.pt +[rank0]:[2024-08-25 14:51:17,887] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063856299966573715, 'preprocessing_with_comm': 0.0013896040036343038, 'state_converting': 0.986684981995495, : 0.9958492820005631}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600, took 4.25s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006600 + eval ppl=24.03587532043457, eval loss=3.1795475482940674 +------------------------------------------------------------------ +iteration: 8601 , TFLOPS: 64.10342898811966, Tokens per sec: 158855.3586082124, Loss: 3.4764256477355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8602 , TFLOPS: 69.71987060853024, Tokens per sec: 172773.51964570957, Loss: 3.4634101390838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8603 , TFLOPS: 68.84288429893715, Tokens per sec: 170600.25096252066, Loss: 3.510985851287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8604 , TFLOPS: 69.93897550767485, Tokens per sec: 173316.48572218706, Loss: 3.503296375274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8605 , TFLOPS: 68.58561507681436, Tokens per sec: 169962.709489556, Loss: 3.50563383102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8606 , TFLOPS: 68.57275985474729, Tokens per sec: 169930.85283315417, Loss: 3.5226635932922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8607 , TFLOPS: 68.8654961834431, Tokens per sec: 170656.28570323111, Loss: 3.518782138824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8608 , TFLOPS: 69.56159342036625, Tokens per sec: 172381.29133776756, Loss: 3.466933250427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8609 , TFLOPS: 68.60231002248136, Tokens per sec: 170004.0814040189, Loss: 3.4857420921325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8610 , TFLOPS: 67.23103408356053, Tokens per sec: 166605.9085688582, Loss: 3.5044875144958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8611 , TFLOPS: 70.5610517402212, Tokens per sec: 174858.05915379102, Loss: 3.4941635131835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8612 , TFLOPS: 68.52005416452792, Tokens per sec: 169800.2423267795, Loss: 3.476778984069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8613 , TFLOPS: 67.9267476265494, Tokens per sec: 168329.9633675581, Loss: 3.4827096462249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8614 , TFLOPS: 70.55586063087777, Tokens per sec: 174845.1950129901, Loss: 3.506863594055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8615 , TFLOPS: 69.63379197886583, Tokens per sec: 172560.20731905728, Loss: 3.4815995693206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8616 , TFLOPS: 66.17382909508856, Tokens per sec: 163986.0381466811, Loss: 3.52105712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8617 , TFLOPS: 69.67782869087331, Tokens per sec: 172669.33514245614, Loss: 3.495060443878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8618 , TFLOPS: 69.5821291512262, Tokens per sec: 172432.181141036, Loss: 3.4904603958129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8619 , TFLOPS: 68.26304341338437, Tokens per sec: 169163.3413733743, Loss: 3.5000641345977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8620 , TFLOPS: 67.6981653397467, Tokens per sec: 167763.51128044343, Loss: 3.494898796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8621 , TFLOPS: 69.67390285399212, Tokens per sec: 172659.60648619788, Loss: 3.479212760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8622 , TFLOPS: 69.77329981729406, Tokens per sec: 172905.92311073953, Loss: 3.4882616996765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8623 , TFLOPS: 66.95221746018035, Tokens per sec: 165914.97026193552, Loss: 3.490525007247925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8624 , TFLOPS: 70.01209611255597, Tokens per sec: 173497.68663598204, Loss: 3.509965658187866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8625 , TFLOPS: 67.89628589483259, Tokens per sec: 168254.47584072707, Loss: 3.4946506023406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8626 , TFLOPS: 68.99077416635511, Tokens per sec: 170966.73834539615, Loss: 3.492530107498169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8627 , TFLOPS: 67.91781866710241, Tokens per sec: 168307.83642244237, Loss: 3.501952886581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8628 , TFLOPS: 69.62121023587767, Tokens per sec: 172529.02837393925, Loss: 3.487759828567505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8629 , TFLOPS: 69.77931964515818, Tokens per sec: 172920.8409073256, Loss: 3.504697799682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8630 , TFLOPS: 67.17038411692582, Tokens per sec: 166455.611270392, Loss: 3.4937424659729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8631 , TFLOPS: 69.58221488552937, Tokens per sec: 172432.39360008365, Loss: 3.4889590740203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8632 , TFLOPS: 68.65454370778502, Tokens per sec: 170133.52223021685, Loss: 3.5043516159057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8633 , TFLOPS: 67.94250785648896, Tokens per sec: 168369.018953481, Loss: 3.4847545623779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8634 , TFLOPS: 69.55639983559502, Tokens per sec: 172368.42106258383, Loss: 3.4956607818603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8635 , TFLOPS: 69.61276619786182, Tokens per sec: 172508.10311754837, Loss: 3.5120413303375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8636 , TFLOPS: 67.86956105615167, Tokens per sec: 168188.24874649176, Loss: 3.489553451538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8637 , TFLOPS: 69.09396682575502, Tokens per sec: 171222.461122999, Loss: 3.4951469898223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8638 , TFLOPS: 68.58791044479985, Tokens per sec: 169968.3976642797, Loss: 3.485172748565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8639 , TFLOPS: 68.79357842220764, Tokens per sec: 170478.06556849426, Loss: 3.514169216156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8640 , TFLOPS: 68.36962488445148, Tokens per sec: 169427.46199959694, Loss: 3.4904403686523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8641 , TFLOPS: 70.60182822398092, Tokens per sec: 174959.10777244807, Loss: 3.5011396408081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8642 , TFLOPS: 68.16197041936957, Tokens per sec: 168912.8713600962, Loss: 3.5213141441345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8643 , TFLOPS: 67.3510208143345, Tokens per sec: 166903.24890534565, Loss: 3.4960930347442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8644 , TFLOPS: 69.665658152414, Tokens per sec: 172639.17520745844, Loss: 3.5093770027160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8645 , TFLOPS: 69.53932263805726, Tokens per sec: 172326.10188587665, Loss: 3.4931297302246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8646 , TFLOPS: 67.3883300207569, Tokens per sec: 166995.7052288087, Loss: 3.5011491775512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8647 , TFLOPS: 68.8945590920562, Tokens per sec: 170728.30679231804, Loss: 3.4964544773101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8648 , TFLOPS: 69.61747399830823, Tokens per sec: 172519.7695656616, Loss: 3.490849018096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8649 , TFLOPS: 69.56575899282333, Tokens per sec: 172391.6140851919, Loss: 3.510817527770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8650 , TFLOPS: 67.05675561213467, Tokens per sec: 166174.0273183064, Loss: 3.499891757965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8651 , TFLOPS: 70.55816375839356, Tokens per sec: 174850.90241668466, Loss: 3.493314266204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8652 , TFLOPS: 68.79525125605564, Tokens per sec: 170482.21103504742, Loss: 3.486410617828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8653 , TFLOPS: 68.70184297706022, Tokens per sec: 170250.73502992542, Loss: 3.5159192085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8654 , TFLOPS: 69.80310822870187, Tokens per sec: 172979.79163787566, Loss: 3.5053446292877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8655 , TFLOPS: 69.6072446170923, Tokens per sec: 172494.4200321492, Loss: 3.492091417312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8656 , TFLOPS: 68.85899445359355, Tokens per sec: 170640.17370043925, Loss: 3.4930953979492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8657 , TFLOPS: 67.93463692628114, Tokens per sec: 168349.5139213715, Loss: 3.5129027366638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8658 , TFLOPS: 70.62214508383364, Tokens per sec: 175009.45518925003, Loss: 3.487457275390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8659 , TFLOPS: 69.14685300788356, Tokens per sec: 171353.51890820742, Loss: 3.5059947967529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8660 , TFLOPS: 67.10491926396601, Tokens per sec: 166293.38215320875, Loss: 3.4852681159973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8661 , TFLOPS: 69.56037224881794, Tokens per sec: 172378.2651401477, Loss: 3.465057134628296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8662 , TFLOPS: 69.4839430356277, Tokens per sec: 172188.8651304894, Loss: 3.490842342376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8663 , TFLOPS: 66.80276369920189, Tokens per sec: 165544.60737854004, Loss: 3.4817323684692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8664 , TFLOPS: 68.59815282490214, Tokens per sec: 169993.7794104645, Loss: 3.4663779735565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8665 , TFLOPS: 69.66060597798217, Tokens per sec: 172626.65536267342, Loss: 3.5070064067840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8666 , TFLOPS: 68.1764499445138, Tokens per sec: 168948.75321845498, Loss: 3.5377004146575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8667 , TFLOPS: 68.57108178423228, Tokens per sec: 169926.69438956323, Loss: 3.504171133041382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8668 , TFLOPS: 68.2639617508704, Tokens per sec: 169165.6171148279, Loss: 3.5060811042785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8669 , TFLOPS: 69.46207400705306, Tokens per sec: 172134.67126860906, Loss: 3.4993178844451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8670 , TFLOPS: 66.16175781848595, Tokens per sec: 163956.12419349892, Loss: 3.515871524810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8671 , TFLOPS: 69.03223007700217, Tokens per sec: 171069.47065872513, Loss: 3.466559410095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8672 , TFLOPS: 68.09172935265198, Tokens per sec: 168738.80625907777, Loss: 3.526151180267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8673 , TFLOPS: 67.91952664029705, Tokens per sec: 168312.0689681675, Loss: 3.498016119003296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8674 , TFLOPS: 68.85294848670473, Tokens per sec: 170625.19112847053, Loss: 3.502824306488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8675 , TFLOPS: 67.87863731166348, Tokens per sec: 168210.74070748212, Loss: 3.496516704559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8676 , TFLOPS: 69.24436196447549, Tokens per sec: 171595.15684414053, Loss: 3.493021249771118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8677 , TFLOPS: 66.33125051065551, Tokens per sec: 164376.14575585647, Loss: 3.513270139694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8678 , TFLOPS: 70.41578259849221, Tokens per sec: 174498.06621786958, Loss: 3.5037262439727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8679 , TFLOPS: 67.69149957469332, Tokens per sec: 167746.9927803465, Loss: 3.5017476081848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8680 , TFLOPS: 66.97597831374314, Tokens per sec: 165973.85227453822, Loss: 3.505598545074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8681 , TFLOPS: 70.57523722214775, Tokens per sec: 174893.21234066482, Loss: 3.4857499599456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8682 , TFLOPS: 69.51317265227176, Tokens per sec: 172261.29934044147, Loss: 3.4928197860717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8683 , TFLOPS: 66.90568351667442, Tokens per sec: 165799.6540237908, Loss: 3.5053157806396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8684 , TFLOPS: 69.46768061866524, Tokens per sec: 172148.5650698033, Loss: 3.5115153789520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8685 , TFLOPS: 69.01891094779283, Tokens per sec: 171036.46438932116, Loss: 3.517397403717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8686 , TFLOPS: 66.10403652061088, Tokens per sec: 163813.08445883862, Loss: 3.508513927459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8687 , TFLOPS: 68.33525015915907, Tokens per sec: 169342.27764363372, Loss: 3.4833767414093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8688 , TFLOPS: 70.16988386948337, Tokens per sec: 173888.70207940447, Loss: 3.503622531890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8689 , TFLOPS: 67.89900856437345, Tokens per sec: 168261.22291577616, Loss: 3.49284029006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8690 , TFLOPS: 65.91890681518886, Tokens per sec: 163354.31265508226, Loss: 3.482530355453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8691 , TFLOPS: 69.41025079146971, Tokens per sec: 172006.24763159506, Loss: 3.486804485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8692 , TFLOPS: 69.05543013522073, Tokens per sec: 171126.9629586881, Loss: 3.5038750171661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8693 , TFLOPS: 67.69220895218483, Tokens per sec: 167748.75069591484, Loss: 3.520486831665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8694 , TFLOPS: 68.44773084831971, Tokens per sec: 169621.01718214457, Loss: 3.495718240737915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8695 , TFLOPS: 69.21296502444149, Tokens per sec: 171517.3517680778, Loss: 3.482530117034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8696 , TFLOPS: 69.187763731521, Tokens per sec: 171454.90018806924, Loss: 3.477385997772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8697 , TFLOPS: 64.50702425492699, Tokens per sec: 159855.51213904985, Loss: 3.5034918785095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8698 , TFLOPS: 70.28916133357254, Tokens per sec: 174184.2847749148, Loss: 3.4881861209869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8699 , TFLOPS: 67.87102980308491, Tokens per sec: 168191.88846318817, Loss: 3.5194506645202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8700 , TFLOPS: 66.02285015056319, Tokens per sec: 163611.8956904435, Loss: 3.5011558532714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8701 , TFLOPS: 69.03489017188463, Tokens per sec: 171076.0626668775, Loss: 3.492032527923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8702 , TFLOPS: 69.10131199187214, Tokens per sec: 171240.66325377405, Loss: 3.5037457942962646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8703 , TFLOPS: 67.09171684851256, Tokens per sec: 166260.66511334895, Loss: 3.4872677326202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8704 , TFLOPS: 67.882826518079, Tokens per sec: 168221.12202245896, Loss: 3.506786346435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8705 , TFLOPS: 68.57437021996384, Tokens per sec: 169934.84349555717, Loss: 3.511937141418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8706 , TFLOPS: 68.14175715777478, Tokens per sec: 168862.78067118974, Loss: 3.474680185317993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8707 , TFLOPS: 67.20372884164503, Tokens per sec: 166538.24317147088, Loss: 3.5026791095733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8708 , TFLOPS: 69.05900328974167, Tokens per sec: 171135.81762920643, Loss: 3.5277116298675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8709 , TFLOPS: 67.9987027914509, Tokens per sec: 168508.27619269976, Loss: 3.497987985610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8710 , TFLOPS: 67.04367134400285, Tokens per sec: 166141.60306052465, Loss: 3.503570318222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8711 , TFLOPS: 69.32056717310479, Tokens per sec: 171784.00174582066, Loss: 3.466919183731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8712 , TFLOPS: 67.61583272607646, Tokens per sec: 167559.48199408248, Loss: 3.4995317459106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8713 , TFLOPS: 66.97595190423569, Tokens per sec: 165973.78682886952, Loss: 3.487062454223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8714 , TFLOPS: 68.99075621292604, Tokens per sec: 170966.69385482118, Loss: 3.4932467937469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8715 , TFLOPS: 69.54490246403594, Tokens per sec: 172339.9293093204, Loss: 3.517641067504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8716 , TFLOPS: 68.50263314253615, Tokens per sec: 169757.0711151999, Loss: 3.487759828567505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8717 , TFLOPS: 66.78894859860618, Tokens per sec: 165510.3720374049, Loss: 3.4927473068237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8718 , TFLOPS: 70.53360969087757, Tokens per sec: 174790.0547325262, Loss: 3.4781911373138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8719 , TFLOPS: 68.72956307137784, Tokens per sec: 170319.42847726494, Loss: 3.506147861480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8720 , TFLOPS: 66.95696018348107, Tokens per sec: 165926.72325272879, Loss: 3.5170273780822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8721 , TFLOPS: 69.91680996146403, Tokens per sec: 173261.55705694016, Loss: 3.516493558883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8722 , TFLOPS: 69.53956938641826, Tokens per sec: 172326.71335549682, Loss: 3.4840688705444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8723 , TFLOPS: 67.84129179153805, Tokens per sec: 168118.19439466263, Loss: 3.4812886714935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8724 , TFLOPS: 68.06780041027659, Tokens per sec: 168679.50770388092, Loss: 3.5098490715026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8725 , TFLOPS: 70.57737820860746, Tokens per sec: 174898.51794096112, Loss: 3.516857624053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8726 , TFLOPS: 69.05225081190754, Tokens per sec: 171119.08424528665, Loss: 3.4994335174560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8727 , TFLOPS: 66.37218010269184, Tokens per sec: 164477.57379368928, Loss: 3.502800941467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8728 , TFLOPS: 70.52068603856513, Tokens per sec: 174758.02849843813, Loss: 3.490199565887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8729 , TFLOPS: 69.50116894539853, Tokens per sec: 172231.55283249213, Loss: 3.526416301727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8730 , TFLOPS: 67.2867542748933, Tokens per sec: 166743.98934106657, Loss: 3.482353687286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8731 , TFLOPS: 68.64391205049782, Tokens per sec: 170107.1758122854, Loss: 3.5091686248779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8732 , TFLOPS: 69.56908491900361, Tokens per sec: 172399.85609664762, Loss: 3.4902687072753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8733 , TFLOPS: 67.93289623793935, Tokens per sec: 168345.20030390786, Loss: 3.498222827911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8734 , TFLOPS: 69.65048133211901, Tokens per sec: 172601.5653749027, Loss: 3.499833822250366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8735 , TFLOPS: 68.53946593484918, Tokens per sec: 169848.3468320186, Loss: 3.491274833679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8736 , TFLOPS: 69.64974999025297, Tokens per sec: 172599.7530291947, Loss: 3.476433753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8737 , TFLOPS: 67.0057154698123, Tokens per sec: 166047.54422309576, Loss: 3.4905569553375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8738 , TFLOPS: 69.54940739816864, Tokens per sec: 172351.0930323607, Loss: 3.4978349208831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8739 , TFLOPS: 68.74172394389834, Tokens per sec: 170349.5644590014, Loss: 3.4554331302642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8740 , TFLOPS: 68.71787162540897, Tokens per sec: 170290.45578623502, Loss: 3.500108003616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8741 , TFLOPS: 69.03583745072137, Tokens per sec: 171078.4101281863, Loss: 3.483264446258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8742 , TFLOPS: 68.70018324079577, Tokens per sec: 170246.62202062708, Loss: 3.517606735229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8743 , TFLOPS: 69.56314212772895, Tokens per sec: 172385.12920521654, Loss: 3.4898171424865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8744 , TFLOPS: 67.45160711369705, Tokens per sec: 167152.51283566165, Loss: 3.474914312362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8745 , TFLOPS: 69.5509012266905, Tokens per sec: 172354.7949039967, Loss: 3.494274377822876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8746 , TFLOPS: 68.78793703725216, Tokens per sec: 170464.0855951247, Loss: 3.493612766265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8747 , TFLOPS: 67.03748789995295, Tokens per sec: 166126.27980500643, Loss: 3.4933125972747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8748 , TFLOPS: 70.55923267841011, Tokens per sec: 174853.55131823692, Loss: 3.5149216651916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8749 , TFLOPS: 68.58475613052373, Tokens per sec: 169960.58092602706, Loss: 3.510730504989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8750 , TFLOPS: 68.04262570307792, Tokens per sec: 168617.1220062197, Loss: 3.480696678161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8751 , TFLOPS: 69.20150084092509, Tokens per sec: 171488.94225844007, Loss: 3.4713611602783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8752 , TFLOPS: 69.55649683775871, Tokens per sec: 172368.66144463117, Loss: 3.5198864936828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8753 , TFLOPS: 67.68230053782155, Tokens per sec: 167724.1965536207, Loss: 3.496194362640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8754 , TFLOPS: 68.94806287663205, Tokens per sec: 170860.89506442542, Loss: 3.4988839626312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8755 , TFLOPS: 69.56509433562987, Tokens per sec: 172389.96699144843, Loss: 3.4899072647094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8756 , TFLOPS: 69.75763462081092, Tokens per sec: 172867.10302819044, Loss: 3.4931678771972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8757 , TFLOPS: 66.39495932056194, Tokens per sec: 164534.02320490938, Loss: 3.4984099864959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8758 , TFLOPS: 69.60905123889026, Tokens per sec: 172498.89703998034, Loss: 3.487652540206909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8759 , TFLOPS: 68.03835676048361, Tokens per sec: 168606.54309620813, Loss: 3.502331495285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8760 , TFLOPS: 68.04718970031338, Tokens per sec: 168628.43209413497, Loss: 3.488109827041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8761 , TFLOPS: 67.62227572761563, Tokens per sec: 167575.44846165174, Loss: 3.495941400527954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8762 , TFLOPS: 68.97112862991355, Tokens per sec: 170918.05454195946, Loss: 3.4902734756469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8763 , TFLOPS: 69.19565430545725, Tokens per sec: 171474.45389950174, Loss: 3.5094356536865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8764 , TFLOPS: 64.73833954813878, Tokens per sec: 160428.7369171118, Loss: 3.4758493900299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8765 , TFLOPS: 70.11131928330131, Tokens per sec: 173743.57258342358, Loss: 3.481222152709961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8766 , TFLOPS: 68.32223946430268, Tokens per sec: 169310.03570853322, Loss: 3.4871342182159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8767 , TFLOPS: 66.9525292583355, Tokens per sec: 165915.7429321114, Loss: 3.473670721054077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8768 , TFLOPS: 69.6397985478345, Tokens per sec: 172575.09225863987, Loss: 3.4847850799560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8769 , TFLOPS: 69.50374330178639, Tokens per sec: 172237.93237121, Loss: 3.4938931465148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8770 , TFLOPS: 68.15810176539887, Tokens per sec: 168903.2844093894, Loss: 3.4881839752197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8771 , TFLOPS: 67.82809570239628, Tokens per sec: 168085.4930321001, Loss: 3.4800970554351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8772 , TFLOPS: 69.48084672033309, Tokens per sec: 172181.1921201021, Loss: 3.509434700012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8773 , TFLOPS: 69.53521072523245, Tokens per sec: 172315.9121129322, Loss: 3.4804606437683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8774 , TFLOPS: 67.13961310376843, Tokens per sec: 166379.35731008105, Loss: 3.4865357875823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8775 , TFLOPS: 69.21320475711059, Tokens per sec: 171517.94585204058, Loss: 3.505725145339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8776 , TFLOPS: 68.38865697793396, Tokens per sec: 169474.62562380463, Loss: 3.495504379272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8777 , TFLOPS: 65.8635164721849, Tokens per sec: 163217.04928336845, Loss: 3.4962639808654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8778 , TFLOPS: 69.18353293667691, Tokens per sec: 171444.41581238506, Loss: 3.515723466873169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8779 , TFLOPS: 67.88392596796116, Tokens per sec: 168223.84658038136, Loss: 3.5014429092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8780 , TFLOPS: 67.12689994497542, Tokens per sec: 166347.85270212195, Loss: 3.502265453338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8781 , TFLOPS: 68.1788541549758, Tokens per sec: 168954.71111682546, Loss: 3.4850995540618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8782 , TFLOPS: 68.40318762742316, Tokens per sec: 169510.6341739221, Loss: 3.480396032333374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8783 , TFLOPS: 69.78686898917316, Tokens per sec: 172939.5490134236, Loss: 3.480112314224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8784 , TFLOPS: 66.91416917785229, Tokens per sec: 165820.68242696227, Loss: 3.513662099838257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8785 , TFLOPS: 70.60751893625131, Tokens per sec: 174973.20998433782, Loss: 3.492539405822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8786 , TFLOPS: 68.4445866975083, Tokens per sec: 169613.2256300771, Loss: 3.5288829803466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8787 , TFLOPS: 67.92815885303477, Tokens per sec: 168333.46054224827, Loss: 3.517568826675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8788 , TFLOPS: 69.91884973502881, Tokens per sec: 173266.6118405333, Loss: 3.520341396331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8789 , TFLOPS: 69.60686545117846, Tokens per sec: 172493.48041724612, Loss: 3.497185468673706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8790 , TFLOPS: 67.7079814627797, Tokens per sec: 167787.83671465443, Loss: 3.4981822967529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8791 , TFLOPS: 68.08717404581988, Tokens per sec: 168727.51770693934, Loss: 3.497981071472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8792 , TFLOPS: 70.54812378392114, Tokens per sec: 174826.0222539475, Loss: 3.4878289699554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8793 , TFLOPS: 68.88024477104149, Tokens per sec: 170692.83432799097, Loss: 3.4783003330230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8794 , TFLOPS: 67.34145122225101, Tokens per sec: 166879.53440198515, Loss: 3.5051701068878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8795 , TFLOPS: 70.61028062537187, Tokens per sec: 174980.05375420375, Loss: 3.510408639907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8796 , TFLOPS: 69.48420056216067, Tokens per sec: 172189.50330960698, Loss: 3.502455472946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8797 , TFLOPS: 67.26912450289055, Tokens per sec: 166700.30082396406, Loss: 3.500551700592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8798 , TFLOPS: 69.72504393399153, Tokens per sec: 172786.33971609146, Loss: 3.5156126022338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8799 , TFLOPS: 69.52451300892605, Tokens per sec: 172289.4019934742, Loss: 3.478637218475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8800 , TFLOPS: 67.898944866111, Tokens per sec: 168261.06506446487, Loss: 3.50449538230896 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/model.pt +[rank0]:[2024-08-25 15:17:08,846] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00641285399615299, 'preprocessing_with_comm': 0.0015331129980040714, 'state_converting': 0.9626840249984525, : 0.9720075869990978}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800, took 4.19s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0006800 + eval ppl=23.88520050048828, eval loss=3.1732590198516846 +------------------------------------------------------------------ +iteration: 8801 , TFLOPS: 64.12992904211796, Tokens per sec: 158921.02866124213, Loss: 3.473850965499878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8802 , TFLOPS: 70.59274157143342, Tokens per sec: 174936.59004645798, Loss: 3.496565818786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8803 , TFLOPS: 68.0955252605293, Tokens per sec: 168748.21293695152, Loss: 3.4943811893463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8804 , TFLOPS: 68.74966949332753, Tokens per sec: 170369.2544057615, Loss: 3.5085108280181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8805 , TFLOPS: 65.29638693034755, Tokens per sec: 161811.63980421907, Loss: 3.5176186561584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8806 , TFLOPS: 68.84489598084564, Tokens per sec: 170605.23613189496, Loss: 3.5357296466827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8807 , TFLOPS: 66.81086767443085, Tokens per sec: 165564.68992188395, Loss: 3.498795509338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8808 , TFLOPS: 69.67497758150895, Tokens per sec: 172662.2697793756, Loss: 3.4940786361694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8809 , TFLOPS: 69.57536741524937, Tokens per sec: 172415.42481441871, Loss: 3.5124387741088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8810 , TFLOPS: 67.88884128400672, Tokens per sec: 168236.027275009, Loss: 3.489555835723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8811 , TFLOPS: 69.96382825729187, Tokens per sec: 173378.0735734944, Loss: 3.5023441314697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8812 , TFLOPS: 68.60033678517603, Tokens per sec: 169999.19150460628, Loss: 3.473465919494629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8813 , TFLOPS: 68.80348448186356, Tokens per sec: 170502.6138755642, Loss: 3.4918360710144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8814 , TFLOPS: 68.00595834798175, Tokens per sec: 168526.25626104875, Loss: 3.496185779571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8815 , TFLOPS: 70.54913484711679, Tokens per sec: 174828.52777992413, Loss: 3.500267744064331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8816 , TFLOPS: 66.92706350639037, Tokens per sec: 165852.6359935072, Loss: 3.4964096546173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8817 , TFLOPS: 68.09425612444436, Tokens per sec: 168745.06787792617, Loss: 3.5113492012023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8818 , TFLOPS: 70.6096596688026, Tokens per sec: 174978.5149554211, Loss: 3.4921178817749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8819 , TFLOPS: 68.81166841398618, Tokens per sec: 170522.89456089854, Loss: 3.507052421569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8820 , TFLOPS: 67.62699326902307, Tokens per sec: 167587.13904893945, Loss: 3.4878125190734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8821 , TFLOPS: 68.81971153812847, Tokens per sec: 170542.8263085464, Loss: 3.4846460819244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8822 , TFLOPS: 70.54342320991908, Tokens per sec: 174814.3737137594, Loss: 3.4968111515045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8823 , TFLOPS: 67.72998593348444, Tokens per sec: 167842.36621705335, Loss: 3.5053658485412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8824 , TFLOPS: 67.96063106368543, Tokens per sec: 168413.93025734992, Loss: 3.5095088481903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8825 , TFLOPS: 70.59891840447344, Tokens per sec: 174951.89691916548, Loss: 3.5198941230773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8826 , TFLOPS: 68.79903044702559, Tokens per sec: 170491.57628658324, Loss: 3.5004615783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8827 , TFLOPS: 66.55092694971731, Tokens per sec: 164920.5281712091, Loss: 3.504366874694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8828 , TFLOPS: 69.95495823465401, Tokens per sec: 173356.0926802841, Loss: 3.510406732559204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8829 , TFLOPS: 69.52625607410572, Tokens per sec: 172293.7215010039, Loss: 3.5113003253936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8830 , TFLOPS: 67.88429771939059, Tokens per sec: 168224.76782137528, Loss: 3.507674217224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8831 , TFLOPS: 69.59671449078134, Tokens per sec: 172468.32521916143, Loss: 3.4874074459075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8832 , TFLOPS: 69.63450165488487, Tokens per sec: 172561.96597440974, Loss: 3.4636054039001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8833 , TFLOPS: 68.65636276186869, Tokens per sec: 170138.0300466215, Loss: 3.5049831867218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8834 , TFLOPS: 66.96441224044469, Tokens per sec: 165945.1902707965, Loss: 3.4949634075164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8835 , TFLOPS: 70.5210734969962, Tokens per sec: 174758.98866311242, Loss: 3.514744997024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8836 , TFLOPS: 68.744418877349, Tokens per sec: 170356.2428009636, Loss: 3.4922707080841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8837 , TFLOPS: 66.84967495430968, Tokens per sec: 165660.85863639848, Loss: 3.4942357540130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8838 , TFLOPS: 69.66814053739934, Tokens per sec: 172645.32683090944, Loss: 3.4978926181793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8839 , TFLOPS: 69.5737522897763, Tokens per sec: 172411.4223555757, Loss: 3.507317066192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8840 , TFLOPS: 68.15938688346435, Tokens per sec: 168906.4690735231, Loss: 3.5120861530303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8841 , TFLOPS: 68.87120670371982, Tokens per sec: 170670.43700154257, Loss: 3.486654758453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8842 , TFLOPS: 69.52486704742131, Tokens per sec: 172290.279339866, Loss: 3.47103214263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8843 , TFLOPS: 67.63516377334808, Tokens per sec: 167607.3864587122, Loss: 3.527407169342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8844 , TFLOPS: 67.91195366822554, Tokens per sec: 168293.3023091425, Loss: 3.5000245571136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8845 , TFLOPS: 69.51106355556458, Tokens per sec: 172256.07276646487, Loss: 3.509824752807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8846 , TFLOPS: 68.70042092310833, Tokens per sec: 170247.21102358046, Loss: 3.4714736938476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8847 , TFLOPS: 68.82309706497134, Tokens per sec: 170551.2160170096, Loss: 3.486449718475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8848 , TFLOPS: 69.07128960268172, Tokens per sec: 171166.26446612098, Loss: 3.4942197799682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8849 , TFLOPS: 68.65928758490035, Tokens per sec: 170145.2780802898, Loss: 3.5158486366271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8850 , TFLOPS: 68.42382350915292, Tokens per sec: 169561.77216207847, Loss: 3.4836723804473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8851 , TFLOPS: 69.04211063347812, Tokens per sec: 171093.95576610565, Loss: 3.4826974868774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8852 , TFLOPS: 70.48892002318284, Tokens per sec: 174679.30881300537, Loss: 3.5014867782592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8853 , TFLOPS: 68.00906929987586, Tokens per sec: 168533.96554254202, Loss: 3.493830680847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8854 , TFLOPS: 67.28575644958539, Tokens per sec: 166741.51662003406, Loss: 3.513401985168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8855 , TFLOPS: 70.54414918901875, Tokens per sec: 174816.1727699416, Loss: 3.474517583847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8856 , TFLOPS: 69.51566672702823, Tokens per sec: 172267.47993243302, Loss: 3.5028865337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8857 , TFLOPS: 66.40127232698701, Tokens per sec: 164549.66753026578, Loss: 3.5206005573272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8858 , TFLOPS: 70.59041196906988, Tokens per sec: 174930.81703517423, Loss: 3.492467164993286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8859 , TFLOPS: 69.63085198771054, Tokens per sec: 172552.92169710773, Loss: 3.5046985149383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8860 , TFLOPS: 66.91695667056923, Tokens per sec: 165827.59014098253, Loss: 3.501680612564087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8861 , TFLOPS: 68.81519181572841, Tokens per sec: 170531.62593855054, Loss: 3.4755825996398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8862 , TFLOPS: 70.45006902448614, Tokens per sec: 174583.03175844415, Loss: 3.48907208442688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8863 , TFLOPS: 67.82960750504151, Tokens per sec: 168089.23944559382, Loss: 3.4862985610961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8864 , TFLOPS: 67.86727548080191, Tokens per sec: 168182.58483899815, Loss: 3.485536813735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8865 , TFLOPS: 70.57757707551134, Tokens per sec: 174899.01075505593, Loss: 3.4899463653564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8866 , TFLOPS: 68.66294938273899, Tokens per sec: 170154.3524187143, Loss: 3.510991096496582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8867 , TFLOPS: 67.8368431165761, Tokens per sec: 168107.17008804495, Loss: 3.47723650932312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8868 , TFLOPS: 68.89155118884625, Tokens per sec: 170720.8528768159, Loss: 3.5034146308898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8869 , TFLOPS: 69.4961422780206, Tokens per sec: 172219.09619699648, Loss: 3.505999803543091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8870 , TFLOPS: 68.68312990403378, Tokens per sec: 170204.36197355075, Loss: 3.4838550090789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8871 , TFLOPS: 67.79166419692322, Tokens per sec: 167995.21174827626, Loss: 3.4886741638183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8872 , TFLOPS: 70.49326189338126, Tokens per sec: 174690.06844565354, Loss: 3.51615309715271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8873 , TFLOPS: 68.66461589616945, Tokens per sec: 170158.48222257657, Loss: 3.4897656440734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8874 , TFLOPS: 67.06116918669349, Tokens per sec: 166184.96464225854, Loss: 3.4867725372314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8875 , TFLOPS: 69.55089400856798, Tokens per sec: 172354.77701669393, Loss: 3.492109775543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8876 , TFLOPS: 69.56955934077233, Tokens per sec: 172401.03176605242, Loss: 3.4936375617980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8877 , TFLOPS: 67.00457409374685, Tokens per sec: 166044.71576747263, Loss: 3.515866756439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8878 , TFLOPS: 69.92172143843156, Tokens per sec: 173273.72823790953, Loss: 3.4939041137695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8879 , TFLOPS: 68.7934216974501, Tokens per sec: 170477.67718728038, Loss: 3.4987685680389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8880 , TFLOPS: 67.99532972535775, Tokens per sec: 168499.91736334708, Loss: 3.487989902496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8881 , TFLOPS: 68.94677635397575, Tokens per sec: 170857.70691956097, Loss: 3.5252368450164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8882 , TFLOPS: 69.57668674797183, Tokens per sec: 172418.69426624148, Loss: 3.4906606674194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8883 , TFLOPS: 67.73876345734044, Tokens per sec: 167864.1178880926, Loss: 3.5008060932159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8884 , TFLOPS: 67.9411404557178, Tokens per sec: 168365.6303837384, Loss: 3.5027196407318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8885 , TFLOPS: 70.61052784696167, Tokens per sec: 174980.666396537, Loss: 3.4979450702667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8886 , TFLOPS: 67.85844313963979, Tokens per sec: 168160.6973246357, Loss: 3.5131802558898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8887 , TFLOPS: 68.76738306930854, Tokens per sec: 170413.1505983539, Loss: 3.502978563308716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8888 , TFLOPS: 68.79422982167779, Tokens per sec: 170479.67980813884, Loss: 3.45932674407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8889 , TFLOPS: 70.45121037357923, Tokens per sec: 174585.86014722675, Loss: 3.4905920028686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8890 , TFLOPS: 67.11307648030122, Tokens per sec: 166313.59663387891, Loss: 3.493206262588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8891 , TFLOPS: 68.72775859570946, Tokens per sec: 170314.9567877793, Loss: 3.4950966835021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8892 , TFLOPS: 70.5645631676801, Tokens per sec: 174866.76085784985, Loss: 3.4845173358917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8893 , TFLOPS: 68.83067795595846, Tokens per sec: 170570.0022999798, Loss: 3.4869701862335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8894 , TFLOPS: 66.18995483852567, Tokens per sec: 164025.99951531555, Loss: 3.508723258972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8895 , TFLOPS: 69.8940730485231, Tokens per sec: 173205.21248199476, Loss: 3.5134775638580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8896 , TFLOPS: 69.65523244078535, Tokens per sec: 172613.33914555574, Loss: 3.494194269180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8897 , TFLOPS: 66.86829449123239, Tokens per sec: 165706.99990000427, Loss: 3.5133016109466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8898 , TFLOPS: 69.13369137892485, Tokens per sec: 171320.90294177464, Loss: 3.471447706222534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8899 , TFLOPS: 69.54446140190274, Tokens per sec: 172338.83630876735, Loss: 3.501966714859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8900 , TFLOPS: 68.81044771873337, Tokens per sec: 170519.86954359143, Loss: 3.533055067062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8901 , TFLOPS: 67.29787622594459, Tokens per sec: 166771.55076095575, Loss: 3.506070613861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8902 , TFLOPS: 70.62175330172182, Tokens per sec: 175008.4843100186, Loss: 3.493710994720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8903 , TFLOPS: 68.80103852072435, Tokens per sec: 170496.55251441634, Loss: 3.5114500522613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8904 , TFLOPS: 67.12173500715349, Tokens per sec: 166335.05341723436, Loss: 3.493276357650757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8905 , TFLOPS: 70.51713270011057, Tokens per sec: 174749.2229343152, Loss: 3.470733165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8906 , TFLOPS: 69.52835940904579, Tokens per sec: 172298.93379668688, Loss: 3.5021116733551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8907 , TFLOPS: 68.35102423269723, Tokens per sec: 169381.36753551892, Loss: 3.5154812335968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8908 , TFLOPS: 67.73838582744347, Tokens per sec: 167863.18207960852, Loss: 3.5122036933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8909 , TFLOPS: 70.5126893093987, Tokens per sec: 174738.2117226513, Loss: 3.5022850036621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8910 , TFLOPS: 67.88095435745473, Tokens per sec: 168216.48260219648, Loss: 3.516317844390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8911 , TFLOPS: 68.79659501155064, Tokens per sec: 170485.54100919093, Loss: 3.488297939300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8912 , TFLOPS: 69.6981912054542, Tokens per sec: 172719.79569670372, Loss: 3.498760461807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8913 , TFLOPS: 68.7258326976655, Tokens per sec: 170310.18420027115, Loss: 3.4841039180755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8914 , TFLOPS: 68.67315095306053, Tokens per sec: 170179.63303376682, Loss: 3.5144219398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8915 , TFLOPS: 68.99254124208865, Tokens per sec: 170971.11735372333, Loss: 3.495323657989502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8916 , TFLOPS: 68.51182134446164, Tokens per sec: 169779.84049173526, Loss: 3.474325180053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8917 , TFLOPS: 68.69389118689094, Tokens per sec: 170231.02961792392, Loss: 3.4873740673065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8918 , TFLOPS: 68.2576396591269, Tokens per sec: 169149.95027505164, Loss: 3.4859867095947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8919 , TFLOPS: 68.56931648083551, Tokens per sec: 169922.31977328184, Loss: 3.4890260696411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8920 , TFLOPS: 69.79482384094366, Tokens per sec: 172959.2620124102, Loss: 3.4728944301605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8921 , TFLOPS: 67.2417907009905, Tokens per sec: 166632.5646517889, Loss: 3.5100624561309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8922 , TFLOPS: 70.56079995600037, Tokens per sec: 174857.43520475546, Loss: 3.5052034854888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8923 , TFLOPS: 68.76990777379267, Tokens per sec: 170419.40709418477, Loss: 3.4905951023101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8924 , TFLOPS: 67.35795511316681, Tokens per sec: 166920.43286172816, Loss: 3.4952239990234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8925 , TFLOPS: 69.88934172423059, Tokens per sec: 173193.48773919957, Loss: 3.47945237159729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8926 , TFLOPS: 69.60508083189507, Tokens per sec: 172489.05793406977, Loss: 3.4786362648010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8927 , TFLOPS: 67.65854452072448, Tokens per sec: 167665.32652631286, Loss: 3.4958882331848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8928 , TFLOPS: 68.66458133442009, Tokens per sec: 170158.3965747544, Loss: 3.5088882446289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8929 , TFLOPS: 70.59858385406491, Tokens per sec: 174951.0678663994, Loss: 3.5025060176849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8930 , TFLOPS: 67.87556617049542, Tokens per sec: 168203.13008135353, Loss: 3.522692918777466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8931 , TFLOPS: 67.7716156867157, Tokens per sec: 167945.5293314563, Loss: 3.512620449066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8932 , TFLOPS: 70.52322795753417, Tokens per sec: 174764.3276536589, Loss: 3.4635396003723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8933 , TFLOPS: 68.68580685186832, Tokens per sec: 170210.99574517485, Loss: 3.4919660091400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8934 , TFLOPS: 67.73211379324596, Tokens per sec: 167847.63928794547, Loss: 3.498950719833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8935 , TFLOPS: 68.8249904112333, Tokens per sec: 170555.90793761605, Loss: 3.4845852851867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8936 , TFLOPS: 69.58716138763434, Tokens per sec: 172444.65157720196, Loss: 3.4985036849975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8937 , TFLOPS: 68.66838650142478, Tokens per sec: 170167.82619776574, Loss: 3.462557554244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8938 , TFLOPS: 67.06093053949053, Tokens per sec: 166184.37324820046, Loss: 3.529822826385498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8939 , TFLOPS: 70.58286339095761, Tokens per sec: 174912.11082706752, Loss: 3.5032026767730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8940 , TFLOPS: 69.07114818136526, Tokens per sec: 171165.91400852, Loss: 3.498835802078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8941 , TFLOPS: 66.79667206606494, Tokens per sec: 165529.51164057787, Loss: 3.492739200592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8942 , TFLOPS: 70.60405005601349, Tokens per sec: 174964.61371697983, Loss: 3.502204656600952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8943 , TFLOPS: 69.59075644992151, Tokens per sec: 172453.56053757842, Loss: 3.4770517349243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8944 , TFLOPS: 67.14613136549319, Tokens per sec: 166395.51028068084, Loss: 3.4852559566497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8945 , TFLOPS: 68.6391548673358, Tokens per sec: 170095.38698836294, Loss: 3.503316640853882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8946 , TFLOPS: 69.60528321605403, Tokens per sec: 172489.5594643102, Loss: 3.5050652027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8947 , TFLOPS: 68.0187922852867, Tokens per sec: 168558.06017146533, Loss: 3.495422840118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8948 , TFLOPS: 68.93162643188242, Tokens per sec: 170820.163743712, Loss: 3.4884705543518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8949 , TFLOPS: 69.64162509873863, Tokens per sec: 172579.61865299148, Loss: 3.4880480766296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8950 , TFLOPS: 68.76850331999933, Tokens per sec: 170415.92670297148, Loss: 3.5109119415283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8951 , TFLOPS: 67.3416410852303, Tokens per sec: 166880.00490336295, Loss: 3.4886081218719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8952 , TFLOPS: 70.54761043705562, Tokens per sec: 174824.75012386494, Loss: 3.4780426025390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8953 , TFLOPS: 68.6301029463469, Tokens per sec: 170072.95533100102, Loss: 3.4756810665130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8954 , TFLOPS: 68.18564946463465, Tokens per sec: 168971.55064272523, Loss: 3.4864745140075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8955 , TFLOPS: 68.8131918508347, Tokens per sec: 170526.6698052296, Loss: 3.479724645614624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8956 , TFLOPS: 69.63294907245032, Tokens per sec: 172558.11850410586, Loss: 3.507136106491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8957 , TFLOPS: 68.01775005926564, Tokens per sec: 168555.47742057446, Loss: 3.49888277053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8958 , TFLOPS: 67.90136187002206, Tokens per sec: 168267.05466641142, Loss: 3.4757471084594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8959 , TFLOPS: 70.61648684467757, Tokens per sec: 174995.43344931473, Loss: 3.5011990070343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8960 , TFLOPS: 67.87389445640375, Tokens per sec: 168198.98738967985, Loss: 3.4860422611236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8961 , TFLOPS: 68.00232232740935, Tokens per sec: 168517.24580152964, Loss: 3.4915316104888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8962 , TFLOPS: 69.93293121556873, Tokens per sec: 173301.50730051432, Loss: 3.5008418560028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8963 , TFLOPS: 69.64718117012329, Tokens per sec: 172593.38720993444, Loss: 3.5283641815185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8964 , TFLOPS: 67.09420715431357, Tokens per sec: 166266.83636545367, Loss: 3.474693775177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8965 , TFLOPS: 69.04884934004869, Tokens per sec: 171110.65502331735, Loss: 3.5198779106140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8966 , TFLOPS: 69.56208709773229, Tokens per sec: 172382.51472466363, Loss: 3.4815657138824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8967 , TFLOPS: 68.87965483778235, Tokens per sec: 170691.37240838833, Loss: 3.479444980621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8968 , TFLOPS: 67.73212398344944, Tokens per sec: 167847.66454039226, Loss: 3.480621099472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8969 , TFLOPS: 70.55997148043703, Tokens per sec: 174855.3821510454, Loss: 3.503746747970581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8970 , TFLOPS: 68.78062816233512, Tokens per sec: 170445.97339794124, Loss: 3.5115268230438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8971 , TFLOPS: 67.00728604708637, Tokens per sec: 166051.43628659425, Loss: 3.498856782913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8972 , TFLOPS: 70.5907042573936, Tokens per sec: 174931.54135783864, Loss: 3.502218723297119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8973 , TFLOPS: 69.6176160726934, Tokens per sec: 172520.12164163883, Loss: 3.505563735961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8974 , TFLOPS: 67.54817070132584, Tokens per sec: 167391.8080431631, Loss: 3.4950106143951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8975 , TFLOPS: 67.86173671307027, Tokens per sec: 168168.85916241352, Loss: 3.481889486312866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8976 , TFLOPS: 70.61388097390167, Tokens per sec: 174988.97581447172, Loss: 3.501584053039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8977 , TFLOPS: 68.04062490379349, Tokens per sec: 168612.16380518663, Loss: 3.509537696838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8978 , TFLOPS: 67.05605160049372, Tokens per sec: 166172.28269990685, Loss: 3.482468605041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8979 , TFLOPS: 70.60343088495439, Tokens per sec: 174963.07934288803, Loss: 3.47821307182312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8980 , TFLOPS: 68.69036295928493, Tokens per sec: 170222.2862812499, Loss: 3.501275062561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8981 , TFLOPS: 68.72951117972654, Tokens per sec: 170319.29988403668, Loss: 3.4772286415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8982 , TFLOPS: 68.27219675984911, Tokens per sec: 169186.0243742383, Loss: 3.4658594131469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8983 , TFLOPS: 69.62565932771064, Tokens per sec: 172540.05371360923, Loss: 3.5095083713531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8984 , TFLOPS: 68.95642423115933, Tokens per sec: 170881.61542202326, Loss: 3.489541530609131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8985 , TFLOPS: 68.07449226685888, Tokens per sec: 168696.09086166072, Loss: 3.484908103942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8986 , TFLOPS: 68.74656558162789, Tokens per sec: 170361.56257064568, Loss: 3.500133991241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8987 , TFLOPS: 69.69241403659068, Tokens per sec: 172705.4792358522, Loss: 3.499854803085327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8988 , TFLOPS: 66.96502278542727, Tokens per sec: 165946.7032685204, Loss: 3.494844675064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8989 , TFLOPS: 70.58420364120333, Tokens per sec: 174915.43211481717, Loss: 3.4710209369659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8990 , TFLOPS: 68.61474883789462, Tokens per sec: 170034.90615886162, Loss: 3.5125317573547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8991 , TFLOPS: 67.46437395738252, Tokens per sec: 167184.15048069812, Loss: 3.4731369018554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8992 , TFLOPS: 70.46062542030452, Tokens per sec: 174609.1916701651, Loss: 3.5172605514526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8993 , TFLOPS: 69.54150999514826, Tokens per sec: 172331.52239770544, Loss: 3.49436092376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8994 , TFLOPS: 67.90893053742813, Tokens per sec: 168285.81065800122, Loss: 3.506971597671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8995 , TFLOPS: 68.90916967784796, Tokens per sec: 170764.51343339888, Loss: 3.4833569526672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8996 , TFLOPS: 70.53333126544037, Tokens per sec: 174789.3647636217, Loss: 3.466681718826294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8997 , TFLOPS: 66.87143177543808, Tokens per sec: 165714.77443586636, Loss: 3.4925334453582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8998 , TFLOPS: 67.94268174993715, Tokens per sec: 168369.44988060155, Loss: 3.4798009395599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8999 , TFLOPS: 70.59812671794766, Tokens per sec: 174949.93503274315, Loss: 3.4911551475524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9000 , TFLOPS: 68.89470995742853, Tokens per sec: 170728.68065332988, Loss: 3.4992685317993164 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/optimizer.pt +[rank0]:[2024-08-25 15:42:55,397] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063315830047940835, 'preprocessing_with_comm': 0.0016162480023922399, 'state_converting': 0.9865404360025423, : 0.9958562510000775}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000, took 4.24s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007000 + eval ppl=24.888025283813477, eval loss=3.2143867015838623 +------------------------------------------------------------------ +iteration: 9001 , TFLOPS: 63.480462700652275, Tokens per sec: 157311.57952246646, Loss: 3.4892759323120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9002 , TFLOPS: 69.06455504745564, Tokens per sec: 171149.5754963983, Loss: 3.4782633781433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9003 , TFLOPS: 69.5423229477599, Tokens per sec: 172333.5369838312, Loss: 3.512655258178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9004 , TFLOPS: 68.05610593357846, Tokens per sec: 168650.52750237, Loss: 3.495053291320801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9005 , TFLOPS: 68.19366701648728, Tokens per sec: 168991.41901942232, Loss: 3.509195566177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9006 , TFLOPS: 70.52718426109232, Tokens per sec: 174774.13180969883, Loss: 3.5119972229003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9007 , TFLOPS: 67.98828394984547, Tokens per sec: 168482.45715547138, Loss: 3.484311819076538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9008 , TFLOPS: 67.79804637062807, Tokens per sec: 168011.02747777133, Loss: 3.5052592754364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9009 , TFLOPS: 70.60382582515301, Tokens per sec: 174964.05804820647, Loss: 3.494354486465454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9010 , TFLOPS: 68.63102450500908, Tokens per sec: 170075.239054884, Loss: 3.513730525970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9011 , TFLOPS: 68.22490692170132, Tokens per sec: 169068.83494590237, Loss: 3.4951202869415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9012 , TFLOPS: 68.70755201341751, Tokens per sec: 170264.88265092103, Loss: 3.5031394958496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9013 , TFLOPS: 69.59287220579385, Tokens per sec: 172458.80361369983, Loss: 3.489901065826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9014 , TFLOPS: 68.22760492742583, Tokens per sec: 169075.52090129754, Loss: 3.5063326358795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9015 , TFLOPS: 68.75092676168833, Tokens per sec: 170372.37005525912, Loss: 3.5144453048706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9016 , TFLOPS: 70.63594276256762, Tokens per sec: 175043.6473570926, Loss: 3.493499994277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9017 , TFLOPS: 67.84687131205118, Tokens per sec: 168132.02106112908, Loss: 3.4961864948272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9018 , TFLOPS: 68.92314479154892, Tokens per sec: 170799.14530463578, Loss: 3.5030784606933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9019 , TFLOPS: 68.84298464758137, Tokens per sec: 170600.49963751534, Loss: 3.4999842643737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9020 , TFLOPS: 68.73588504260962, Tokens per sec: 170335.09501839994, Loss: 3.49393367767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9021 , TFLOPS: 68.66446970565016, Tokens per sec: 170158.11994636556, Loss: 3.5094213485717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9022 , TFLOPS: 68.67888339219871, Tokens per sec: 170193.8386494315, Loss: 3.5035009384155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9023 , TFLOPS: 68.67261756131619, Tokens per sec: 170178.31123026638, Loss: 3.494049549102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9024 , TFLOPS: 68.93188969907528, Tokens per sec: 170820.8161488172, Loss: 3.5101890563964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9025 , TFLOPS: 68.1588950584157, Tokens per sec: 168905.25027687353, Loss: 3.4894607067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9026 , TFLOPS: 70.51777390564565, Tokens per sec: 174750.81191226575, Loss: 3.515862464904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9027 , TFLOPS: 66.84260983645939, Tokens per sec: 165643.3504960785, Loss: 3.4824624061584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9028 , TFLOPS: 68.59996672318216, Tokens per sec: 169998.2744502198, Loss: 3.4794812202453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9029 , TFLOPS: 70.5815840808912, Tokens per sec: 174908.9405557953, Loss: 3.473513126373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9030 , TFLOPS: 68.75679256921352, Tokens per sec: 170386.90617247857, Loss: 3.485079526901245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9031 , TFLOPS: 67.99952254448478, Tokens per sec: 168510.30763102102, Loss: 3.5023622512817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9032 , TFLOPS: 69.69559423165252, Tokens per sec: 172713.36010954287, Loss: 3.4828710556030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9033 , TFLOPS: 69.56756528999912, Tokens per sec: 172396.09028857356, Loss: 3.505760908126831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9034 , TFLOPS: 66.92563203930631, Tokens per sec: 165849.08866038377, Loss: 3.4863078594207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9035 , TFLOPS: 69.60367328981657, Tokens per sec: 172485.5698897457, Loss: 3.4801340103149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9036 , TFLOPS: 70.55353938530304, Tokens per sec: 174839.44271074957, Loss: 3.4896023273468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9037 , TFLOPS: 68.03331077063145, Tokens per sec: 168594.0385774928, Loss: 3.5021612644195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9038 , TFLOPS: 67.8942522573001, Tokens per sec: 168249.4362628979, Loss: 3.492220640182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9039 , TFLOPS: 69.63254465209661, Tokens per sec: 172557.11630591878, Loss: 3.5033345222473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9040 , TFLOPS: 68.65662999381522, Tokens per sec: 170138.692276823, Loss: 3.486278772354126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9041 , TFLOPS: 68.77845499120026, Tokens per sec: 170440.58804047448, Loss: 3.499892234802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9042 , TFLOPS: 67.50622021837955, Tokens per sec: 167287.850125195, Loss: 3.5047519207000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9043 , TFLOPS: 70.56848678741734, Tokens per sec: 174876.4840197811, Loss: 3.496755838394165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9044 , TFLOPS: 68.2443392905419, Tokens per sec: 169116.99049654143, Loss: 3.4869272708892822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9045 , TFLOPS: 67.83796811038471, Tokens per sec: 168109.95794663095, Loss: 3.5009849071502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9046 , TFLOPS: 70.57259398155415, Tokens per sec: 174886.6620992969, Loss: 3.4781417846679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9047 , TFLOPS: 68.88759922287355, Tokens per sec: 170711.05946979058, Loss: 3.476689100265503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9048 , TFLOPS: 68.03030933584745, Tokens per sec: 168586.60069146275, Loss: 3.485372304916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9049 , TFLOPS: 69.52757678622649, Tokens per sec: 172296.99437112766, Loss: 3.497042179107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9050 , TFLOPS: 69.5266800325764, Tokens per sec: 172294.77211679713, Loss: 3.5009069442749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9051 , TFLOPS: 67.60359328987103, Tokens per sec: 167529.15132288018, Loss: 3.49869966506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9052 , TFLOPS: 68.24795390735872, Tokens per sec: 169125.94791519648, Loss: 3.4872798919677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9053 , TFLOPS: 69.61759974150155, Tokens per sec: 172520.08117114642, Loss: 3.502032518386841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9054 , TFLOPS: 68.84489299084734, Tokens per sec: 170605.2287223498, Loss: 3.486886978149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9055 , TFLOPS: 68.98838489057573, Tokens per sec: 170960.8174568144, Loss: 3.496265172958374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9056 , TFLOPS: 69.55539361479434, Tokens per sec: 172365.9275365954, Loss: 3.5012638568878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9057 , TFLOPS: 67.68515046685691, Tokens per sec: 167731.25899171596, Loss: 3.4931137561798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9058 , TFLOPS: 68.94961594790655, Tokens per sec: 170864.74374612857, Loss: 3.5008368492126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9059 , TFLOPS: 69.77224069253674, Tokens per sec: 172903.29848291865, Loss: 3.4988574981689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9060 , TFLOPS: 67.8609105021258, Tokens per sec: 168166.81172067847, Loss: 3.469640016555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9061 , TFLOPS: 68.72014559855896, Tokens per sec: 170296.0909421995, Loss: 3.5141053199768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9062 , TFLOPS: 68.79610979743265, Tokens per sec: 170484.3385951558, Loss: 3.4871246814727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9063 , TFLOPS: 70.60792039148134, Tokens per sec: 174974.20483462178, Loss: 3.493018388748169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9064 , TFLOPS: 67.90224010825335, Tokens per sec: 168269.2310374945, Loss: 3.485677719116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9065 , TFLOPS: 68.02667855131213, Tokens per sec: 168577.60320742027, Loss: 3.488781690597534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9066 , TFLOPS: 70.58729724116372, Tokens per sec: 174923.09839630718, Loss: 3.483759880065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9067 , TFLOPS: 68.01429076011567, Tokens per sec: 168546.90489620762, Loss: 3.483030319213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9068 , TFLOPS: 68.59270297911084, Tokens per sec: 169980.27409224358, Loss: 3.4901511669158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9069 , TFLOPS: 69.92548716343981, Tokens per sec: 173283.06011930885, Loss: 3.4884204864501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9070 , TFLOPS: 69.57616209581057, Tokens per sec: 172417.39412039067, Loss: 3.4777145385742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9071 , TFLOPS: 67.93187486846787, Tokens per sec: 168342.6692378444, Loss: 3.5109000205993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9072 , TFLOPS: 68.21783284535873, Tokens per sec: 169051.3046054516, Loss: 3.4800028800964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9073 , TFLOPS: 69.11674475364237, Tokens per sec: 171278.90733749972, Loss: 3.4981918334960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9074 , TFLOPS: 68.58656191896323, Tokens per sec: 169965.05586870416, Loss: 3.494452714920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9075 , TFLOPS: 67.86237283781746, Tokens per sec: 168170.43554961195, Loss: 3.492427110671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9076 , TFLOPS: 70.58430238317825, Tokens per sec: 174915.67680830832, Loss: 3.4976508617401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9077 , TFLOPS: 67.96361259463903, Tokens per sec: 168421.3188194957, Loss: 3.465623617172241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9078 , TFLOPS: 67.77053449776757, Tokens per sec: 167942.85002614002, Loss: 3.5093934535980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9079 , TFLOPS: 69.7190115791858, Tokens per sec: 172771.39087636414, Loss: 3.471327781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9080 , TFLOPS: 68.76151368653187, Tokens per sec: 170398.60562126746, Loss: 3.5134644508361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9081 , TFLOPS: 68.69180398157545, Tokens per sec: 170225.85729322748, Loss: 3.4933292865753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9082 , TFLOPS: 68.7731640111315, Tokens per sec: 170427.47640901452, Loss: 3.4801368713378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9083 , TFLOPS: 70.61957463729934, Tokens per sec: 175003.08533957993, Loss: 3.501232862472534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9084 , TFLOPS: 67.96406276353345, Tokens per sec: 168422.43438760625, Loss: 3.49849009513855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9085 , TFLOPS: 68.78185522536452, Tokens per sec: 170449.01419530026, Loss: 3.5089199542999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9086 , TFLOPS: 68.80356670743672, Tokens per sec: 170502.8176395923, Loss: 3.495546579360962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9087 , TFLOPS: 68.70845812280038, Tokens per sec: 170267.128089788, Loss: 3.505791187286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9088 , TFLOPS: 68.75301298141251, Tokens per sec: 170377.53993755172, Loss: 3.5170416831970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9089 , TFLOPS: 68.73297356200402, Tokens per sec: 170327.88004873318, Loss: 3.50253963470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9090 , TFLOPS: 69.50639811287144, Tokens per sec: 172244.51128553035, Loss: 3.4931955337524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9091 , TFLOPS: 68.88008507904252, Tokens per sec: 170692.438593626, Loss: 3.4895904064178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9092 , TFLOPS: 67.94447799502906, Tokens per sec: 168373.90117381205, Loss: 3.5014684200286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9093 , TFLOPS: 69.43254694784108, Tokens per sec: 172061.49996320767, Loss: 3.493757963180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9094 , TFLOPS: 68.04109929223341, Tokens per sec: 168613.339391999, Loss: 3.487704038619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9095 , TFLOPS: 68.79486463338847, Tokens per sec: 170481.2529414882, Loss: 3.5037825107574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9096 , TFLOPS: 70.62104985884532, Tokens per sec: 175006.74110107997, Loss: 3.4954898357391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9097 , TFLOPS: 67.90104158128, Tokens per sec: 168266.26095562588, Loss: 3.459704875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9098 , TFLOPS: 68.19172519183657, Tokens per sec: 168986.60696402824, Loss: 3.4982192516326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9099 , TFLOPS: 69.56258381668027, Tokens per sec: 172383.74564893488, Loss: 3.485642910003662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9100 , TFLOPS: 69.62165568709204, Tokens per sec: 172530.13225111898, Loss: 3.4894492626190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9101 , TFLOPS: 67.26910677625287, Tokens per sec: 166700.25689540314, Loss: 3.4845101833343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9102 , TFLOPS: 68.86308002680956, Tokens per sec: 170650.29820093155, Loss: 3.483340263366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9103 , TFLOPS: 70.55353061319175, Tokens per sec: 174839.4209724915, Loss: 3.5435092449188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9104 , TFLOPS: 67.62026349746849, Tokens per sec: 167570.46193368165, Loss: 3.4863972663879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9105 , TFLOPS: 68.81780684476217, Tokens per sec: 170538.10626856546, Loss: 3.4921748638153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9106 , TFLOPS: 69.61171933648009, Tokens per sec: 172505.50887972314, Loss: 3.4979023933410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9107 , TFLOPS: 68.79396066834492, Tokens per sec: 170479.0128165303, Loss: 3.509920358657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9108 , TFLOPS: 67.94395010770444, Tokens per sec: 168372.59301087004, Loss: 3.5039634704589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9109 , TFLOPS: 69.73269156902039, Tokens per sec: 172805.2913981495, Loss: 3.492616653442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9110 , TFLOPS: 70.53415673797433, Tokens per sec: 174791.4103754942, Loss: 3.5029971599578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9111 , TFLOPS: 68.06845949985903, Tokens per sec: 168681.14100047093, Loss: 3.4782166481018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9112 , TFLOPS: 66.85556179823571, Tokens per sec: 165675.4468841366, Loss: 3.504709005355835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9113 , TFLOPS: 70.60976000764035, Tokens per sec: 174978.76360611428, Loss: 3.488772392272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9114 , TFLOPS: 68.79394078797004, Tokens per sec: 170478.96355077135, Loss: 3.5012636184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9115 , TFLOPS: 68.19943043658867, Tokens per sec: 169005.70140932672, Loss: 3.5027763843536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9116 , TFLOPS: 69.66786838021393, Tokens per sec: 172644.65239542339, Loss: 3.4877190589904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9117 , TFLOPS: 69.60784283284393, Tokens per sec: 172495.9024766806, Loss: 3.5027718544006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9118 , TFLOPS: 68.30208772890376, Tokens per sec: 169260.09748831778, Loss: 3.5286412239074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9119 , TFLOPS: 68.0198878331956, Tokens per sec: 168560.77505986815, Loss: 3.5238940715789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9120 , TFLOPS: 69.62178679211902, Tokens per sec: 172530.4571438182, Loss: 3.479140281677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9121 , TFLOPS: 68.71958845478146, Tokens per sec: 170294.71027854437, Loss: 3.4879369735717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9122 , TFLOPS: 68.26592240903823, Tokens per sec: 169170.47584175214, Loss: 3.5031936168670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9123 , TFLOPS: 69.68316116885003, Tokens per sec: 172682.54961030337, Loss: 3.49123477935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9124 , TFLOPS: 67.75885372196046, Tokens per sec: 167913.90377694633, Loss: 3.480571746826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9125 , TFLOPS: 68.66414633624558, Tokens per sec: 170157.31860135894, Loss: 3.491729736328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9126 , TFLOPS: 68.60459035531234, Tokens per sec: 170009.73231997344, Loss: 3.4874801635742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9127 , TFLOPS: 68.80869347275588, Tokens per sec: 170515.52232881304, Loss: 3.4879419803619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9128 , TFLOPS: 68.75504952244064, Tokens per sec: 170382.58671056296, Loss: 3.483980894088745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9129 , TFLOPS: 69.69754220996265, Tokens per sec: 172718.1874143829, Loss: 3.5011954307556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9130 , TFLOPS: 69.53568505044636, Tokens per sec: 172317.0875430635, Loss: 3.497666597366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9131 , TFLOPS: 67.94488029231037, Tokens per sec: 168374.89811079187, Loss: 3.496253490447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9132 , TFLOPS: 67.83375164673372, Tokens per sec: 168099.509085222, Loss: 3.512861967086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9133 , TFLOPS: 70.53034408564181, Tokens per sec: 174781.9622030159, Loss: 3.4853217601776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9134 , TFLOPS: 68.7285166870703, Tokens per sec: 170316.8354216821, Loss: 3.4968318939208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9135 , TFLOPS: 68.70143979642224, Tokens per sec: 170249.73590389048, Loss: 3.496155261993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9136 , TFLOPS: 69.92835461843552, Tokens per sec: 173290.16598866446, Loss: 3.4947636127471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9137 , TFLOPS: 69.59035705638011, Tokens per sec: 172452.57079638602, Loss: 3.4873390197753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9138 , TFLOPS: 68.67610925888629, Tokens per sec: 170186.96404148734, Loss: 3.505858898162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9139 , TFLOPS: 68.1077756595153, Tokens per sec: 168778.5707751317, Loss: 3.491844892501831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9140 , TFLOPS: 69.68760740442619, Tokens per sec: 172693.56787185438, Loss: 3.480435371398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9141 , TFLOPS: 67.84339897030054, Tokens per sec: 168123.416215761, Loss: 3.517855644226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9142 , TFLOPS: 68.08490181749771, Tokens per sec: 168721.8868748504, Loss: 3.4971399307250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9143 , TFLOPS: 70.56283450949185, Tokens per sec: 174862.47705243193, Loss: 3.495026111602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9144 , TFLOPS: 68.00742730882531, Tokens per sec: 168529.89650783854, Loss: 3.507105827331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9145 , TFLOPS: 67.78653917717133, Tokens per sec: 167982.51138476355, Loss: 3.4885802268981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9146 , TFLOPS: 69.68578369624412, Tokens per sec: 172689.04852208117, Loss: 3.5199921131134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9147 , TFLOPS: 68.66829272649036, Tokens per sec: 170167.5938131481, Loss: 3.4947381019592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9148 , TFLOPS: 68.72316258430637, Tokens per sec: 170303.56736523, Loss: 3.4772019386291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9149 , TFLOPS: 68.76847270892046, Tokens per sec: 170415.85084534597, Loss: 3.5006210803985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9150 , TFLOPS: 70.57997972997765, Tokens per sec: 174904.9647974977, Loss: 3.508401393890381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9151 , TFLOPS: 67.96473196421874, Tokens per sec: 168424.09274062296, Loss: 3.512935161590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9152 , TFLOPS: 67.79095495554819, Tokens per sec: 167993.4541700195, Loss: 3.506925344467163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9153 , TFLOPS: 69.89373674825818, Tokens per sec: 173204.37909289182, Loss: 3.473607063293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9154 , TFLOPS: 68.83800283368512, Tokens per sec: 170588.15415388838, Loss: 3.524106502532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9155 , TFLOPS: 68.74922221244361, Tokens per sec: 170368.14599445908, Loss: 3.4782397747039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9156 , TFLOPS: 68.71349341278436, Tokens per sec: 170279.6060930513, Loss: 3.4789156913757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9157 , TFLOPS: 69.54136161801013, Tokens per sec: 172331.15470281205, Loss: 3.483130931854248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9158 , TFLOPS: 68.7381358622567, Tokens per sec: 170340.67279743362, Loss: 3.5133056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9159 , TFLOPS: 67.96770766849859, Tokens per sec: 168431.46686362932, Loss: 3.4921798706054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9160 , TFLOPS: 69.59787059831464, Tokens per sec: 172471.19018098473, Loss: 3.504517078399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9161 , TFLOPS: 68.08399425392557, Tokens per sec: 168719.63783234224, Loss: 3.4944069385528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9162 , TFLOPS: 67.99216881824697, Tokens per sec: 168492.08428732387, Loss: 3.497910499572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9163 , TFLOPS: 69.55178143683469, Tokens per sec: 172356.97616169712, Loss: 3.4955434799194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9164 , TFLOPS: 68.74496935421955, Tokens per sec: 170357.60694328885, Loss: 3.513125419616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9165 , TFLOPS: 69.00804834634583, Tokens per sec: 171009.54566632118, Loss: 3.5119218826293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9166 , TFLOPS: 69.16371608003455, Tokens per sec: 171395.3074586183, Loss: 3.496513605117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9167 , TFLOPS: 68.50819931695177, Tokens per sec: 169770.86470856657, Loss: 3.4973883628845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9168 , TFLOPS: 69.08196487470966, Tokens per sec: 171192.7189661845, Loss: 3.515109062194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9169 , TFLOPS: 68.98327806248183, Tokens per sec: 170948.16217423417, Loss: 3.4932548999786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9170 , TFLOPS: 70.60019501021615, Tokens per sec: 174955.06048882563, Loss: 3.501352310180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9171 , TFLOPS: 64.34855935185296, Tokens per sec: 159462.81865288227, Loss: 3.4838850498199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9172 , TFLOPS: 68.79881676594287, Tokens per sec: 170491.04676132134, Loss: 3.519476890563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9173 , TFLOPS: 69.74782346482688, Tokens per sec: 172842.78990287395, Loss: 3.5085222721099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9174 , TFLOPS: 68.7627325926662, Tokens per sec: 170401.6262049419, Loss: 3.4993364810943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9175 , TFLOPS: 67.86433556559882, Tokens per sec: 168175.2994052665, Loss: 3.495126247406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9176 , TFLOPS: 69.60673215122087, Tokens per sec: 172493.150085267, Loss: 3.48593807220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9177 , TFLOPS: 70.57007966663787, Tokens per sec: 174880.4313499597, Loss: 3.4858036041259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9178 , TFLOPS: 67.15761284201642, Tokens per sec: 166423.96264429446, Loss: 3.5140841007232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9179 , TFLOPS: 68.0171405272981, Tokens per sec: 168553.9669332134, Loss: 3.5065360069274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9180 , TFLOPS: 70.56478611454382, Tokens per sec: 174867.31334473775, Loss: 3.487980842590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9181 , TFLOPS: 68.72689769993507, Tokens per sec: 170312.8233932148, Loss: 3.5022401809692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9182 , TFLOPS: 67.31561522997632, Tokens per sec: 166815.50996705896, Loss: 3.4913558959960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9183 , TFLOPS: 69.94441197718845, Tokens per sec: 173329.95789251832, Loss: 3.5165908336639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9184 , TFLOPS: 69.56902156818694, Tokens per sec: 172399.69910634524, Loss: 3.478949785232544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9185 , TFLOPS: 68.30790297184623, Tokens per sec: 169274.50830093113, Loss: 3.526073694229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9186 , TFLOPS: 67.80663309701482, Tokens per sec: 168032.30633166368, Loss: 3.5119221210479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9187 , TFLOPS: 69.51857234390481, Tokens per sec: 172274.6803711333, Loss: 3.5104496479034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9188 , TFLOPS: 68.65496248798947, Tokens per sec: 170134.56001369588, Loss: 3.5061378479003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9189 , TFLOPS: 68.2550822686336, Tokens per sec: 169143.61277968943, Loss: 3.5038604736328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9190 , TFLOPS: 69.6529198416412, Tokens per sec: 172607.60827012267, Loss: 3.5042691230773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9191 , TFLOPS: 67.7254327905856, Tokens per sec: 167831.0830273799, Loss: 3.491898775100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9192 , TFLOPS: 68.74341805306558, Tokens per sec: 170353.7626481394, Loss: 3.489515781402588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9193 , TFLOPS: 68.55540108010334, Tokens per sec: 169887.83587735958, Loss: 3.5038013458251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9194 , TFLOPS: 68.62561521904986, Tokens per sec: 170061.83424839503, Loss: 3.4981417655944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9195 , TFLOPS: 68.66799833753454, Tokens per sec: 170166.864284886, Loss: 3.500347852706909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9196 , TFLOPS: 69.70103489177701, Tokens per sec: 172726.84266466956, Loss: 3.5159542560577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9197 , TFLOPS: 69.6506474657361, Tokens per sec: 172601.97707230697, Loss: 3.510859727859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9198 , TFLOPS: 67.96063307007955, Tokens per sec: 168413.9352294156, Loss: 3.4769489765167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9199 , TFLOPS: 67.90379116031428, Tokens per sec: 168273.074715364, Loss: 3.507863759994507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9200 , TFLOPS: 69.54432613922643, Tokens per sec: 172338.50111295504, Loss: 3.504260540008545 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/optimizer.pt +[rank0]:[2024-08-25 16:08:41,297] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063571409991709515, 'preprocessing_with_comm': 0.001307212995016016, 'state_converting': 1.002910802999395, : 1.0119475510000484}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007200 + eval ppl=25.73151969909668, eval loss=3.2477166652679443 +------------------------------------------------------------------ +iteration: 9201 , TFLOPS: 64.56304904919818, Tokens per sec: 159994.34775089446, Loss: 3.4998481273651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9202 , TFLOPS: 68.94556781157965, Tokens per sec: 170854.71201837147, Loss: 3.5033161640167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9203 , TFLOPS: 68.79795777257327, Tokens per sec: 170488.91808112554, Loss: 3.4673686027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9204 , TFLOPS: 68.29282269948173, Tokens per sec: 169237.13772478278, Loss: 3.488676071166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9205 , TFLOPS: 68.18254967206022, Tokens per sec: 168963.86901525545, Loss: 3.4997105598449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9206 , TFLOPS: 68.84293452689163, Tokens per sec: 170600.3754329249, Loss: 3.5020511150360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9207 , TFLOPS: 69.62543566466194, Tokens per sec: 172539.49945193605, Loss: 3.479645013809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9208 , TFLOPS: 67.14130493722391, Tokens per sec: 166383.5498597546, Loss: 3.490360975265503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9209 , TFLOPS: 69.00968438455699, Tokens per sec: 171013.5999492329, Loss: 3.507570743560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9210 , TFLOPS: 69.81685881683786, Tokens per sec: 173013.86711002895, Loss: 3.509901285171509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9211 , TFLOPS: 67.69561946525933, Tokens per sec: 167757.2023230125, Loss: 3.4939751625061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9212 , TFLOPS: 69.5703507344772, Tokens per sec: 172402.99292683226, Loss: 3.5164737701416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9213 , TFLOPS: 68.87372640446435, Tokens per sec: 170676.68109755547, Loss: 3.5035409927368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9214 , TFLOPS: 68.87343439973617, Tokens per sec: 170675.95747767203, Loss: 3.4946985244750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9215 , TFLOPS: 67.8024886383236, Tokens per sec: 168022.03590647705, Loss: 3.4739110469818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9216 , TFLOPS: 68.43759703001005, Tokens per sec: 169595.904464041, Loss: 3.5011472702026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9217 , TFLOPS: 70.60353548073941, Tokens per sec: 174963.33854276547, Loss: 3.4916796684265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9218 , TFLOPS: 67.03083396471087, Tokens per sec: 166109.79062048503, Loss: 3.507754325866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9219 , TFLOPS: 68.79351876421984, Tokens per sec: 170477.9177294286, Loss: 3.510449171066284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9220 , TFLOPS: 69.62898849358542, Tokens per sec: 172548.30375338488, Loss: 3.503570795059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9221 , TFLOPS: 68.26696361025496, Tokens per sec: 169173.056053065, Loss: 3.4932734966278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9222 , TFLOPS: 67.7098774937441, Tokens per sec: 167792.53528824926, Loss: 3.509770393371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9223 , TFLOPS: 68.85142319953046, Tokens per sec: 170621.4112988284, Loss: 3.478142499923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9224 , TFLOPS: 69.60075571486554, Tokens per sec: 172478.33981761968, Loss: 3.4825024604797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9225 , TFLOPS: 68.84512946414314, Tokens per sec: 170605.81472922646, Loss: 3.492731809616089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9226 , TFLOPS: 67.3594318384515, Tokens per sec: 166924.09234965837, Loss: 3.493211507797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9227 , TFLOPS: 69.58350516688817, Tokens per sec: 172435.591059427, Loss: 3.5049991607666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9228 , TFLOPS: 68.73182313143622, Tokens per sec: 170325.02915505576, Loss: 3.5027568340301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9229 , TFLOPS: 69.70172108921314, Tokens per sec: 172728.54313750716, Loss: 3.4823238849639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9230 , TFLOPS: 68.85859991362135, Tokens per sec: 170639.19598692597, Loss: 3.4758386611938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9231 , TFLOPS: 67.9580226676941, Tokens per sec: 168407.46636474485, Loss: 3.5027332305908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9232 , TFLOPS: 69.69770485218507, Tokens per sec: 172718.59045972658, Loss: 3.5041511058807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9233 , TFLOPS: 67.94033557958517, Tokens per sec: 168363.63581201734, Loss: 3.491154432296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9234 , TFLOPS: 68.65594856750752, Tokens per sec: 170137.00362736726, Loss: 3.487816095352173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9235 , TFLOPS: 68.68888554886674, Tokens per sec: 170218.62509548338, Loss: 3.4999659061431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9236 , TFLOPS: 68.85359668592871, Tokens per sec: 170626.7974375526, Loss: 3.477815866470337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9237 , TFLOPS: 69.60551652298466, Tokens per sec: 172490.13762458516, Loss: 3.4974613189697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9238 , TFLOPS: 68.0223379348451, Tokens per sec: 168566.8466816566, Loss: 3.4970006942749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9239 , TFLOPS: 68.79264865835971, Tokens per sec: 170475.76151125712, Loss: 3.475651502609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9240 , TFLOPS: 69.64526265697842, Tokens per sec: 172588.6329230198, Loss: 3.522268295288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9241 , TFLOPS: 67.67810686294195, Tokens per sec: 167713.80416529762, Loss: 3.484757423400879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9242 , TFLOPS: 68.69005742839076, Tokens per sec: 170221.52914203712, Loss: 3.4865317344665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9243 , TFLOPS: 68.77728237196764, Tokens per sec: 170437.6821608414, Loss: 3.496807098388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9244 , TFLOPS: 69.67712457795967, Tokens per sec: 172667.5902730917, Loss: 3.5065369606018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9245 , TFLOPS: 68.95424864860506, Tokens per sec: 170876.2240887936, Loss: 3.4853148460388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9246 , TFLOPS: 67.89178648323966, Tokens per sec: 168243.3258031487, Loss: 3.476579189300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9247 , TFLOPS: 70.44351368658805, Tokens per sec: 174566.78690900255, Loss: 3.4929871559143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9248 , TFLOPS: 67.95962766768751, Tokens per sec: 168411.4437315339, Loss: 3.495631694793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9249 , TFLOPS: 69.54928987733392, Tokens per sec: 172350.80180278627, Loss: 3.4975695610046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9250 , TFLOPS: 68.89795742061229, Tokens per sec: 170736.72822483684, Loss: 3.4852280616760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9251 , TFLOPS: 68.62884859845113, Tokens per sec: 170069.8469187375, Loss: 3.5024266242980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9252 , TFLOPS: 68.16573810947442, Tokens per sec: 168922.20811122135, Loss: 3.4892072677612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9253 , TFLOPS: 68.70938668240535, Tokens per sec: 170269.4291627784, Loss: 3.526956558227539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9254 , TFLOPS: 69.57988823067564, Tokens per sec: 172426.62789305393, Loss: 3.4862892627716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9255 , TFLOPS: 67.17727600041003, Tokens per sec: 166472.69011686955, Loss: 3.5020179748535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9256 , TFLOPS: 68.88924408512845, Tokens per sec: 170715.13561965476, Loss: 3.5008349418640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9257 , TFLOPS: 70.49147734435276, Tokens per sec: 174685.6461365766, Loss: 3.4796786308288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9258 , TFLOPS: 68.0449974140452, Tokens per sec: 168622.99936726215, Loss: 3.49090838432312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9259 , TFLOPS: 68.67563058897973, Tokens per sec: 170185.77784472902, Loss: 3.4913136959075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9260 , TFLOPS: 69.07081710440124, Tokens per sec: 171165.093563332, Loss: 3.490265369415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9261 , TFLOPS: 68.77298337503579, Tokens per sec: 170427.02877287063, Loss: 3.484456777572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9262 , TFLOPS: 68.67114648084447, Tokens per sec: 170174.66573080458, Loss: 3.476921796798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9263 , TFLOPS: 67.84291588518565, Tokens per sec: 168122.21907763006, Loss: 3.4769136905670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9264 , TFLOPS: 70.56356954987949, Tokens per sec: 174864.29856348387, Loss: 3.5059967041015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9265 , TFLOPS: 68.22957629147442, Tokens per sec: 169080.40615857363, Loss: 3.490856170654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9266 , TFLOPS: 68.76858089762462, Tokens per sec: 170416.11894887284, Loss: 3.513740062713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9267 , TFLOPS: 69.0828264316346, Tokens per sec: 171194.85399915287, Loss: 3.5124216079711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9268 , TFLOPS: 68.77481050656637, Tokens per sec: 170431.55660607858, Loss: 3.5486197471618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9269 , TFLOPS: 68.91009864769052, Tokens per sec: 170766.8155230033, Loss: 3.4814558029174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9270 , TFLOPS: 67.79420769943994, Tokens per sec: 168001.51482770257, Loss: 3.502974510192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9271 , TFLOPS: 69.64261352464479, Tokens per sec: 172582.06808127105, Loss: 3.5214293003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9272 , TFLOPS: 68.33348408286787, Tokens per sec: 169337.90111203477, Loss: 3.486436605453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9273 , TFLOPS: 68.76954831419938, Tokens per sec: 170418.51631371485, Loss: 3.473726511001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9274 , TFLOPS: 68.61919702598644, Tokens per sec: 170045.9292589615, Loss: 3.489839553833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9275 , TFLOPS: 68.82469519614374, Tokens per sec: 170555.1763621035, Loss: 3.4831979274749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9276 , TFLOPS: 69.0415949216583, Tokens per sec: 171092.6777754058, Loss: 3.5018951892852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9277 , TFLOPS: 69.90540686615465, Tokens per sec: 173233.29893060887, Loss: 3.4916346073150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9278 , TFLOPS: 67.03096338616582, Tokens per sec: 166110.11134110737, Loss: 3.5098204612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9279 , TFLOPS: 69.72803007826953, Tokens per sec: 172793.73971056327, Loss: 3.5001795291900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9280 , TFLOPS: 68.77576482202781, Tokens per sec: 170433.92150491857, Loss: 3.522965908050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9281 , TFLOPS: 68.79742579251476, Tokens per sec: 170487.5997759381, Loss: 3.5202364921569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9282 , TFLOPS: 67.62499196139206, Tokens per sec: 167582.17958816755, Loss: 3.476869821548462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9283 , TFLOPS: 68.83256130251509, Tokens per sec: 170574.66944021484, Loss: 3.5120158195495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9284 , TFLOPS: 70.59307417448231, Tokens per sec: 174937.4142734524, Loss: 3.5137996673583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9285 , TFLOPS: 67.79860574082258, Tokens per sec: 168012.41365873304, Loss: 3.510816812515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9286 , TFLOPS: 68.8120187086397, Tokens per sec: 170523.7626296381, Loss: 3.5051119327545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9287 , TFLOPS: 69.63282521542511, Tokens per sec: 172557.8115727534, Loss: 3.513270378112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9288 , TFLOPS: 68.6716625787821, Tokens per sec: 170175.94467834692, Loss: 3.498844623565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9289 , TFLOPS: 66.64892588018945, Tokens per sec: 165163.38031639246, Loss: 3.490386962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9290 , TFLOPS: 68.77782485142322, Tokens per sec: 170439.02648469133, Loss: 3.505845308303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9291 , TFLOPS: 69.6321039336783, Tokens per sec: 172556.02415712905, Loss: 3.495321035385132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9292 , TFLOPS: 67.9114556598367, Tokens per sec: 168292.06818949486, Loss: 3.4884605407714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9293 , TFLOPS: 68.29016132100372, Tokens per sec: 169230.54253573922, Loss: 3.47601318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9294 , TFLOPS: 69.56791515986758, Tokens per sec: 172396.95730464894, Loss: 3.5015456676483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9295 , TFLOPS: 68.73941006761063, Tokens per sec: 170343.830418666, Loss: 3.5050361156463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9296 , TFLOPS: 67.79664824564638, Tokens per sec: 168007.5627700505, Loss: 3.486344337463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9297 , TFLOPS: 69.54295531737085, Tokens per sec: 172335.1040653884, Loss: 3.505293846130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9298 , TFLOPS: 67.91278184589142, Tokens per sec: 168295.35462462992, Loss: 3.5130860805511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9299 , TFLOPS: 69.76241821161871, Tokens per sec: 172878.95729316818, Loss: 3.5014138221740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9300 , TFLOPS: 68.74127439244889, Tokens per sec: 170348.4504209875, Loss: 3.4971237182617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9301 , TFLOPS: 68.81863473878926, Tokens per sec: 170540.1578811646, Loss: 3.4944229125976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9302 , TFLOPS: 68.72820276899377, Tokens per sec: 170316.05749810755, Loss: 3.4812934398651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9303 , TFLOPS: 68.83557918180153, Tokens per sec: 170582.14807753352, Loss: 3.4972918033599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9304 , TFLOPS: 69.53512796533663, Tokens per sec: 172315.7070247936, Loss: 3.481678009033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9305 , TFLOPS: 67.76314423631771, Tokens per sec: 167924.53614416276, Loss: 3.4954092502593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9306 , TFLOPS: 69.70159603318086, Tokens per sec: 172728.23323488314, Loss: 3.514431953430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9307 , TFLOPS: 68.33406635570569, Tokens per sec: 169339.34404826953, Loss: 3.4801058769226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9308 , TFLOPS: 68.85193926186744, Tokens per sec: 170622.69015814844, Loss: 3.4970901012420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9309 , TFLOPS: 68.79462462558666, Tokens per sec: 170480.65817571626, Loss: 3.4993221759796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9310 , TFLOPS: 68.8155370388788, Tokens per sec: 170532.48143954668, Loss: 3.5044808387756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9311 , TFLOPS: 67.38837028323958, Tokens per sec: 166995.80500367607, Loss: 3.482790470123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9312 , TFLOPS: 68.65725232336807, Tokens per sec: 170140.23447801024, Loss: 3.5265769958496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9313 , TFLOPS: 68.08997216299232, Tokens per sec: 168734.4517495307, Loss: 3.51271915435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9314 , TFLOPS: 70.56214125867048, Tokens per sec: 174860.75910052867, Loss: 3.4998037815093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9315 , TFLOPS: 67.31830462354196, Tokens per sec: 166822.1745805758, Loss: 3.5043277740478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9316 , TFLOPS: 69.6143533041417, Tokens per sec: 172512.03614174412, Loss: 3.4807658195495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9317 , TFLOPS: 69.0800499276924, Tokens per sec: 171187.97351652695, Loss: 3.518571138381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9318 , TFLOPS: 68.57626299836392, Tokens per sec: 169939.53400893928, Loss: 3.4843297004699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9319 , TFLOPS: 69.00224551335802, Tokens per sec: 170995.16560694255, Loss: 3.4976966381073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9320 , TFLOPS: 68.7692473842313, Tokens per sec: 170417.77057610377, Loss: 3.474634885787964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9321 , TFLOPS: 69.71481982160388, Tokens per sec: 172761.0032393159, Loss: 3.4932196140289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9322 , TFLOPS: 68.06031738952136, Tokens per sec: 168660.9639541267, Loss: 3.4730629920959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9323 , TFLOPS: 68.21417459964809, Tokens per sec: 169042.23906959157, Loss: 3.4829659461975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9324 , TFLOPS: 70.55090240464301, Tokens per sec: 174832.90798218665, Loss: 3.511272430419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9325 , TFLOPS: 67.83774330332842, Tokens per sec: 168109.40084998097, Loss: 3.512564182281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9326 , TFLOPS: 68.78903755303817, Tokens per sec: 170466.81279447424, Loss: 3.494342088699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9327 , TFLOPS: 68.82701331454558, Tokens per sec: 170560.9209148651, Loss: 3.4643168449401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9328 , TFLOPS: 68.77274607475391, Tokens per sec: 170426.4407166314, Loss: 3.4853501319885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9329 , TFLOPS: 67.81213011956704, Tokens per sec: 168045.92855909513, Loss: 3.518279552459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9330 , TFLOPS: 68.76391103018678, Tokens per sec: 170404.54650293352, Loss: 3.522106647491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9331 , TFLOPS: 70.56671300639543, Tokens per sec: 174872.08839501068, Loss: 3.499814987182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9332 , TFLOPS: 68.03988415885645, Tokens per sec: 168610.32815763287, Loss: 3.50553560256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9333 , TFLOPS: 68.68670349463065, Tokens per sec: 170213.21772471326, Loss: 3.4949631690979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9334 , TFLOPS: 69.94137339872817, Tokens per sec: 173322.42796036668, Loss: 3.4922327995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9335 , TFLOPS: 68.8907493014173, Tokens per sec: 170718.86571143178, Loss: 3.4947736263275146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9336 , TFLOPS: 68.9762956745716, Tokens per sec: 170930.8590478191, Loss: 3.5014851093292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9337 , TFLOPS: 67.96320197936483, Tokens per sec: 168420.30126961324, Loss: 3.5061752796173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9338 , TFLOPS: 69.50318520463361, Tokens per sec: 172236.54934498595, Loss: 3.51401424407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9339 , TFLOPS: 68.06118877255116, Tokens per sec: 168663.12333726534, Loss: 3.5054099559783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9340 , TFLOPS: 68.78218855104946, Tokens per sec: 170449.84021306637, Loss: 3.50079345703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9341 , TFLOPS: 68.57447893567995, Tokens per sec: 169935.1129050777, Loss: 3.483431339263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9342 , TFLOPS: 68.93081801110975, Tokens per sec: 170818.1603879824, Loss: 3.4906740188598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9343 , TFLOPS: 68.74378528190638, Tokens per sec: 170354.67268166048, Loss: 3.4843811988830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9344 , TFLOPS: 69.14159894254463, Tokens per sec: 171340.49875551445, Loss: 3.5012996196746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9345 , TFLOPS: 67.77225950118046, Tokens per sec: 167947.12477462195, Loss: 3.4926581382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9346 , TFLOPS: 69.67701686769567, Tokens per sec: 172667.32335519223, Loss: 3.4974403381347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9347 , TFLOPS: 68.74303686832302, Tokens per sec: 170352.81803035637, Loss: 3.497370958328247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9348 , TFLOPS: 68.76492584824754, Tokens per sec: 170407.06133387954, Loss: 3.493032217025757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9349 , TFLOPS: 68.7535593012119, Tokens per sec: 170378.89377819668, Loss: 3.4854111671447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9350 , TFLOPS: 68.82600833024364, Tokens per sec: 170558.4304530567, Loss: 3.484858989715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9351 , TFLOPS: 70.58467156968969, Tokens per sec: 174916.59169315294, Loss: 3.5042784214019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9352 , TFLOPS: 66.88504542610369, Tokens per sec: 165748.51056188316, Loss: 3.5073819160461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9353 , TFLOPS: 68.78287428166968, Tokens per sec: 170451.53952908271, Loss: 3.483205795288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9354 , TFLOPS: 69.71022392406606, Tokens per sec: 172749.61409893728, Loss: 3.501160144805908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9355 , TFLOPS: 67.48725816735384, Tokens per sec: 167240.86007391094, Loss: 3.502333641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9356 , TFLOPS: 67.78164465091326, Tokens per sec: 167970.38220951945, Loss: 3.4879698753356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9357 , TFLOPS: 68.94815124878035, Tokens per sec: 170861.11406034394, Loss: 3.5066397190093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9358 , TFLOPS: 69.53699383674527, Tokens per sec: 172320.33085968692, Loss: 3.522120237350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9359 , TFLOPS: 67.85305005564136, Tokens per sec: 168147.33266839036, Loss: 3.5007309913635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9360 , TFLOPS: 68.22294228044115, Tokens per sec: 169063.96634843654, Loss: 3.4956960678100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9361 , TFLOPS: 70.56022159430775, Tokens per sec: 174856.00196076932, Loss: 3.477856159210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9362 , TFLOPS: 68.24276895652037, Tokens per sec: 169113.0990358495, Loss: 3.5063414573669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9363 , TFLOPS: 67.94006177597993, Tokens per sec: 168362.9572965216, Loss: 3.4968745708465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9364 , TFLOPS: 69.69567627302378, Tokens per sec: 172713.56341709848, Loss: 3.483337879180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9365 , TFLOPS: 68.75225653691622, Tokens per sec: 170375.66538476065, Loss: 3.491452693939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9366 , TFLOPS: 68.00518546975182, Tokens per sec: 168524.34098365545, Loss: 3.490841865539551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9367 , TFLOPS: 68.89946490976592, Tokens per sec: 170740.4639490232, Loss: 3.5131895542144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9368 , TFLOPS: 69.691306057533, Tokens per sec: 172702.73354169333, Loss: 3.475508213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9369 , TFLOPS: 68.1278823581498, Tokens per sec: 168828.3973892837, Loss: 3.5137276649475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9370 , TFLOPS: 68.69734819836646, Tokens per sec: 170239.5964731815, Loss: 3.494069814682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9371 , TFLOPS: 70.59117076615051, Tokens per sec: 174932.69741792846, Loss: 3.493539810180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9372 , TFLOPS: 67.99016625282565, Tokens per sec: 168487.12170960905, Loss: 3.4806108474731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9373 , TFLOPS: 68.88090430088693, Tokens per sec: 170694.46871560116, Loss: 3.493569850921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9374 , TFLOPS: 69.43555281233407, Tokens per sec: 172068.9488265446, Loss: 3.483494997024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9375 , TFLOPS: 68.80508615532023, Tokens per sec: 170506.58299882867, Loss: 3.49458646774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9376 , TFLOPS: 68.82941984492507, Tokens per sec: 170566.88456224668, Loss: 3.488396644592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9377 , TFLOPS: 68.74218851405665, Tokens per sec: 170350.71571503035, Loss: 3.490147829055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9378 , TFLOPS: 68.67466000496155, Tokens per sec: 170183.37263061336, Loss: 3.504568576812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9379 , TFLOPS: 68.86819824141354, Tokens per sec: 170662.9817005381, Loss: 3.4931516647338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9380 , TFLOPS: 68.02511105111863, Tokens per sec: 168573.71876926636, Loss: 3.5009331703186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9381 , TFLOPS: 69.59042619954282, Tokens per sec: 172452.74214075992, Loss: 3.514400005340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9382 , TFLOPS: 67.88520885401049, Tokens per sec: 168227.02571333313, Loss: 3.4894778728485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9383 , TFLOPS: 69.63740021446196, Tokens per sec: 172569.14892434448, Loss: 3.500514030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9384 , TFLOPS: 68.99931883455295, Tokens per sec: 170987.9129744633, Loss: 3.494550943374634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9385 , TFLOPS: 68.65835007717513, Tokens per sec: 170142.95483287174, Loss: 3.4743127822875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9386 , TFLOPS: 68.73581843483971, Tokens per sec: 170334.92995700866, Loss: 3.4963717460632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9387 , TFLOPS: 68.98587533780353, Tokens per sec: 170954.59850859485, Loss: 3.4827427864074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9388 , TFLOPS: 69.66209406446089, Tokens per sec: 172630.34300489395, Loss: 3.5203754901885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9389 , TFLOPS: 67.08633603210069, Tokens per sec: 166247.33085753446, Loss: 3.489468812942505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9390 , TFLOPS: 68.62690077906355, Tokens per sec: 170065.02000772505, Loss: 3.5235671997070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9391 , TFLOPS: 70.5919505503468, Tokens per sec: 174934.62980906715, Loss: 3.4767749309539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9392 , TFLOPS: 66.87258678024516, Tokens per sec: 165717.63666501257, Loss: 3.5001444816589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9393 , TFLOPS: 69.5719571265301, Tokens per sec: 172406.97374329806, Loss: 3.5072407722473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9394 , TFLOPS: 69.1847890819933, Tokens per sec: 171447.52867885493, Loss: 3.484147071838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9395 , TFLOPS: 68.86608455725067, Tokens per sec: 170657.743758341, Loss: 3.4907562732696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9396 , TFLOPS: 67.87103234368689, Tokens per sec: 168191.89475907973, Loss: 3.488617181777954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9397 , TFLOPS: 68.68105655666837, Tokens per sec: 170199.22399038117, Loss: 3.509974956512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9398 , TFLOPS: 70.55806518340637, Tokens per sec: 174850.65813700744, Loss: 3.5115652084350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9399 , TFLOPS: 68.11561167688647, Tokens per sec: 168797.98928938605, Loss: 3.519932746887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9400 , TFLOPS: 67.78301400499107, Tokens per sec: 167973.77561977107, Loss: 3.504030704498291 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/optimizer.pt +[rank0]:[2024-08-25 16:34:28,513] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006345925998175517, 'preprocessing_with_comm': 0.001262074991245754, 'state_converting': 0.987517988003674, : 0.9964910690032411}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400, took 4.24s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007400 + eval ppl=26.476423263549805, eval loss=3.276254653930664 +------------------------------------------------------------------ +iteration: 9401 , TFLOPS: 65.13597798577914, Tokens per sec: 161414.12876907454, Loss: 3.4895567893981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9402 , TFLOPS: 68.71320069617113, Tokens per sec: 170278.8807090383, Loss: 3.503098964691162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9403 , TFLOPS: 67.61774828097944, Tokens per sec: 167564.22895014787, Loss: 3.499095916748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9404 , TFLOPS: 68.74462728447838, Tokens per sec: 170356.75925678815, Loss: 3.491865634918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9405 , TFLOPS: 68.7041207038705, Tokens per sec: 170256.3794878736, Loss: 3.4935760498046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9406 , TFLOPS: 68.97220294657647, Tokens per sec: 170920.71681699596, Loss: 3.496324062347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9407 , TFLOPS: 67.43090366641964, Tokens per sec: 167101.2074126352, Loss: 3.507830858230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9408 , TFLOPS: 70.58720328101145, Tokens per sec: 174922.86555269928, Loss: 3.5044820308685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9409 , TFLOPS: 69.93391371112334, Tokens per sec: 173303.94203272948, Loss: 3.4780116081237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9410 , TFLOPS: 68.01366056512637, Tokens per sec: 168545.34320360242, Loss: 3.506427049636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9411 , TFLOPS: 68.66875925253993, Tokens per sec: 170168.74991609104, Loss: 3.49466872215271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9412 , TFLOPS: 68.71305491421467, Tokens per sec: 170278.51944529096, Loss: 3.499155044555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9413 , TFLOPS: 68.63621682844024, Tokens per sec: 170088.10620432804, Loss: 3.5126466751098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9414 , TFLOPS: 68.90154783338033, Tokens per sec: 170745.62566319268, Loss: 3.4960670471191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9415 , TFLOPS: 67.7787813667564, Tokens per sec: 167963.28667596198, Loss: 3.5064666271209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9416 , TFLOPS: 68.83554828710187, Tokens per sec: 170582.07151706444, Loss: 3.5242247581481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9417 , TFLOPS: 68.67658901351541, Tokens per sec: 170188.15292630764, Loss: 3.484165668487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9418 , TFLOPS: 69.63251636241309, Tokens per sec: 172557.04620096664, Loss: 3.495695114135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9419 , TFLOPS: 67.81684744199576, Tokens per sec: 168057.61860372944, Loss: 3.4814629554748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9420 , TFLOPS: 68.80704055259314, Tokens per sec: 170511.42621056558, Loss: 3.4772965908050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9421 , TFLOPS: 69.71400644788133, Tokens per sec: 172758.98760963092, Loss: 3.4929749965667725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9422 , TFLOPS: 67.71759811840113, Tokens per sec: 167811.66784664636, Loss: 3.5111687183380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9423 , TFLOPS: 68.9209624351553, Tokens per sec: 170793.73718508598, Loss: 3.5218377113342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9424 , TFLOPS: 68.75378045128481, Tokens per sec: 170379.44181243906, Loss: 3.511920690536499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9425 , TFLOPS: 69.6530177455388, Tokens per sec: 172607.85088676595, Loss: 3.502387523651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9426 , TFLOPS: 68.18991498075069, Tokens per sec: 168982.1210615462, Loss: 3.4945523738861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9427 , TFLOPS: 67.95273744404592, Tokens per sec: 168394.36899832924, Loss: 3.4961636066436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9428 , TFLOPS: 70.54079769043145, Tokens per sec: 174807.86738724422, Loss: 3.4976534843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9429 , TFLOPS: 69.57152918762318, Tokens per sec: 172405.91326354534, Loss: 3.5068442821502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9430 , TFLOPS: 67.8433805746021, Tokens per sec: 168123.37062919379, Loss: 3.487226963043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9431 , TFLOPS: 69.04645545494307, Tokens per sec: 171104.72271231742, Loss: 3.508180618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9432 , TFLOPS: 68.8718373731676, Tokens per sec: 170671.99986990812, Loss: 3.488003969192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9433 , TFLOPS: 68.72270811955005, Tokens per sec: 170302.44115150042, Loss: 3.4882853031158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9434 , TFLOPS: 68.08917416866826, Tokens per sec: 168732.47423168938, Loss: 3.500016212463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9435 , TFLOPS: 68.85637783380768, Tokens per sec: 170633.68942836544, Loss: 3.475508213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9436 , TFLOPS: 68.76980469931178, Tokens per sec: 170419.1516642665, Loss: 3.5161266326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9437 , TFLOPS: 67.45842844177511, Tokens per sec: 167169.4168380704, Loss: 3.5042977333068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9438 , TFLOPS: 70.61440425178128, Tokens per sec: 174990.2725547012, Loss: 3.4957165718078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9439 , TFLOPS: 67.8945879799577, Tokens per sec: 168250.26822062637, Loss: 3.4946165084838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9440 , TFLOPS: 69.59850740942771, Tokens per sec: 172472.76826907342, Loss: 3.4916512966156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9441 , TFLOPS: 68.20204869945968, Tokens per sec: 169012.18975314652, Loss: 3.500122547149658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9442 , TFLOPS: 68.78118781520445, Tokens per sec: 170447.36027940226, Loss: 3.487321376800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9443 , TFLOPS: 68.72072641588072, Tokens per sec: 170297.53027150512, Loss: 3.502082586288452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9444 , TFLOPS: 67.87278142678888, Tokens per sec: 168196.2291796839, Loss: 3.497084856033325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9445 , TFLOPS: 70.57109231897034, Tokens per sec: 174882.94081399243, Loss: 3.482219934463501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9446 , TFLOPS: 68.10375316123536, Tokens per sec: 168768.60258127924, Loss: 3.5186846256256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9447 , TFLOPS: 68.75541561637895, Tokens per sec: 170383.49393167064, Loss: 3.479984760284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9448 , TFLOPS: 68.8759339056732, Tokens per sec: 170682.1515287273, Loss: 3.4725565910339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9449 , TFLOPS: 69.58912699938608, Tokens per sec: 172449.52257965278, Loss: 3.4955360889434814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9450 , TFLOPS: 68.73876567667162, Tokens per sec: 170342.23354693357, Loss: 3.5093274116516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9451 , TFLOPS: 68.9664993117542, Tokens per sec: 170906.58258159913, Loss: 3.477658987045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9452 , TFLOPS: 67.56613984628257, Tokens per sec: 167436.3375638306, Loss: 3.5129148960113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9453 , TFLOPS: 68.74967780468731, Tokens per sec: 170369.2750022266, Loss: 3.4866576194763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9454 , TFLOPS: 68.6554235623405, Tokens per sec: 170135.7026067293, Loss: 3.4903604984283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9455 , TFLOPS: 68.83522895146302, Tokens per sec: 170581.28016817363, Loss: 3.5027987957000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9456 , TFLOPS: 68.68413506822522, Tokens per sec: 170206.8528811451, Loss: 3.483654737472534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9457 , TFLOPS: 69.13122962010092, Tokens per sec: 171314.8024322238, Loss: 3.5207011699676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9458 , TFLOPS: 70.00820711443396, Tokens per sec: 173488.04927022717, Loss: 3.50376558303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9459 , TFLOPS: 67.80024824748152, Tokens per sec: 168016.48397117094, Loss: 3.489467144012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9460 , TFLOPS: 68.93742559709386, Tokens per sec: 170834.53471393674, Loss: 3.472835063934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9461 , TFLOPS: 68.71205057975382, Tokens per sec: 170276.03059386043, Loss: 3.484206438064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9462 , TFLOPS: 68.75955565036028, Tokens per sec: 170393.75339193878, Loss: 3.4892728328704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9463 , TFLOPS: 68.61362508107445, Tokens per sec: 170032.12136566918, Loss: 3.485619306564331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9464 , TFLOPS: 68.9203948708933, Tokens per sec: 170792.33069832236, Loss: 3.4796555042266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9465 , TFLOPS: 70.55437010735366, Tokens per sec: 174841.5013315026, Loss: 3.4989893436431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9466 , TFLOPS: 67.77807275060704, Tokens per sec: 167961.5306470833, Loss: 3.4966766834259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9467 , TFLOPS: 67.90425671876274, Tokens per sec: 168274.22842048472, Loss: 3.495330810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9468 , TFLOPS: 69.5720010278664, Tokens per sec: 172407.08253564555, Loss: 3.490886688232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9469 , TFLOPS: 69.5448490141933, Tokens per sec: 172339.79685472237, Loss: 3.478099822998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9470 , TFLOPS: 68.77810156263976, Tokens per sec: 170439.7122055682, Loss: 3.501211166381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9471 , TFLOPS: 67.95431159739299, Tokens per sec: 168398.26992373058, Loss: 3.5073390007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9472 , TFLOPS: 68.71206687245255, Tokens per sec: 170276.07096896268, Loss: 3.496784210205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9473 , TFLOPS: 69.66429069216947, Tokens per sec: 172635.78649033437, Loss: 3.465536117553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9474 , TFLOPS: 67.19796254371079, Tokens per sec: 166523.95364997967, Loss: 3.4679598808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9475 , TFLOPS: 69.58572651960557, Tokens per sec: 172441.0958161629, Loss: 3.4702630043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9476 , TFLOPS: 70.01610722007463, Tokens per sec: 173507.62660227326, Loss: 3.5022261142730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9477 , TFLOPS: 69.05035788300083, Tokens per sec: 171114.3933589325, Loss: 3.499587059020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9478 , TFLOPS: 68.18835222627781, Tokens per sec: 168978.24838381077, Loss: 3.4831900596618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9479 , TFLOPS: 68.74242775452119, Tokens per sec: 170351.30857925597, Loss: 3.4782731533050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9480 , TFLOPS: 68.73761659605253, Tokens per sec: 170339.3859985778, Loss: 3.498246431350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9481 , TFLOPS: 67.92568026254705, Tokens per sec: 168327.31832198045, Loss: 3.498182773590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9482 , TFLOPS: 68.75598263657393, Tokens per sec: 170384.8990701762, Loss: 3.503303289413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9483 , TFLOPS: 68.13937276020253, Tokens per sec: 168856.87187134265, Loss: 3.5007920265197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9484 , TFLOPS: 68.86440451498275, Tokens per sec: 170653.58042852915, Loss: 3.511906385421753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9485 , TFLOPS: 69.61601451938927, Tokens per sec: 172516.15281612554, Loss: 3.4971978664398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9486 , TFLOPS: 67.7298482271602, Tokens per sec: 167842.02496561228, Loss: 3.4975576400756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9487 , TFLOPS: 68.80773442469217, Tokens per sec: 170513.1457020633, Loss: 3.473576784133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9488 , TFLOPS: 69.58744102742179, Tokens per sec: 172445.34455540017, Loss: 3.5106492042541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9489 , TFLOPS: 68.56452831487998, Tokens per sec: 169910.454170584, Loss: 3.4976649284362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9490 , TFLOPS: 68.17878812195245, Tokens per sec: 168954.54747971945, Loss: 3.493497133255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9491 , TFLOPS: 68.86941502912683, Tokens per sec: 170665.9970345319, Loss: 3.492947578430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9492 , TFLOPS: 69.57280588008501, Tokens per sec: 172409.07704810507, Loss: 3.475659132003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9493 , TFLOPS: 68.25035096930289, Tokens per sec: 169131.88809875227, Loss: 3.498931884765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9494 , TFLOPS: 67.8940060940806, Tokens per sec: 168248.8262433229, Loss: 3.5106523036956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9495 , TFLOPS: 70.57304665738769, Tokens per sec: 174887.78387987879, Loss: 3.4782168865203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9496 , TFLOPS: 68.04224382073649, Tokens per sec: 168616.17565970984, Loss: 3.497800350189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9497 , TFLOPS: 68.85831106427541, Tokens per sec: 170638.4801864274, Loss: 3.513223886489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9498 , TFLOPS: 68.76992329893267, Tokens per sec: 170419.4455671919, Loss: 3.5055322647094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9499 , TFLOPS: 68.77214243746292, Tokens per sec: 170424.94483692816, Loss: 3.4989869594573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9500 , TFLOPS: 67.77455638013805, Tokens per sec: 167952.816693701, Loss: 3.4965641498565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9501 , TFLOPS: 68.85328959370298, Tokens per sec: 170626.0364291882, Loss: 3.4885306358337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9502 , TFLOPS: 68.75879953515246, Tokens per sec: 170391.87965516082, Loss: 3.4871015548706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9503 , TFLOPS: 68.8485554359266, Tokens per sec: 170614.3046647078, Loss: 3.500777006149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9504 , TFLOPS: 67.75529031031016, Tokens per sec: 167905.07325033468, Loss: 3.489753484725952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9505 , TFLOPS: 70.56797688781718, Tokens per sec: 174875.2204324018, Loss: 3.511836051940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9506 , TFLOPS: 68.72118412821467, Tokens per sec: 170298.66453308982, Loss: 3.5071749687194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9507 , TFLOPS: 68.9928842191352, Tokens per sec: 170971.96728862674, Loss: 3.5099008083343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9508 , TFLOPS: 68.79855049113753, Tokens per sec: 170490.38690302114, Loss: 3.513958215713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9509 , TFLOPS: 69.57346318763585, Tokens per sec: 172410.70592862443, Loss: 3.4664015769958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9510 , TFLOPS: 68.74473634430755, Tokens per sec: 170357.02951905876, Loss: 3.490903377532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9511 , TFLOPS: 67.7998881302543, Tokens per sec: 168015.59156101168, Loss: 3.5068209171295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9512 , TFLOPS: 70.56904870136013, Tokens per sec: 174877.87650443125, Loss: 3.4680111408233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9513 , TFLOPS: 68.12142777949695, Tokens per sec: 168812.40223235142, Loss: 3.5109188556671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9514 , TFLOPS: 68.8660170699131, Tokens per sec: 170657.57651728415, Loss: 3.4915881156921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9515 , TFLOPS: 68.98516137396398, Tokens per sec: 170952.8292275508, Loss: 3.5029494762420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9516 , TFLOPS: 68.7936036924048, Tokens per sec: 170478.12819082654, Loss: 3.502753257751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9517 , TFLOPS: 68.5291793248953, Tokens per sec: 169822.85547938928, Loss: 3.5005359649658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9518 , TFLOPS: 67.60373598190894, Tokens per sec: 167529.504929469, Loss: 3.480368137359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9519 , TFLOPS: 68.74100086936559, Tokens per sec: 170347.77260065606, Loss: 3.4763717651367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9520 , TFLOPS: 68.76534996860748, Tokens per sec: 170408.11235085217, Loss: 3.483670234680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9521 , TFLOPS: 68.96163811767816, Tokens per sec: 170894.53600717397, Loss: 3.5170209407806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9522 , TFLOPS: 69.62365906579416, Tokens per sec: 172535.0968442332, Loss: 3.470625400543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9523 , TFLOPS: 69.19629109380664, Tokens per sec: 171476.03193117952, Loss: 3.4906532764434814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9524 , TFLOPS: 69.03815143118351, Tokens per sec: 171084.1444266775, Loss: 3.5057365894317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9525 , TFLOPS: 69.94321550815609, Tokens per sec: 173326.99291045417, Loss: 3.505388021469116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9526 , TFLOPS: 67.74992506207302, Tokens per sec: 167891.77757417219, Loss: 3.4903924465179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9527 , TFLOPS: 68.72499858675633, Tokens per sec: 170308.11718155336, Loss: 3.486307144165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9528 , TFLOPS: 68.8941209330088, Tokens per sec: 170727.22098593134, Loss: 3.5056958198547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9529 , TFLOPS: 69.60319996739558, Tokens per sec: 172484.39694464576, Loss: 3.513537645339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9530 , TFLOPS: 67.06151365053879, Tokens per sec: 166185.818261613, Loss: 3.502636432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9531 , TFLOPS: 68.40708857387709, Tokens per sec: 169520.30114895967, Loss: 3.499310255050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9532 , TFLOPS: 70.51717152836075, Tokens per sec: 174749.31915499642, Loss: 3.499154806137085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9533 , TFLOPS: 67.7001399802334, Tokens per sec: 167768.40465709407, Loss: 3.48335862159729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9534 , TFLOPS: 68.89296788075315, Tokens per sec: 170724.3635954227, Loss: 3.4840705394744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9535 , TFLOPS: 69.78710029547807, Tokens per sec: 172940.12221592764, Loss: 3.4929544925689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9536 , TFLOPS: 68.88339228594563, Tokens per sec: 170700.63421665083, Loss: 3.524200439453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9537 , TFLOPS: 67.58694827076391, Tokens per sec: 167487.90313193254, Loss: 3.4907450675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9538 , TFLOPS: 68.77838275241822, Tokens per sec: 170440.40902481487, Loss: 3.5077810287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9539 , TFLOPS: 68.65890284497941, Tokens per sec: 170144.3246523831, Loss: 3.51016902923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9540 , TFLOPS: 69.55290716003655, Tokens per sec: 172359.7658278, Loss: 3.4875388145446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9541 , TFLOPS: 67.36727420581262, Tokens per sec: 166943.52659988732, Loss: 3.5005927085876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9542 , TFLOPS: 69.64840627127359, Tokens per sec: 172596.42314554093, Loss: 3.505523204803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9543 , TFLOPS: 69.74847978323153, Tokens per sec: 172844.4163321801, Loss: 3.487607717514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9544 , TFLOPS: 68.06177023782718, Tokens per sec: 168664.56427227298, Loss: 3.4577975273132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9545 , TFLOPS: 69.56226175782724, Tokens per sec: 172382.94755161935, Loss: 3.5211260318756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9546 , TFLOPS: 68.54338739253035, Tokens per sec: 169858.06463613678, Loss: 3.4954781532287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9547 , TFLOPS: 68.69915294080194, Tokens per sec: 170244.06882374545, Loss: 3.485039472579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9548 , TFLOPS: 68.69744872025355, Tokens per sec: 170239.84557749113, Loss: 3.4994685649871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9549 , TFLOPS: 69.68896399261797, Tokens per sec: 172696.92964683412, Loss: 3.518350601196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9550 , TFLOPS: 68.63848273989439, Tokens per sec: 170093.72138252217, Loss: 3.5137665271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9551 , TFLOPS: 68.8838215253274, Tokens per sec: 170701.6979191235, Loss: 3.499082326889038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9552 , TFLOPS: 69.4823561889501, Tokens per sec: 172184.93274961674, Loss: 3.501099109649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9553 , TFLOPS: 67.89905096329508, Tokens per sec: 168261.3279849746, Loss: 3.4908576011657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9554 , TFLOPS: 69.71196913580742, Tokens per sec: 172753.93892588333, Loss: 3.503035306930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9555 , TFLOPS: 68.17149170070113, Tokens per sec: 168936.46614409154, Loss: 3.497274398803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9556 , TFLOPS: 68.895882989191, Tokens per sec: 170731.58755525737, Loss: 3.4934229850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9557 , TFLOPS: 68.70663526398452, Tokens per sec: 170262.61084483782, Loss: 3.487231492996216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9558 , TFLOPS: 68.78807647886114, Tokens per sec: 170464.4311467925, Loss: 3.510807752609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9559 , TFLOPS: 69.52192688907917, Tokens per sec: 172282.99330360902, Loss: 3.4868268966674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9560 , TFLOPS: 69.17179933295888, Tokens per sec: 171415.33864980866, Loss: 3.4837844371795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9561 , TFLOPS: 68.01289647609987, Tokens per sec: 168543.4497068234, Loss: 3.4697108268737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9562 , TFLOPS: 70.59455508519083, Tokens per sec: 174941.0841333239, Loss: 3.5267701148986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9563 , TFLOPS: 68.04354608622688, Tokens per sec: 168619.40281705142, Loss: 3.4752655029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9564 , TFLOPS: 68.74724409453894, Tokens per sec: 170363.24400038362, Loss: 3.4806432723999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9565 , TFLOPS: 69.116015942615, Tokens per sec: 171277.1012634887, Loss: 3.4920806884765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9566 , TFLOPS: 68.73984689463107, Tokens per sec: 170344.91292414308, Loss: 3.4832634925842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9567 , TFLOPS: 67.88698030797023, Tokens per sec: 168231.41557138716, Loss: 3.5094187259674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9568 , TFLOPS: 68.74727956079808, Tokens per sec: 170363.33188968073, Loss: 3.4975385665893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9569 , TFLOPS: 69.63846180783968, Tokens per sec: 172571.7796696786, Loss: 3.5119762420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9570 , TFLOPS: 68.58708229071297, Tokens per sec: 169966.3454072237, Loss: 3.4880099296569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9571 , TFLOPS: 68.28043133412471, Tokens per sec: 169206.4305564065, Loss: 3.476073980331421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9572 , TFLOPS: 70.55566182996347, Tokens per sec: 174844.7023624247, Loss: 3.4973526000976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9573 , TFLOPS: 67.94804332693029, Tokens per sec: 168382.736459017, Loss: 3.4932658672332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9574 , TFLOPS: 68.79485518458515, Tokens per sec: 170481.22952631267, Loss: 3.4831225872039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9575 , TFLOPS: 68.78519024498212, Tokens per sec: 170457.27874129536, Loss: 3.5015804767608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9576 , TFLOPS: 68.67128647370942, Tokens per sec: 170175.01264854538, Loss: 3.4851560592651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9577 , TFLOPS: 68.68096248859946, Tokens per sec: 170198.99087934397, Loss: 3.4689078330993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9578 , TFLOPS: 67.75566785123566, Tokens per sec: 167906.0088383376, Loss: 3.4995265007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9579 , TFLOPS: 70.52459275544796, Tokens per sec: 174767.70977323418, Loss: 3.468388557434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9580 , TFLOPS: 68.17565597938507, Tokens per sec: 168946.7856854035, Loss: 3.51491641998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9581 , TFLOPS: 68.7992516394065, Tokens per sec: 170492.1244256695, Loss: 3.4791958332061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9582 , TFLOPS: 69.63824080127175, Tokens per sec: 172571.23199105743, Loss: 3.491879463195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9583 , TFLOPS: 68.61035858426736, Tokens per sec: 170024.02662674687, Loss: 3.502432346343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9584 , TFLOPS: 68.68320487448986, Tokens per sec: 170204.54775859957, Loss: 3.5020227432250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9585 , TFLOPS: 67.91581315191135, Tokens per sec: 168302.8665348732, Loss: 3.481706142425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9586 , TFLOPS: 68.76713309741328, Tokens per sec: 170412.53114046078, Loss: 3.4855430126190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9587 , TFLOPS: 68.85493799366637, Tokens per sec: 170630.12134588376, Loss: 3.5180444717407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9588 , TFLOPS: 68.73950262685072, Tokens per sec: 170344.05979065914, Loss: 3.504978656768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9589 , TFLOPS: 68.66206221144554, Tokens per sec: 170152.15391051915, Loss: 3.5031790733337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9590 , TFLOPS: 69.64936239881374, Tokens per sec: 172598.79253491157, Loss: 3.4935052394866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9591 , TFLOPS: 68.75775648394398, Tokens per sec: 170389.29485936463, Loss: 3.482762575149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9592 , TFLOPS: 68.8695722615929, Tokens per sec: 170666.38667390335, Loss: 3.5106942653656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9593 , TFLOPS: 68.80076891169736, Tokens per sec: 170495.88439354757, Loss: 3.493597984313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9594 , TFLOPS: 68.99045901663102, Tokens per sec: 170965.95736966323, Loss: 3.5002524852752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9595 , TFLOPS: 68.78889624858571, Tokens per sec: 170466.46262647508, Loss: 3.471222400665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9596 , TFLOPS: 67.82793754983435, Tokens per sec: 168085.10111262972, Loss: 3.489807367324829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9597 , TFLOPS: 68.83208014500053, Tokens per sec: 170573.47707888993, Loss: 3.4860470294952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9598 , TFLOPS: 68.72724335823621, Tokens per sec: 170313.6799725625, Loss: 3.491072416305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9599 , TFLOPS: 70.61515671763364, Tokens per sec: 174992.13724797431, Loss: 3.5019145011901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9600 , TFLOPS: 67.04748504376536, Tokens per sec: 166151.0538286499, Loss: 3.507477283477783 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/optimizer.pt +[rank0]:[2024-08-25 17:00:14,870] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0064155050058616325, 'preprocessing_with_comm': 0.0013215850049164146, 'state_converting': 0.9900920890067937, : 0.9992145049909595}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007600 + eval ppl=24.391294479370117, eval loss=3.1942262649536133 +------------------------------------------------------------------ +iteration: 9601 , TFLOPS: 64.43666784220369, Tokens per sec: 159681.16119792283, Loss: 3.497530460357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9602 , TFLOPS: 69.71985576256957, Tokens per sec: 172773.4828557836, Loss: 3.5116915702819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9603 , TFLOPS: 68.72387732651747, Tokens per sec: 170305.33857516444, Loss: 3.5095715522766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9604 , TFLOPS: 67.50731333017413, Tokens per sec: 167290.55897663816, Loss: 3.490510940551758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9605 , TFLOPS: 68.72714946701132, Tokens per sec: 170313.4472997642, Loss: 3.4906582832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9606 , TFLOPS: 69.553855587014, Tokens per sec: 172362.11613432798, Loss: 3.4877610206604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9607 , TFLOPS: 68.22648432037045, Tokens per sec: 169072.74391356925, Loss: 3.4895269870758057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9608 , TFLOPS: 68.94753747993387, Tokens per sec: 170859.59307353012, Loss: 3.4927289485931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9609 , TFLOPS: 69.57246451840965, Tokens per sec: 172408.2311162694, Loss: 3.4675960540771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9610 , TFLOPS: 68.72762052258992, Tokens per sec: 170314.61462737905, Loss: 3.508759021759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9611 , TFLOPS: 66.75384643093486, Tokens per sec: 165423.38499909235, Loss: 3.493924379348755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9612 , TFLOPS: 70.48591425110585, Tokens per sec: 174671.86017868546, Loss: 3.500985622406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9613 , TFLOPS: 68.79052535291693, Tokens per sec: 170470.4997264685, Loss: 3.509610652923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9614 , TFLOPS: 68.73729922147152, Tokens per sec: 170338.59950940436, Loss: 3.4994423389434814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9615 , TFLOPS: 68.72213508288858, Tokens per sec: 170301.02110352807, Loss: 3.4974029064178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9616 , TFLOPS: 68.85989625051013, Tokens per sec: 170642.4084525397, Loss: 3.4945082664489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9617 , TFLOPS: 68.68207871386967, Tokens per sec: 170201.75700852592, Loss: 3.5226149559020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9618 , TFLOPS: 68.6418467657778, Tokens per sec: 170102.05780923928, Loss: 3.469278335571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9619 , TFLOPS: 69.56284318774061, Tokens per sec: 172384.38839899443, Loss: 3.526918649673462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9620 , TFLOPS: 68.84926855375679, Tokens per sec: 170616.07184924747, Loss: 3.520972967147827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9621 , TFLOPS: 68.92498874924279, Tokens per sec: 170803.71483492988, Loss: 3.505967855453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9622 , TFLOPS: 68.72205182074171, Tokens per sec: 170300.81477075606, Loss: 3.4855666160583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9623 , TFLOPS: 69.11041563720633, Tokens per sec: 171263.22308976148, Loss: 3.485386610031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9624 , TFLOPS: 68.72591375641949, Tokens per sec: 170310.38507279294, Loss: 3.499988555908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9625 , TFLOPS: 69.65692778466082, Tokens per sec: 172617.54039443668, Loss: 3.5111312866210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9626 , TFLOPS: 68.51846200928466, Tokens per sec: 169796.29679069633, Loss: 3.4777002334594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9627 , TFLOPS: 67.97516186341375, Tokens per sec: 168449.9391797764, Loss: 3.4877469539642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9628 , TFLOPS: 68.88988262347554, Tokens per sec: 170716.71798801963, Loss: 3.4832375049591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9629 , TFLOPS: 69.65737724908918, Tokens per sec: 172618.65421680274, Loss: 3.528552532196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9630 , TFLOPS: 69.7267882716025, Tokens per sec: 172790.6623768461, Loss: 3.506401538848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9631 , TFLOPS: 67.99033992744386, Tokens per sec: 168487.55209444484, Loss: 3.4987740516662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9632 , TFLOPS: 69.92357186795641, Tokens per sec: 173278.3138061138, Loss: 3.479029417037964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9633 , TFLOPS: 67.77025446175058, Tokens per sec: 167942.1560660414, Loss: 3.4843506813049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9634 , TFLOPS: 68.67146580112804, Tokens per sec: 170175.45704164327, Loss: 3.4888267517089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9635 , TFLOPS: 68.88422150782064, Tokens per sec: 170702.68911980343, Loss: 3.4882900714874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9636 , TFLOPS: 68.77000544430882, Tokens per sec: 170419.64913248306, Loss: 3.5030882358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9637 , TFLOPS: 67.8572413052793, Tokens per sec: 168157.71904669743, Loss: 3.5033528804779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9638 , TFLOPS: 69.6829599548704, Tokens per sec: 172682.05097989627, Loss: 3.505682945251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9639 , TFLOPS: 70.53626276444884, Tokens per sec: 174796.62934109595, Loss: 3.514563798904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9640 , TFLOPS: 68.92600413673895, Tokens per sec: 170806.23107699954, Loss: 3.482102632522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9641 , TFLOPS: 68.03276765416345, Tokens per sec: 168592.6926750559, Loss: 3.5024220943450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9642 , TFLOPS: 69.76663312628912, Tokens per sec: 172889.40231603265, Loss: 3.4901411533355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9643 , TFLOPS: 68.83951536211539, Tokens per sec: 170591.9023659573, Loss: 3.498983144760132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9644 , TFLOPS: 68.61872083890162, Tokens per sec: 170044.74921490898, Loss: 3.4853339195251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9645 , TFLOPS: 69.56838105505757, Tokens per sec: 172398.11184425233, Loss: 3.4896867275238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9646 , TFLOPS: 69.50208300602891, Tokens per sec: 172233.8179754263, Loss: 3.4953126907348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9647 , TFLOPS: 67.88757936151073, Tokens per sec: 168232.9000920517, Loss: 3.5053811073303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9648 , TFLOPS: 67.91835999328988, Tokens per sec: 168309.17788836692, Loss: 3.527585983276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9649 , TFLOPS: 69.61852477695248, Tokens per sec: 172522.37351089486, Loss: 3.4865472316741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9650 , TFLOPS: 69.71611945979747, Tokens per sec: 172764.2238859266, Loss: 3.487055778503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9651 , TFLOPS: 68.82200865495741, Tokens per sec: 170548.51881709686, Loss: 3.4996538162231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9652 , TFLOPS: 68.73738667924405, Tokens per sec: 170338.816239399, Loss: 3.4942290782928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9653 , TFLOPS: 68.65241909987624, Tokens per sec: 170128.257217774, Loss: 3.492640972137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9654 , TFLOPS: 68.65792855796587, Tokens per sec: 170141.91026183698, Loss: 3.4925742149353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9655 , TFLOPS: 68.86256885798389, Tokens per sec: 170649.03146827166, Loss: 3.5100698471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9656 , TFLOPS: 68.62433694759599, Tokens per sec: 170058.66655091906, Loss: 3.4860572814941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9657 , TFLOPS: 69.14921506944837, Tokens per sec: 171359.3723569676, Loss: 3.490837335586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9658 , TFLOPS: 68.8896714433019, Tokens per sec: 170716.19466028595, Loss: 3.4950459003448486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9659 , TFLOPS: 69.65229662266252, Tokens per sec: 172606.06386484028, Loss: 3.4920525550842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9660 , TFLOPS: 68.64083375278325, Tokens per sec: 170099.5474514462, Loss: 3.501199722290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9661 , TFLOPS: 67.9100624965423, Tokens per sec: 168288.6157773817, Loss: 3.4891152381896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9662 , TFLOPS: 69.6978650728491, Tokens per sec: 172718.98750418183, Loss: 3.4895503520965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9663 , TFLOPS: 67.86309000286076, Tokens per sec: 168172.21276359152, Loss: 3.507995128631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9664 , TFLOPS: 68.9169195015934, Tokens per sec: 170783.71835035368, Loss: 3.4863715171813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9665 , TFLOPS: 69.445432673229, Tokens per sec: 172093.4322101988, Loss: 3.5111241340637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9666 , TFLOPS: 69.5963135468337, Tokens per sec: 172467.33163589158, Loss: 3.5162017345428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9667 , TFLOPS: 68.8741022573318, Tokens per sec: 170677.61250236465, Loss: 3.5044422149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9668 , TFLOPS: 69.09023058390487, Tokens per sec: 171213.20230411328, Loss: 3.4862561225891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9669 , TFLOPS: 69.94069018940544, Tokens per sec: 173320.7348924032, Loss: 3.5148813724517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9670 , TFLOPS: 68.05403985311133, Tokens per sec: 168645.40752737492, Loss: 3.4899067878723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9671 , TFLOPS: 68.68314404440208, Tokens per sec: 170204.39701494106, Loss: 3.5053443908691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9672 , TFLOPS: 68.69957950378253, Tokens per sec: 170245.12589380113, Loss: 3.487135887145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9673 , TFLOPS: 69.53801096013257, Tokens per sec: 172322.85140348604, Loss: 3.4888463020324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9674 , TFLOPS: 67.1783274148676, Tokens per sec: 166475.29563771823, Loss: 3.5133728981018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9675 , TFLOPS: 68.77681959675006, Tokens per sec: 170436.53535287332, Loss: 3.506131172180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9676 , TFLOPS: 69.5992456264178, Tokens per sec: 172474.59765209618, Loss: 3.513097047805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9677 , TFLOPS: 68.72110391019265, Tokens per sec: 170298.4657439945, Loss: 3.4997808933258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9678 , TFLOPS: 66.60586480608768, Tokens per sec: 165056.67023119936, Loss: 3.474311113357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9679 , TFLOPS: 70.62478916422265, Tokens per sec: 175016.00751172344, Loss: 3.4919373989105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9680 , TFLOPS: 68.73351485200678, Tokens per sec: 170329.221424988, Loss: 3.5014917850494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9681 , TFLOPS: 66.8307064653261, Tokens per sec: 165613.8526311456, Loss: 3.476104974746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9682 , TFLOPS: 69.70065189563842, Tokens per sec: 172725.8935580476, Loss: 3.4835422039031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9683 , TFLOPS: 68.78181765030246, Tokens per sec: 170448.92108015722, Loss: 3.509460210800171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9684 , TFLOPS: 68.9028589787065, Tokens per sec: 170748.87482574547, Loss: 3.4915149211883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9685 , TFLOPS: 68.69396094882389, Tokens per sec: 170231.20249567868, Loss: 3.4932925701141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9686 , TFLOPS: 69.66687337032404, Tokens per sec: 172642.1866513068, Loss: 3.476656913757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9687 , TFLOPS: 68.87697384300614, Tokens per sec: 170684.7286079969, Loss: 3.507437229156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9688 , TFLOPS: 68.43630035567837, Tokens per sec: 169592.69116220664, Loss: 3.5023434162139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9689 , TFLOPS: 68.727097623535, Tokens per sec: 170313.31882591886, Loss: 3.4972128868103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9690 , TFLOPS: 68.65520705090992, Tokens per sec: 170135.1660675536, Loss: 3.490415096282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9691 , TFLOPS: 68.79261557648739, Tokens per sec: 170475.67953073324, Loss: 3.5157856941223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9692 , TFLOPS: 67.96359108010073, Tokens per sec: 168421.26550409975, Loss: 3.4871840476989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9693 , TFLOPS: 69.63301286946134, Tokens per sec: 172558.276600127, Loss: 3.489992380142212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9694 , TFLOPS: 67.74823983438854, Tokens per sec: 167887.6013943269, Loss: 3.524139881134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9695 , TFLOPS: 68.80250361943634, Tokens per sec: 170500.18319041855, Loss: 3.5028374195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9696 , TFLOPS: 68.52071257024186, Tokens per sec: 169801.87392866804, Loss: 3.509415864944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9697 , TFLOPS: 68.94841991280215, Tokens per sec: 170861.77983938574, Loss: 3.479098081588745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9698 , TFLOPS: 68.75518013397452, Tokens per sec: 170382.91038033192, Loss: 3.486696481704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9699 , TFLOPS: 70.56189204343785, Tokens per sec: 174860.14151772892, Loss: 3.4794278144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9700 , TFLOPS: 67.82571194270818, Tokens per sec: 168079.5858130002, Loss: 3.48299241065979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9701 , TFLOPS: 68.72415421913763, Tokens per sec: 170306.02474557943, Loss: 3.4681949615478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9702 , TFLOPS: 69.11246502546098, Tokens per sec: 171268.30169961526, Loss: 3.495699882507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9703 , TFLOPS: 67.88949468376137, Tokens per sec: 168237.64647157895, Loss: 3.5082364082336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9704 , TFLOPS: 68.82618652509397, Tokens per sec: 170558.87203952562, Loss: 3.471221685409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9705 , TFLOPS: 69.67963007394012, Tokens per sec: 172673.79916813437, Loss: 3.5097758769989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9706 , TFLOPS: 70.57257880772796, Tokens per sec: 174886.62449688406, Loss: 3.513035297393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9707 , TFLOPS: 68.0386006577688, Tokens per sec: 168607.14750054834, Loss: 3.5096845626831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9708 , TFLOPS: 68.18947125630034, Tokens per sec: 168981.0214634779, Loss: 3.4651060104370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9709 , TFLOPS: 69.76806560876861, Tokens per sec: 172892.95216541778, Loss: 3.4795734882354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9710 , TFLOPS: 68.78976567853255, Tokens per sec: 170468.61716965897, Loss: 3.4994308948516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9711 , TFLOPS: 67.93249140670636, Tokens per sec: 168344.1970875167, Loss: 3.4915757179260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9712 , TFLOPS: 68.88694339192129, Tokens per sec: 170709.43424844512, Loss: 3.4765870571136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9713 , TFLOPS: 69.55691157895943, Tokens per sec: 172369.68921901405, Loss: 3.5137181282043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9714 , TFLOPS: 67.87283544380846, Tokens per sec: 168196.36303980887, Loss: 3.4918041229248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9715 , TFLOPS: 67.69226869735691, Tokens per sec: 167748.89875103274, Loss: 3.5199155807495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9716 , TFLOPS: 69.62534224260617, Tokens per sec: 172539.26794179057, Loss: 3.4844884872436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9717 , TFLOPS: 69.68731654788435, Tokens per sec: 172692.8470973032, Loss: 3.5015182495117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9718 , TFLOPS: 68.10099505598704, Tokens per sec: 168761.76769264354, Loss: 3.4920451641082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9719 , TFLOPS: 70.02068175344756, Tokens per sec: 173518.96279989782, Loss: 3.4770517349243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9720 , TFLOPS: 68.79253182166562, Tokens per sec: 170475.47197705857, Loss: 3.5071120262145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9721 , TFLOPS: 68.78086552585648, Tokens per sec: 170446.56161089492, Loss: 3.5007240772247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9722 , TFLOPS: 67.81164567302307, Tokens per sec: 168044.72804719294, Loss: 3.5214948654174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9723 , TFLOPS: 69.59481461986277, Tokens per sec: 172463.6171297372, Loss: 3.5087156295776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9724 , TFLOPS: 69.19752491146345, Tokens per sec: 171479.08946724937, Loss: 3.4732489585876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9725 , TFLOPS: 69.04212105946606, Tokens per sec: 171093.98160285232, Loss: 3.495365619659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9726 , TFLOPS: 69.61067354103974, Tokens per sec: 172502.9172834181, Loss: 3.50661563873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9727 , TFLOPS: 68.74179691810583, Tokens per sec: 170349.74529712636, Loss: 3.506652593612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9728 , TFLOPS: 67.9664719197948, Tokens per sec: 168428.40454221237, Loss: 3.5017282962799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9729 , TFLOPS: 69.70831298670124, Tokens per sec: 172744.87858564014, Loss: 3.5047152042388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9730 , TFLOPS: 67.86878125234296, Tokens per sec: 168186.31630675128, Loss: 3.4854984283447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9731 , TFLOPS: 68.76434807040725, Tokens per sec: 170405.6295367438, Loss: 3.50595760345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9732 , TFLOPS: 68.8598206789154, Tokens per sec: 170642.22117780303, Loss: 3.5029940605163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9733 , TFLOPS: 68.60345911736799, Tokens per sec: 170006.92898773123, Loss: 3.479703426361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9734 , TFLOPS: 68.78286065416519, Tokens per sec: 170451.50575862543, Loss: 3.5030391216278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9735 , TFLOPS: 69.03395655623156, Tokens per sec: 171073.74906444293, Loss: 3.523486614227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9736 , TFLOPS: 68.98463060268533, Tokens per sec: 170951.51391785304, Loss: 3.520380973815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9737 , TFLOPS: 68.75278298835981, Tokens per sec: 170376.96998943127, Loss: 3.4999804496765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9738 , TFLOPS: 68.76745911702416, Tokens per sec: 170413.33905297058, Loss: 3.489008665084839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9739 , TFLOPS: 69.61154391555708, Tokens per sec: 172505.07416735173, Loss: 3.4881393909454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9740 , TFLOPS: 68.65279119691563, Tokens per sec: 170129.17931522746, Loss: 3.482783079147339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9741 , TFLOPS: 68.29885065055181, Tokens per sec: 169252.07565156804, Loss: 3.514713764190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9742 , TFLOPS: 68.93731715469973, Tokens per sec: 170834.26598173814, Loss: 3.503467082977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9743 , TFLOPS: 69.6611225463036, Tokens per sec: 172627.93547587865, Loss: 3.4937057495117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9744 , TFLOPS: 68.71203169219253, Tokens per sec: 170275.98378840287, Loss: 3.49723482131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9745 , TFLOPS: 68.73848002712656, Tokens per sec: 170341.52567589402, Loss: 3.4931399822235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9746 , TFLOPS: 70.56700425194981, Tokens per sec: 174872.8101335778, Loss: 3.4936482906341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9747 , TFLOPS: 68.81207737016153, Tokens per sec: 170523.9079993513, Loss: 3.467705249786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9748 , TFLOPS: 67.861108506898, Tokens per sec: 168167.302398316, Loss: 3.4816994667053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9749 , TFLOPS: 69.67922247916638, Tokens per sec: 172672.7891033848, Loss: 3.5020108222961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9750 , TFLOPS: 68.53473800614348, Tokens per sec: 169836.63050386155, Loss: 3.4896421432495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9751 , TFLOPS: 68.81982009379341, Tokens per sec: 170543.09532144252, Loss: 3.4942359924316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9752 , TFLOPS: 67.93313954512462, Tokens per sec: 168345.80324591548, Loss: 3.4983177185058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9753 , TFLOPS: 69.52427474859732, Tokens per sec: 172288.81155813308, Loss: 3.4814910888671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9754 , TFLOPS: 69.56320734160467, Tokens per sec: 172385.29081238445, Loss: 3.489703416824341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9755 , TFLOPS: 67.16051042477027, Tokens per sec: 166431.14317355357, Loss: 3.4862282276153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9756 , TFLOPS: 69.60599094758422, Tokens per sec: 172491.31330100505, Loss: 3.468743085861206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9757 , TFLOPS: 69.12780224802778, Tokens per sec: 171306.30902667035, Loss: 3.501890182495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9758 , TFLOPS: 68.79068726662646, Tokens per sec: 170470.900966477, Loss: 3.4836578369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9759 , TFLOPS: 68.72949466478532, Tokens per sec: 170319.2589581931, Loss: 3.47578501701355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9760 , TFLOPS: 69.56739110608773, Tokens per sec: 172395.65864165316, Loss: 3.4886229038238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9761 , TFLOPS: 67.82814453875305, Tokens per sec: 168085.61405397198, Loss: 3.525392532348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9762 , TFLOPS: 68.92597370833415, Tokens per sec: 170806.1556720605, Loss: 3.5151498317718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9763 , TFLOPS: 68.5253573033136, Tokens per sec: 169813.3840888794, Loss: 3.496476650238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9764 , TFLOPS: 68.92574075310645, Tokens per sec: 170805.5783833441, Loss: 3.499083995819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9765 , TFLOPS: 68.78936983995362, Tokens per sec: 170467.63623805516, Loss: 3.489304304122925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9766 , TFLOPS: 69.5584348457122, Tokens per sec: 172373.46404182908, Loss: 3.495573043823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9767 , TFLOPS: 68.77236096454372, Tokens per sec: 170425.48637110714, Loss: 3.489370822906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9768 , TFLOPS: 68.7433640339698, Tokens per sec: 170353.62878286935, Loss: 3.4851019382476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9769 , TFLOPS: 69.11067405602843, Tokens per sec: 171263.86348006985, Loss: 3.505012035369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9770 , TFLOPS: 68.0301826961902, Tokens per sec: 168586.28686444156, Loss: 3.46791672706604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9771 , TFLOPS: 68.25911725133425, Tokens per sec: 169153.61191131166, Loss: 3.4925243854522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9772 , TFLOPS: 68.7492983092458, Tokens per sec: 170368.33457071768, Loss: 3.504818916320801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9773 , TFLOPS: 70.6006598934112, Tokens per sec: 174956.21252059413, Loss: 3.4920654296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9774 , TFLOPS: 68.08785027905778, Tokens per sec: 168729.19348739609, Loss: 3.4867866039276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9775 , TFLOPS: 67.89020958406775, Tokens per sec: 168239.41807329093, Loss: 3.507157325744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9776 , TFLOPS: 69.75565538867193, Tokens per sec: 172862.1982729196, Loss: 3.507966995239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9777 , TFLOPS: 67.82226671022411, Tokens per sec: 168071.048147381, Loss: 3.5060837268829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9778 , TFLOPS: 68.69631949867838, Tokens per sec: 170237.04724203382, Loss: 3.488490343093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9779 , TFLOPS: 69.64293087719518, Tokens per sec: 172582.8545158502, Loss: 3.5129332542419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9780 , TFLOPS: 69.58488708123818, Tokens per sec: 172439.01559541677, Loss: 3.485321283340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9781 , TFLOPS: 67.90470131150262, Tokens per sec: 168275.3301702701, Loss: 3.51049542427063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9782 , TFLOPS: 68.32256268969536, Tokens per sec: 169310.8366966625, Loss: 3.4864139556884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9783 , TFLOPS: 69.65836425236336, Tokens per sec: 172621.10011964355, Loss: 3.5130324363708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9784 , TFLOPS: 69.64001225434382, Tokens per sec: 172575.62184691167, Loss: 3.499063491821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9785 , TFLOPS: 68.08688292858756, Tokens per sec: 168726.79628636842, Loss: 3.512014865875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9786 , TFLOPS: 69.8723736882245, Tokens per sec: 173151.439076793, Loss: 3.5006103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9787 , TFLOPS: 68.78274276959132, Tokens per sec: 170451.21362766533, Loss: 3.4841396808624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9788 , TFLOPS: 68.84657620842928, Tokens per sec: 170609.39992093958, Loss: 3.497972249984741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9789 , TFLOPS: 67.7787516910599, Tokens per sec: 167963.21313631695, Loss: 3.4889159202575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9790 , TFLOPS: 69.69913484504943, Tokens per sec: 172722.13413957116, Loss: 3.4878954887390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9791 , TFLOPS: 68.675037723675, Tokens per sec: 170184.30865919442, Loss: 3.492033004760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9792 , TFLOPS: 67.31750568933067, Tokens per sec: 166820.19473359035, Loss: 3.508209228515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9793 , TFLOPS: 70.57002433765214, Tokens per sec: 174880.2942386381, Loss: 3.4798424243927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9794 , TFLOPS: 68.71637073422244, Tokens per sec: 170286.73641253792, Loss: 3.4856069087982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9795 , TFLOPS: 67.91045098904941, Tokens per sec: 168289.5785046103, Loss: 3.485231876373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9796 , TFLOPS: 68.70291249592198, Tokens per sec: 170253.38541548207, Loss: 3.4828310012817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9797 , TFLOPS: 68.15960293785423, Tokens per sec: 168907.00448010158, Loss: 3.4916272163391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9798 , TFLOPS: 68.70165808846558, Tokens per sec: 170250.2768556207, Loss: 3.496222734451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9799 , TFLOPS: 69.68830293240953, Tokens per sec: 172695.29146681583, Loss: 3.4967055320739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9800 , TFLOPS: 68.60192958325115, Tokens per sec: 170003.1386336976, Loss: 3.4913768768310547 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/model.pt +[rank0]:[2024-08-25 17:26:00,079] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006314181999186985, 'preprocessing_with_comm': 0.0013921880017733201, 'state_converting': 0.9999666469957447, : 1.0090634640073404}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0007800 + eval ppl=23.77872657775879, eval loss=3.1687912940979004 +------------------------------------------------------------------ +iteration: 9801 , TFLOPS: 63.645621092211904, Tokens per sec: 157720.86021045, Loss: 3.502394914627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9802 , TFLOPS: 69.5321508594117, Tokens per sec: 172308.32942835634, Loss: 3.4830689430236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9803 , TFLOPS: 69.26543878128935, Tokens per sec: 171647.38751800833, Loss: 3.516775131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9804 , TFLOPS: 68.02612955356601, Tokens per sec: 168576.2427305283, Loss: 3.49930477142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9805 , TFLOPS: 68.78885841546517, Tokens per sec: 170466.36887183474, Loss: 3.485025644302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9806 , TFLOPS: 69.7063916364367, Tokens per sec: 172740.11726803042, Loss: 3.5061378479003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9807 , TFLOPS: 67.74192772493845, Tokens per sec: 167871.9592917733, Loss: 3.502164125442505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9808 , TFLOPS: 68.64982804541731, Tokens per sec: 170121.83629939656, Loss: 3.5112104415893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9809 , TFLOPS: 69.63970167949034, Tokens per sec: 172574.8522082121, Loss: 3.487168550491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9810 , TFLOPS: 69.59006027903415, Tokens per sec: 172451.83534942995, Loss: 3.4902539253234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9811 , TFLOPS: 67.82021568938492, Tokens per sec: 168065.96549180287, Loss: 3.4864144325256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9812 , TFLOPS: 67.94826583955525, Tokens per sec: 168383.28786981347, Loss: 3.5022523403167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9813 , TFLOPS: 70.5671347935202, Tokens per sec: 174873.13362996955, Loss: 3.5036959648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9814 , TFLOPS: 68.49564478336409, Tokens per sec: 169739.75319134022, Loss: 3.481536865234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9815 , TFLOPS: 67.86423043817487, Tokens per sec: 168175.0388879292, Loss: 3.5107457637786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9816 , TFLOPS: 69.03101642198409, Tokens per sec: 171066.4630878952, Loss: 3.4884209632873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9817 , TFLOPS: 69.59002084593294, Tokens per sec: 172451.73762986125, Loss: 3.488715887069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9818 , TFLOPS: 65.54831824815717, Tokens per sec: 162435.953362277, Loss: 3.5017566680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9819 , TFLOPS: 68.75339557138001, Tokens per sec: 170378.4880376369, Loss: 3.5200035572052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9820 , TFLOPS: 69.57736551602068, Tokens per sec: 172420.376328239, Loss: 3.516062021255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9821 , TFLOPS: 68.0393968461619, Tokens per sec: 168609.1205430939, Loss: 3.4686279296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9822 , TFLOPS: 68.6663483136405, Tokens per sec: 170162.77534391137, Loss: 3.486158847808838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9823 , TFLOPS: 70.4952911647425, Tokens per sec: 174695.09720362938, Loss: 3.520895004272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9824 , TFLOPS: 68.69527545429443, Tokens per sec: 170234.45998503943, Loss: 3.5225138664245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9825 , TFLOPS: 67.17863837298464, Tokens per sec: 166476.06622618713, Loss: 3.503863573074341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9826 , TFLOPS: 69.64752815810132, Tokens per sec: 172594.2470843679, Loss: 3.528153419494629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9827 , TFLOPS: 69.57112591900781, Tokens per sec: 172404.91391949277, Loss: 3.498081684112549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9828 , TFLOPS: 68.22758897817579, Tokens per sec: 169075.481377299, Loss: 3.497483730316162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9829 , TFLOPS: 68.601370967984, Tokens per sec: 170001.75432352905, Loss: 3.486903429031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9830 , TFLOPS: 70.58848885568526, Tokens per sec: 174926.0513483587, Loss: 3.472747802734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9831 , TFLOPS: 68.06675377651679, Tokens per sec: 168676.9140301279, Loss: 3.4845948219299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9832 , TFLOPS: 68.81340674821341, Tokens per sec: 170527.20234460695, Loss: 3.5141713619232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9833 , TFLOPS: 68.95972064697331, Tokens per sec: 170889.78430354048, Loss: 3.4995670318603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9834 , TFLOPS: 68.60820639246214, Tokens per sec: 170018.69325837007, Loss: 3.492612838745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9835 , TFLOPS: 68.81620038399174, Tokens per sec: 170534.12528181003, Loss: 3.494964838027954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9836 , TFLOPS: 68.79885054212609, Tokens per sec: 170491.1304624241, Loss: 3.483236312866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9837 , TFLOPS: 67.98708722012154, Tokens per sec: 168479.49152738482, Loss: 3.511463165283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9838 , TFLOPS: 67.83855744169534, Tokens per sec: 168111.4183745389, Loss: 3.483793020248413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9839 , TFLOPS: 68.77362711234231, Tokens per sec: 170428.62402482974, Loss: 3.483680486679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9840 , TFLOPS: 68.59216394915529, Tokens per sec: 169978.93831663465, Loss: 3.499783992767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9841 , TFLOPS: 68.97418636669947, Tokens per sec: 170925.6319505525, Loss: 3.5006051063537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9842 , TFLOPS: 68.41725399333068, Tokens per sec: 169545.49217817903, Loss: 3.5073230266571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9843 , TFLOPS: 70.50967579116092, Tokens per sec: 174730.74389249162, Loss: 3.5040676593780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9844 , TFLOPS: 67.87725296969356, Tokens per sec: 168207.31015558282, Loss: 3.4826576709747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9845 , TFLOPS: 67.91871463491469, Tokens per sec: 168310.05672938022, Loss: 3.5169458389282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9846 , TFLOPS: 69.65980472065009, Tokens per sec: 172624.66975873875, Loss: 3.4993531703948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9847 , TFLOPS: 69.54509926407184, Tokens per sec: 172340.41700148868, Loss: 3.4840879440307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9848 , TFLOPS: 67.70696709635816, Tokens per sec: 167785.32300292017, Loss: 3.4895215034484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9849 , TFLOPS: 69.55798527737834, Tokens per sec: 172372.34996197376, Loss: 3.5156190395355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9850 , TFLOPS: 70.51603146923729, Tokens per sec: 174746.4939629006, Loss: 3.51383376121521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9851 , TFLOPS: 67.76173065603663, Tokens per sec: 167921.03313650755, Loss: 3.513428211212158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9852 , TFLOPS: 67.9277309478559, Tokens per sec: 168332.40014607742, Loss: 3.5055317878723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9853 , TFLOPS: 69.70708539186279, Tokens per sec: 172741.83647039972, Loss: 3.493408441543579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9854 , TFLOPS: 68.74897842541351, Tokens per sec: 170367.54186334315, Loss: 3.4967243671417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9855 , TFLOPS: 67.9686593322311, Tokens per sec: 168433.82519119288, Loss: 3.480229139328003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9856 , TFLOPS: 68.77728103163226, Tokens per sec: 170437.67883934264, Loss: 3.48917818069458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9857 , TFLOPS: 69.57876359994637, Tokens per sec: 172423.84093421852, Loss: 3.5043222904205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9858 , TFLOPS: 68.79193571191136, Tokens per sec: 170473.99475142057, Loss: 3.4728238582611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9859 , TFLOPS: 68.01993624682504, Tokens per sec: 168560.8950341751, Loss: 3.513423442840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9860 , TFLOPS: 69.6804550079958, Tokens per sec: 172675.84344559838, Loss: 3.5080957412719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9861 , TFLOPS: 69.80445196906611, Tokens per sec: 172983.12157452345, Loss: 3.5067832469940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9862 , TFLOPS: 68.22362867701004, Tokens per sec: 169065.66731474712, Loss: 3.501037120819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9863 , TFLOPS: 70.58590511446523, Tokens per sec: 174919.6485529929, Loss: 3.5173046588897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9864 , TFLOPS: 69.48980618596977, Tokens per sec: 172203.3946629164, Loss: 3.4714765548706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9865 , TFLOPS: 68.21647684983496, Tokens per sec: 169047.94429916845, Loss: 3.4868221282958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9866 , TFLOPS: 68.1519926200262, Tokens per sec: 168888.14527417804, Loss: 3.505716562271118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9867 , TFLOPS: 69.67792022722718, Tokens per sec: 172669.5619796246, Loss: 3.471292018890381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9868 , TFLOPS: 67.8996750390819, Tokens per sec: 168262.87451352197, Loss: 3.480285167694092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9869 , TFLOPS: 69.5363131436409, Tokens per sec: 172318.64402718976, Loss: 3.5133795738220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9870 , TFLOPS: 69.41867808896242, Tokens per sec: 172027.13140312547, Loss: 3.4957284927368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9871 , TFLOPS: 67.80411519912785, Tokens per sec: 168026.06670333043, Loss: 3.5107457637786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9872 , TFLOPS: 68.78056140514764, Tokens per sec: 170445.80796627674, Loss: 3.51375412940979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9873 , TFLOPS: 69.6308074075893, Tokens per sec: 172552.8112226564, Loss: 3.4889111518859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9874 , TFLOPS: 67.92142022124024, Tokens per sec: 168316.76147033987, Loss: 3.4964632987976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9875 , TFLOPS: 68.84930816339289, Tokens per sec: 170616.1700062891, Loss: 3.519343137741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9876 , TFLOPS: 68.79690795459639, Tokens per sec: 170486.31651653184, Loss: 3.511885643005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9877 , TFLOPS: 68.56914279279782, Tokens per sec: 169921.88935519115, Loss: 3.4885425567626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9878 , TFLOPS: 68.82117660023661, Tokens per sec: 170546.4568938403, Loss: 3.4951694011688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9879 , TFLOPS: 67.89579152937497, Tokens per sec: 168253.2507486643, Loss: 3.471982002258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9880 , TFLOPS: 70.59460329857582, Tokens per sec: 174941.203611403, Loss: 3.4882009029388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9881 , TFLOPS: 68.14620926003151, Tokens per sec: 168873.81347102148, Loss: 3.493781089782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9882 , TFLOPS: 68.87430798396647, Tokens per sec: 170678.122315628, Loss: 3.500293493270874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9883 , TFLOPS: 69.91179007251513, Tokens per sec: 173249.11721913837, Loss: 3.5281076431274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9884 , TFLOPS: 69.55211541376255, Tokens per sec: 172357.803793315, Loss: 3.4952914714813232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9885 , TFLOPS: 67.7256555803228, Tokens per sec: 167831.63512489098, Loss: 3.5040736198425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9886 , TFLOPS: 68.8741578529357, Tokens per sec: 170677.75027439542, Loss: 3.4957950115203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9887 , TFLOPS: 69.55483343592397, Tokens per sec: 172364.53935164583, Loss: 3.488389730453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9888 , TFLOPS: 68.78296604656491, Tokens per sec: 170451.7669326019, Loss: 3.502328395843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9889 , TFLOPS: 68.77448587356879, Tokens per sec: 170430.75212974934, Loss: 3.5060315132141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9890 , TFLOPS: 70.60061294257468, Tokens per sec: 174956.09617124914, Loss: 3.4873874187469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9891 , TFLOPS: 67.94514587720192, Tokens per sec: 168375.55625940967, Loss: 3.5093653202056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9892 , TFLOPS: 68.67983331900385, Tokens per sec: 170196.19267269774, Loss: 3.5182018280029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9893 , TFLOPS: 69.68658080009948, Tokens per sec: 172691.02383324297, Loss: 3.5029847621917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9894 , TFLOPS: 68.79463552226561, Tokens per sec: 170480.68517888704, Loss: 3.522458076477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9895 , TFLOPS: 68.92524048026524, Tokens per sec: 170804.33865213397, Loss: 3.500016689300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9896 , TFLOPS: 68.57257146813556, Tokens per sec: 169930.38599037784, Loss: 3.504901647567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9897 , TFLOPS: 70.55811108095521, Tokens per sec: 174850.77187618954, Loss: 3.4900338649749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9898 , TFLOPS: 68.06614588629503, Tokens per sec: 168675.40761119378, Loss: 3.4875648021698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9899 , TFLOPS: 67.86030421267397, Tokens per sec: 168165.30926862854, Loss: 3.4950406551361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9900 , TFLOPS: 69.8694081963623, Tokens per sec: 173144.09026128254, Loss: 3.4995908737182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9901 , TFLOPS: 68.8196096023915, Tokens per sec: 170542.57370056104, Loss: 3.502807855606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9902 , TFLOPS: 68.78783054833333, Tokens per sec: 170463.82170385311, Loss: 3.507627487182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9903 , TFLOPS: 68.71222074761518, Tokens per sec: 170276.4522885664, Loss: 3.492340087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9904 , TFLOPS: 69.50543572298376, Tokens per sec: 172242.1263773727, Loss: 3.5106422901153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9905 , TFLOPS: 67.95773498271954, Tokens per sec: 168406.7534496868, Loss: 3.4781389236450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9906 , TFLOPS: 68.84353205817767, Tokens per sec: 170601.85618127525, Loss: 3.5101730823516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9907 , TFLOPS: 68.74256547657644, Tokens per sec: 170351.64986968026, Loss: 3.4936881065368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9908 , TFLOPS: 68.80248630928742, Tokens per sec: 170500.14029396267, Loss: 3.4943950176239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9909 , TFLOPS: 68.848649999468, Tokens per sec: 170614.53900358043, Loss: 3.491574287414551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9910 , TFLOPS: 70.55640611440762, Tokens per sec: 174846.546781267, Loss: 3.492500066757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9911 , TFLOPS: 67.68161501436005, Tokens per sec: 167722.49775096646, Loss: 3.5077855587005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9912 , TFLOPS: 68.03912591606458, Tokens per sec: 168608.44914846728, Loss: 3.4801900386810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9913 , TFLOPS: 69.72879461967676, Tokens per sec: 172795.63432839164, Loss: 3.5181424617767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9914 , TFLOPS: 68.73676751885701, Tokens per sec: 170337.28189175372, Loss: 3.456864356994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9915 , TFLOPS: 68.32194151953568, Tokens per sec: 169309.29736857928, Loss: 3.4892959594726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9916 , TFLOPS: 68.80477909320109, Tokens per sec: 170505.82206507178, Loss: 3.497718334197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9917 , TFLOPS: 70.5918919773715, Tokens per sec: 174934.48465878202, Loss: 3.483287811279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9918 , TFLOPS: 67.64335229253142, Tokens per sec: 167627.67851128834, Loss: 3.46225905418396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9919 , TFLOPS: 68.67152256662625, Tokens per sec: 170175.5977128011, Loss: 3.4900197982788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9920 , TFLOPS: 69.71779875571578, Tokens per sec: 172768.38536620187, Loss: 3.5067837238311768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9921 , TFLOPS: 68.77074937127811, Tokens per sec: 170421.4926654633, Loss: 3.502584934234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9922 , TFLOPS: 67.991460369727, Tokens per sec: 168490.32867384938, Loss: 3.4993138313293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9923 , TFLOPS: 69.66473443075317, Tokens per sec: 172636.88612342658, Loss: 3.5019748210906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9924 , TFLOPS: 69.556627743243, Tokens per sec: 172368.98584284165, Loss: 3.497248888015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9925 , TFLOPS: 68.91715012766495, Tokens per sec: 170784.28986716457, Loss: 3.4859230518341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9926 , TFLOPS: 67.12211800435058, Tokens per sec: 166336.00252647934, Loss: 3.4896528720855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9927 , TFLOPS: 70.54532828332526, Tokens per sec: 174819.0946955211, Loss: 3.4853265285491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9928 , TFLOPS: 68.93674135912445, Tokens per sec: 170832.83909687353, Loss: 3.492802143096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9929 , TFLOPS: 67.2180700190117, Tokens per sec: 166573.78218879658, Loss: 3.4738404750823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9930 , TFLOPS: 70.54290042900189, Tokens per sec: 174813.07820505754, Loss: 3.503453016281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9931 , TFLOPS: 69.1630785901111, Tokens per sec: 171393.7276883627, Loss: 3.5040526390075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9932 , TFLOPS: 68.2337004986905, Tokens per sec: 169090.626398374, Loss: 3.4818131923675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9933 , TFLOPS: 68.03664510526298, Tokens per sec: 168602.30142601705, Loss: 3.501107692718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9934 , TFLOPS: 69.57510619532454, Tokens per sec: 172414.77748266922, Loss: 3.5209238529205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9935 , TFLOPS: 68.07795372735303, Tokens per sec: 168704.66874207655, Loss: 3.4825589656829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9936 , TFLOPS: 67.98158875268271, Tokens per sec: 168465.86571936522, Loss: 3.5010359287261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9937 , TFLOPS: 70.55505383137788, Tokens per sec: 174843.19567495302, Loss: 3.4860029220581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9938 , TFLOPS: 67.78797901508374, Tokens per sec: 167986.07946172173, Loss: 3.501922130584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9939 , TFLOPS: 68.67052169052577, Tokens per sec: 170173.1174315685, Loss: 3.4908595085144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9940 , TFLOPS: 68.66457776343009, Tokens per sec: 170158.38772544777, Loss: 3.4894418716430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9941 , TFLOPS: 68.83742063178556, Tokens per sec: 170586.7113934464, Loss: 3.516176700592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9942 , TFLOPS: 68.81596284811546, Tokens per sec: 170533.5366417419, Loss: 3.4887542724609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9943 , TFLOPS: 69.652212795755, Tokens per sec: 172605.85613252918, Loss: 3.504983425140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9944 , TFLOPS: 68.72448823463773, Tokens per sec: 170306.85247278356, Loss: 3.4774813652038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9945 , TFLOPS: 68.68987632844866, Tokens per sec: 170221.0803564308, Loss: 3.4850637912750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9946 , TFLOPS: 67.75427687555889, Tokens per sec: 167902.561847382, Loss: 3.5030806064605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9947 , TFLOPS: 70.50363590393508, Tokens per sec: 174715.77638659748, Loss: 3.479780673980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9948 , TFLOPS: 67.8085530688651, Tokens per sec: 168037.06423341105, Loss: 3.4805028438568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9949 , TFLOPS: 68.74465941862687, Tokens per sec: 170356.83888874797, Loss: 3.5109875202178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9950 , TFLOPS: 69.9393500960611, Tokens per sec: 173317.4139934725, Loss: 3.4925410747528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9951 , TFLOPS: 68.58010407017125, Tokens per sec: 169949.05260800687, Loss: 3.501758575439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9952 , TFLOPS: 67.8041212167748, Tokens per sec: 168026.08161572248, Loss: 3.518359661102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9953 , TFLOPS: 68.85878134015809, Tokens per sec: 170639.64558186982, Loss: 3.488501787185669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9954 , TFLOPS: 69.59880093763442, Tokens per sec: 172473.49566430444, Loss: 3.499328136444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9955 , TFLOPS: 67.1069431034442, Tokens per sec: 166298.39745038, Loss: 3.49682354927063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9956 , TFLOPS: 68.92017903556152, Tokens per sec: 170791.795834594, Loss: 3.5024497509002686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9957 , TFLOPS: 70.51028846383252, Tokens per sec: 174732.26216286334, Loss: 3.4811854362487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9958 , TFLOPS: 67.84300223798375, Tokens per sec: 168122.43306937633, Loss: 3.4901206493377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9959 , TFLOPS: 67.7479292330612, Tokens per sec: 167886.83169002214, Loss: 3.485999584197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9960 , TFLOPS: 69.60619695177161, Tokens per sec: 172491.82380207462, Loss: 3.4852004051208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9961 , TFLOPS: 68.81851780495792, Tokens per sec: 170539.86810624923, Loss: 3.4778404235839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9962 , TFLOPS: 68.76594395650923, Tokens per sec: 170409.5843183059, Loss: 3.496670961380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9963 , TFLOPS: 68.56774977746755, Tokens per sec: 169918.43730974942, Loss: 3.5055503845214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9964 , TFLOPS: 70.5745922712409, Tokens per sec: 174891.61408127047, Loss: 3.4871103763580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9965 , TFLOPS: 68.13177141296062, Tokens per sec: 168838.03489551964, Loss: 3.5117619037628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9966 , TFLOPS: 68.63886748226976, Tokens per sec: 170094.67481651122, Loss: 3.510512590408325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9967 , TFLOPS: 69.8721342733066, Tokens per sec: 173150.8457802527, Loss: 3.484814167022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9968 , TFLOPS: 68.82540770325177, Tokens per sec: 170556.94203320637, Loss: 3.4659335613250732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9969 , TFLOPS: 68.67768824112245, Tokens per sec: 170190.87693340782, Loss: 3.49617862701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9970 , TFLOPS: 68.79526292185982, Tokens per sec: 170482.2399441953, Loss: 3.4848406314849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9971 , TFLOPS: 69.63206939848594, Tokens per sec: 172555.93857511805, Loss: 3.4989242553710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9972 , TFLOPS: 67.85084160880233, Tokens per sec: 168141.85989384272, Loss: 3.504152297973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9973 , TFLOPS: 67.7203001956404, Tokens per sec: 167818.3638917037, Loss: 3.498328924179077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9974 , TFLOPS: 69.61411242300375, Tokens per sec: 172511.439211749, Loss: 3.5034611225128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9975 , TFLOPS: 68.83638303420472, Tokens per sec: 170584.14011234025, Loss: 3.510185718536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9976 , TFLOPS: 68.24885984734588, Tokens per sec: 169128.19293428218, Loss: 3.5087642669677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9977 , TFLOPS: 69.64831402420117, Tokens per sec: 172596.1945471336, Loss: 3.4885895252227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9978 , TFLOPS: 67.88532619772313, Tokens per sec: 168227.31650397956, Loss: 3.5127170085906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9979 , TFLOPS: 69.14156091913675, Tokens per sec: 171340.4045293211, Loss: 3.5031280517578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9980 , TFLOPS: 68.96399804343075, Tokens per sec: 170900.38416315612, Loss: 3.4917263984680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9981 , TFLOPS: 68.59241453439951, Tokens per sec: 169979.5592944741, Loss: 3.489513635635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9982 , TFLOPS: 68.27587288037917, Tokens per sec: 169195.1342058699, Loss: 3.51133131980896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9983 , TFLOPS: 69.71704376433381, Tokens per sec: 172766.5144143882, Loss: 3.5028913021087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9984 , TFLOPS: 70.58695893292425, Tokens per sec: 174922.26003122205, Loss: 3.4831178188323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9985 , TFLOPS: 67.22118878788895, Tokens per sec: 166581.51084163517, Loss: 3.515489101409912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9986 , TFLOPS: 68.67155971849347, Tokens per sec: 170175.68977922067, Loss: 3.4967379570007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9987 , TFLOPS: 69.6538067597972, Tokens per sec: 172609.8061510154, Loss: 3.5020952224731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9988 , TFLOPS: 67.59162471051765, Tokens per sec: 167499.4918647947, Loss: 3.476196050643921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9989 , TFLOPS: 68.73215777475774, Tokens per sec: 170325.8584380705, Loss: 3.4957141876220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9990 , TFLOPS: 69.65460610127427, Tokens per sec: 172611.78700725024, Loss: 3.4825897216796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9991 , TFLOPS: 69.68442925871662, Tokens per sec: 172685.6920766844, Loss: 3.499551773071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9992 , TFLOPS: 67.9159311415329, Tokens per sec: 168303.15892615303, Loss: 3.486663341522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9993 , TFLOPS: 68.2391352047559, Tokens per sec: 169104.09419868636, Loss: 3.505841016769409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9994 , TFLOPS: 69.57989400982937, Tokens per sec: 172426.64221443352, Loss: 3.5012385845184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9995 , TFLOPS: 69.66468671812592, Tokens per sec: 172636.76788628026, Loss: 3.5118117332458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9996 , TFLOPS: 68.03114674923587, Tokens per sec: 168588.67589408794, Loss: 3.485245704650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9997 , TFLOPS: 69.90166758546283, Tokens per sec: 173224.03258113874, Loss: 3.499608278274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9998 , TFLOPS: 69.59607796861977, Tokens per sec: 172466.74784712636, Loss: 3.508718252182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9999 , TFLOPS: 68.26493551273, Tokens per sec: 169168.03020398476, Loss: 3.4850411415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10000 , TFLOPS: 68.67429725570412, Tokens per sec: 170182.47369799326, Loss: 3.498141050338745 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/model.pt +[rank0]:[2024-08-25 17:51:45,517] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006320098007563502, 'preprocessing_with_comm': 0.0014538260002154857, 'state_converting': 0.9656076120008947, : 0.974771277004038}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000, took 4.24s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008000 + eval ppl=23.972841262817383, eval loss=3.1769216060638428 +------------------------------------------------------------------ +iteration: 10001 , TFLOPS: 64.96329228357303, Tokens per sec: 160986.19457610731, Loss: 3.497192859649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10002 , TFLOPS: 68.12166904699906, Tokens per sec: 168813.00011979943, Loss: 3.479804277420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10003 , TFLOPS: 67.48650033742484, Tokens per sec: 167238.982087865, Loss: 3.518430709838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10004 , TFLOPS: 70.57694605314649, Tokens per sec: 174897.44701212295, Loss: 3.5090491771698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10005 , TFLOPS: 67.967435572681, Tokens per sec: 168430.7925802195, Loss: 3.476916551589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10006 , TFLOPS: 68.75499374384715, Tokens per sec: 170382.44848506377, Loss: 3.4982337951660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10007 , TFLOPS: 69.44270518706712, Tokens per sec: 172086.67319903226, Loss: 3.4850854873657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10008 , TFLOPS: 68.78686024009731, Tokens per sec: 170461.417173156, Loss: 3.4937751293182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10009 , TFLOPS: 69.56376367913535, Tokens per sec: 172386.66947807113, Loss: 3.504469394683838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10010 , TFLOPS: 69.55489677925354, Tokens per sec: 172364.69632339437, Loss: 3.5082955360412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10011 , TFLOPS: 68.70910413932178, Tokens per sec: 170268.7289898926, Loss: 3.5004215240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10012 , TFLOPS: 68.89038804849207, Tokens per sec: 170717.9704868878, Loss: 3.4728312492370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10013 , TFLOPS: 68.24590976724313, Tokens per sec: 169120.8823108094, Loss: 3.4840340614318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10014 , TFLOPS: 68.73441290142505, Tokens per sec: 170331.4468903748, Loss: 3.4871504306793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10015 , TFLOPS: 69.73736973084348, Tokens per sec: 172816.88439848923, Loss: 3.506269931793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10016 , TFLOPS: 67.23917085697362, Tokens per sec: 166626.0723897154, Loss: 3.503818988800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10017 , TFLOPS: 70.51629065422713, Tokens per sec: 174747.13625185704, Loss: 3.500011920928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10018 , TFLOPS: 67.74312880650992, Tokens per sec: 167874.93570421744, Loss: 3.4921276569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10019 , TFLOPS: 68.7458579551045, Tokens per sec: 170359.8089941692, Loss: 3.486832857131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10020 , TFLOPS: 69.20052989842057, Tokens per sec: 171486.53615595575, Loss: 3.50211501121521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10021 , TFLOPS: 68.8527858411771, Tokens per sec: 170624.78807493616, Loss: 3.50227427482605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10022 , TFLOPS: 67.71238213612222, Tokens per sec: 167798.74206797135, Loss: 3.4792511463165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10023 , TFLOPS: 68.77891659118151, Tokens per sec: 170441.73193607747, Loss: 3.493726968765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10024 , TFLOPS: 70.57980768426093, Tokens per sec: 174904.53844925924, Loss: 3.48576283454895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10025 , TFLOPS: 66.27311431067984, Tokens per sec: 164232.07784808567, Loss: 3.5010507106781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10026 , TFLOPS: 68.83802958642642, Tokens per sec: 170588.2204501283, Loss: 3.5189132690429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10027 , TFLOPS: 69.972988705111, Tokens per sec: 173400.77417229686, Loss: 3.473970890045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10028 , TFLOPS: 68.66730329080463, Tokens per sec: 170165.14188252352, Loss: 3.512770652770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10029 , TFLOPS: 68.57048466590562, Tokens per sec: 169925.21466457177, Loss: 3.4836206436157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10030 , TFLOPS: 68.8179423642922, Tokens per sec: 170538.44210088952, Loss: 3.482778549194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10031 , TFLOPS: 69.52669740757536, Tokens per sec: 172294.81517395854, Loss: 3.5003559589385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10032 , TFLOPS: 68.92960304635876, Tokens per sec: 170815.14957149007, Loss: 3.5002541542053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10033 , TFLOPS: 67.11789337120612, Tokens per sec: 166325.53342016565, Loss: 3.482732057571411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10034 , TFLOPS: 70.62533475303178, Tokens per sec: 175017.35954089405, Loss: 3.4994702339172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10035 , TFLOPS: 69.02312036290705, Tokens per sec: 171046.89578367956, Loss: 3.4680941104888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10036 , TFLOPS: 66.82557948122368, Tokens per sec: 165601.14739975592, Loss: 3.500960111618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10037 , TFLOPS: 69.89160762312234, Tokens per sec: 173199.1028862627, Loss: 3.5067741870880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10038 , TFLOPS: 68.7414745246453, Tokens per sec: 170348.94637061658, Loss: 3.4981448650360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10039 , TFLOPS: 68.78456420898074, Tokens per sec: 170455.72735512038, Loss: 3.5054280757904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10040 , TFLOPS: 68.07616338188404, Tokens per sec: 168700.2320687816, Loss: 3.495274066925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10041 , TFLOPS: 69.51405930081175, Tokens per sec: 172263.49655319596, Loss: 3.4835100173950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10042 , TFLOPS: 69.1477234453235, Tokens per sec: 171355.6759480702, Loss: 3.483534812927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10043 , TFLOPS: 68.2392798974483, Tokens per sec: 169104.45276311733, Loss: 3.508944034576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10044 , TFLOPS: 69.59133096496417, Tokens per sec: 172454.98424914214, Loss: 3.4918105602264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10045 , TFLOPS: 67.83443816958714, Tokens per sec: 168101.2103644795, Loss: 3.485537528991699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10046 , TFLOPS: 68.81747240447561, Tokens per sec: 170537.2774886937, Loss: 3.5105700492858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10047 , TFLOPS: 69.70265568450213, Tokens per sec: 172730.85916758748, Loss: 3.478477716445923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10048 , TFLOPS: 67.90153375897916, Tokens per sec: 168267.48062618222, Loss: 3.4960527420043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10049 , TFLOPS: 68.4565236490202, Tokens per sec: 169642.80671088738, Loss: 3.4912900924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10050 , TFLOPS: 68.91823713520054, Tokens per sec: 170786.98359158152, Loss: 3.518831491470337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10051 , TFLOPS: 69.68723810370099, Tokens per sec: 172692.6527039755, Loss: 3.4891791343688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10052 , TFLOPS: 68.05227847647606, Tokens per sec: 168641.04264204105, Loss: 3.473884105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10053 , TFLOPS: 68.01563062079714, Tokens per sec: 168550.22521857414, Loss: 3.4940006732940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10054 , TFLOPS: 70.48091360729919, Tokens per sec: 174659.46803246616, Loss: 3.527491807937622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10055 , TFLOPS: 67.8378167090134, Tokens per sec: 168109.5827573546, Loss: 3.4936604499816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10056 , TFLOPS: 68.72324647366952, Tokens per sec: 170303.77525231303, Loss: 3.5058069229125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10057 , TFLOPS: 68.97886894085971, Tokens per sec: 170937.2358851497, Loss: 3.4999866485595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10058 , TFLOPS: 69.44581558431365, Tokens per sec: 172094.3811060477, Loss: 3.4896240234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10059 , TFLOPS: 67.89900714501991, Tokens per sec: 168261.21939846175, Loss: 3.5019960403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10060 , TFLOPS: 68.8414323927789, Tokens per sec: 170596.65297911986, Loss: 3.5007505416870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10061 , TFLOPS: 69.59671836973291, Tokens per sec: 172468.33483163075, Loss: 3.4797067642211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10062 , TFLOPS: 68.84913026063599, Tokens per sec: 170615.72914365988, Loss: 3.4837393760681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10063 , TFLOPS: 67.04681140254435, Tokens per sec: 166149.38447149648, Loss: 3.4935977458953857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10064 , TFLOPS: 70.59546995820969, Tokens per sec: 174943.35128944617, Loss: 3.495032548904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10065 , TFLOPS: 68.22992770140921, Tokens per sec: 169081.27699110305, Loss: 3.4964382648468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10066 , TFLOPS: 67.29321002234992, Tokens per sec: 166759.98739442337, Loss: 3.4917945861816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10067 , TFLOPS: 69.62904721013743, Tokens per sec: 172548.44925946888, Loss: 3.4755678176879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10068 , TFLOPS: 68.70231832154138, Tokens per sec: 170251.91298591322, Loss: 3.4827141761779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10069 , TFLOPS: 68.81355291178276, Tokens per sec: 170527.56455403304, Loss: 3.4861607551574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10070 , TFLOPS: 68.66174391352533, Tokens per sec: 170151.36513320945, Loss: 3.514907121658325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10071 , TFLOPS: 70.59591308109758, Tokens per sec: 174944.4493967764, Loss: 3.4887988567352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10072 , TFLOPS: 67.87377548142443, Tokens per sec: 168198.6925565749, Loss: 3.4904749393463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10073 , TFLOPS: 66.94864583400849, Tokens per sec: 165906.11937883453, Loss: 3.5004215240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10074 , TFLOPS: 70.54218577316611, Tokens per sec: 174811.30720917066, Loss: 3.492218494415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10075 , TFLOPS: 68.75081520043138, Tokens per sec: 170372.0935941749, Loss: 3.514310598373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10076 , TFLOPS: 68.6346229449871, Tokens per sec: 170084.1563855504, Loss: 3.49245285987854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10077 , TFLOPS: 68.03290796756083, Tokens per sec: 168593.04038711142, Loss: 3.5028605461120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10078 , TFLOPS: 69.60932869347329, Tokens per sec: 172499.58460300122, Loss: 3.4842610359191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10079 , TFLOPS: 68.84773547938521, Tokens per sec: 170612.27272207258, Loss: 3.5274391174316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10080 , TFLOPS: 68.10495628804313, Tokens per sec: 168771.58406204425, Loss: 3.4951345920562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10081 , TFLOPS: 69.50322859951746, Tokens per sec: 172236.65688228846, Loss: 3.5060014724731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10082 , TFLOPS: 68.97391877766836, Tokens per sec: 170924.968835456, Loss: 3.505066394805908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10083 , TFLOPS: 67.37484908136605, Tokens per sec: 166962.29797595015, Loss: 3.503488302230835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10084 , TFLOPS: 70.59111918772363, Tokens per sec: 174932.5696009048, Loss: 3.499107599258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10085 , TFLOPS: 68.02786205153924, Tokens per sec: 168580.5360513564, Loss: 3.4811668395996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10086 , TFLOPS: 68.81858522604635, Tokens per sec: 170540.0351831334, Loss: 3.493809461593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10087 , TFLOPS: 69.06817787431014, Tokens per sec: 171158.55326043107, Loss: 3.4636685848236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10088 , TFLOPS: 69.56354142827269, Tokens per sec: 172386.1187159505, Loss: 3.4981608390808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10089 , TFLOPS: 68.15965003245137, Tokens per sec: 168907.1211857012, Loss: 3.518594980239868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10090 , TFLOPS: 68.93444596546651, Tokens per sec: 170827.15085853063, Loss: 3.5323667526245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10091 , TFLOPS: 70.59468867497431, Tokens per sec: 174941.41518352347, Loss: 3.5009281635284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10092 , TFLOPS: 66.993851472154, Tokens per sec: 166018.14392997127, Loss: 3.5010836124420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10093 , TFLOPS: 68.20258600853724, Tokens per sec: 169013.5212642302, Loss: 3.4791226387023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10094 , TFLOPS: 69.97921498212857, Tokens per sec: 173416.2035726274, Loss: 3.5087509155273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10095 , TFLOPS: 68.77704153082873, Tokens per sec: 170437.0853299683, Loss: 3.5036559104919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10096 , TFLOPS: 67.83750986652457, Tokens per sec: 168108.8223678658, Loss: 3.5005526542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10097 , TFLOPS: 68.83762774160788, Tokens per sec: 170587.2246344011, Loss: 3.503953218460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10098 , TFLOPS: 69.65628797030774, Tokens per sec: 172615.9548639884, Loss: 3.495512008666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10099 , TFLOPS: 67.36138517338323, Tokens per sec: 166928.93292879677, Loss: 3.494919776916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10100 , TFLOPS: 68.09261188288751, Tokens per sec: 168740.99326622015, Loss: 3.483473539352417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10101 , TFLOPS: 70.57047157168283, Tokens per sec: 174881.402533833, Loss: 3.4956886768341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10102 , TFLOPS: 68.88870990350935, Tokens per sec: 170713.81185875766, Loss: 3.5186898708343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10103 , TFLOPS: 66.91173856035276, Tokens per sec: 165814.65908904382, Loss: 3.475832223892212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10104 , TFLOPS: 70.01472965255029, Tokens per sec: 173504.21283819634, Loss: 3.5080368518829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10105 , TFLOPS: 68.75914286219808, Tokens per sec: 170392.73045740047, Loss: 3.4887800216674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10106 , TFLOPS: 68.76796375656005, Tokens per sec: 170414.58960533113, Loss: 3.482726812362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10107 , TFLOPS: 68.68868592779131, Tokens per sec: 170218.13041246842, Loss: 3.4991257190704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10108 , TFLOPS: 69.5917312844408, Tokens per sec: 172455.97628490406, Loss: 3.5050182342529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10109 , TFLOPS: 67.9428293037888, Tokens per sec: 168369.81553530035, Loss: 3.476693868637085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10110 , TFLOPS: 67.77483977457803, Tokens per sec: 167953.51897634173, Loss: 3.4820709228515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10111 , TFLOPS: 70.57416243596296, Tokens per sec: 174890.54890210147, Loss: 3.4960262775421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10112 , TFLOPS: 67.78616555348046, Tokens per sec: 167981.58550409946, Loss: 3.5113072395324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10113 , TFLOPS: 68.89454570701159, Tokens per sec: 170728.273622703, Loss: 3.4836814403533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10114 , TFLOPS: 68.78663332722384, Tokens per sec: 170460.85485805903, Loss: 3.485830545425415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10115 , TFLOPS: 68.89719240391311, Tokens per sec: 170734.83242918274, Loss: 3.505007743835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10116 , TFLOPS: 68.79838263991333, Tokens per sec: 170489.9709491972, Loss: 3.5112929344177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10117 , TFLOPS: 68.67181348508839, Tokens per sec: 170176.3186407976, Loss: 3.485743999481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10118 , TFLOPS: 69.71164291111623, Tokens per sec: 172753.13050516194, Loss: 3.4843828678131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10119 , TFLOPS: 68.01854333067055, Tokens per sec: 168557.44323450193, Loss: 3.503953218460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10120 , TFLOPS: 68.06686831842673, Tokens per sec: 168677.19787759858, Loss: 3.498469114303589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10121 , TFLOPS: 70.58773394664397, Tokens per sec: 174924.1806005943, Loss: 3.4864399433135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10122 , TFLOPS: 68.71165131897794, Tokens per sec: 170275.04118167557, Loss: 3.4930295944213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10123 , TFLOPS: 67.70087220331425, Tokens per sec: 167770.21918654945, Loss: 3.499434471130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10124 , TFLOPS: 69.64428307902223, Tokens per sec: 172586.20542093494, Loss: 3.500206470489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10125 , TFLOPS: 68.66892582162372, Tokens per sec: 170169.16269262973, Loss: 3.494905948638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10126 , TFLOPS: 68.27955409036251, Tokens per sec: 169204.25664972738, Loss: 3.505221366882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10127 , TFLOPS: 68.88755684441499, Tokens per sec: 170710.9544513019, Loss: 3.4873147010803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10128 , TFLOPS: 69.62452820857352, Tokens per sec: 172537.2506757844, Loss: 3.490469455718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10129 , TFLOPS: 67.87102596597357, Tokens per sec: 168191.8789544036, Loss: 3.5207505226135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10130 , TFLOPS: 68.10697291074118, Tokens per sec: 168776.58147523933, Loss: 3.4968512058258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10131 , TFLOPS: 70.58310928213527, Tokens per sec: 174912.72017249287, Loss: 3.518878936767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10132 , TFLOPS: 68.78696338352863, Tokens per sec: 170461.67277394095, Loss: 3.4971060752868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10133 , TFLOPS: 68.70084308021292, Tokens per sec: 170248.25717539008, Loss: 3.50105881690979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10134 , TFLOPS: 69.18740612886133, Tokens per sec: 171454.01400928537, Loss: 3.5213072299957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10135 , TFLOPS: 68.6977046751914, Tokens per sec: 170240.4798620229, Loss: 3.496530532836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10136 , TFLOPS: 68.74468916056875, Tokens per sec: 170356.91259255633, Loss: 3.5066685676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10137 , TFLOPS: 67.78996387212945, Tokens per sec: 167990.99815613104, Loss: 3.4995949268341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10138 , TFLOPS: 70.59910924951424, Tokens per sec: 174952.3698541999, Loss: 3.4818692207336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10139 , TFLOPS: 67.94819711305033, Tokens per sec: 168383.11755796341, Loss: 3.508730888366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10140 , TFLOPS: 67.07570306423673, Tokens per sec: 166220.98119184948, Loss: 3.494202136993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10141 , TFLOPS: 70.56323665109662, Tokens per sec: 174863.47360362808, Loss: 3.487335205078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10142 , TFLOPS: 68.7240730843085, Tokens per sec: 170305.82368453513, Loss: 3.485832452774048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10143 , TFLOPS: 68.68366352159077, Tokens per sec: 170205.68433663977, Loss: 3.484102964401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10144 , TFLOPS: 68.13843299819769, Tokens per sec: 168854.5430375713, Loss: 3.5160813331604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10145 , TFLOPS: 69.5425178969663, Tokens per sec: 172334.0200894401, Loss: 3.4889211654663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10146 , TFLOPS: 68.92645656543507, Tokens per sec: 170807.35224514766, Loss: 3.5225155353546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10147 , TFLOPS: 67.25718562670689, Tokens per sec: 166670.71497360515, Loss: 3.475992202758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10148 , TFLOPS: 69.58304211451566, Tokens per sec: 172434.44356463852, Loss: 3.487701654434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10149 , TFLOPS: 69.7301517558652, Tokens per sec: 172798.99746136743, Loss: 3.5116114616394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10150 , TFLOPS: 67.37650491162671, Tokens per sec: 166966.4013057408, Loss: 3.496950626373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10151 , TFLOPS: 69.6355527643516, Tokens per sec: 172564.5707394576, Loss: 3.502955913543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10152 , TFLOPS: 68.30514665399048, Tokens per sec: 169267.67784165108, Loss: 3.4995973110198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10153 , TFLOPS: 68.81036354489748, Tokens per sec: 170519.66095155358, Loss: 3.5205776691436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10154 , TFLOPS: 68.89625081784044, Tokens per sec: 170732.49907517037, Loss: 3.4738855361938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10155 , TFLOPS: 67.7892172825323, Tokens per sec: 167989.14802486566, Loss: 3.4851226806640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10156 , TFLOPS: 68.83590052543975, Tokens per sec: 170582.94440246795, Loss: 3.4863440990448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10157 , TFLOPS: 68.88953982722671, Tokens per sec: 170715.86850115293, Loss: 3.4756736755371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10158 , TFLOPS: 70.61883254244434, Tokens per sec: 175001.24634678077, Loss: 3.513392925262451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10159 , TFLOPS: 67.01386291943264, Tokens per sec: 166067.73450077066, Loss: 3.495337724685669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10160 , TFLOPS: 68.831403392721, Tokens per sec: 170571.80001219083, Loss: 3.4974732398986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10161 , TFLOPS: 69.97487292390277, Tokens per sec: 173405.44347402666, Loss: 3.5121283531188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10162 , TFLOPS: 68.44663004096137, Tokens per sec: 169618.28926024665, Loss: 3.478363275527954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10163 , TFLOPS: 68.7696690121002, Tokens per sec: 170418.81541640902, Loss: 3.495098114013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10164 , TFLOPS: 68.87536636604779, Tokens per sec: 170680.7451030159, Loss: 3.4902267456054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10165 , TFLOPS: 69.66226925473322, Tokens per sec: 172630.7771456876, Loss: 3.49055552482605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10166 , TFLOPS: 67.9089190048697, Tokens per sec: 168285.78207905102, Loss: 3.500764846801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10167 , TFLOPS: 68.0118297579306, Tokens per sec: 168540.80626169124, Loss: 3.5108304023742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10168 , TFLOPS: 70.55848050120044, Tokens per sec: 174851.6873402522, Loss: 3.5040688514709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10169 , TFLOPS: 69.06738463728614, Tokens per sec: 171156.5875317033, Loss: 3.485241651535034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10170 , TFLOPS: 66.94143855168525, Tokens per sec: 165888.2589392899, Loss: 3.509876251220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10171 , TFLOPS: 70.02682360244327, Tokens per sec: 173534.18297828973, Loss: 3.511420726776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10172 , TFLOPS: 68.8961951416834, Tokens per sec: 170732.36110352003, Loss: 3.5024824142456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10173 , TFLOPS: 67.9677799334859, Tokens per sec: 168431.6459442285, Loss: 3.4819440841674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10174 , TFLOPS: 68.7463139668892, Tokens per sec: 170360.9390416056, Loss: 3.496971607208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10175 , TFLOPS: 69.50817744724314, Tokens per sec: 172248.92067211328, Loss: 3.478102922439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10176 , TFLOPS: 68.20385195474906, Tokens per sec: 169016.65841839855, Loss: 3.51604962348938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10177 , TFLOPS: 67.85084967819196, Tokens per sec: 168141.87989067915, Loss: 3.495699405670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10178 , TFLOPS: 70.52006675084534, Tokens per sec: 174756.49383524834, Loss: 3.481276512145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10179 , TFLOPS: 68.77243646934923, Tokens per sec: 170425.67348033274, Loss: 3.5038414001464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10180 , TFLOPS: 68.09529253369693, Tokens per sec: 168747.63621422395, Loss: 3.4917101860046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10181 , TFLOPS: 68.34545827337561, Tokens per sec: 169367.57447517893, Loss: 3.4923272132873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10182 , TFLOPS: 68.60899246930849, Tokens per sec: 170020.64124338882, Loss: 3.465336799621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10183 , TFLOPS: 68.69999357392928, Tokens per sec: 170246.15200523834, Loss: 3.5010764598846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10184 , TFLOPS: 65.79922298830617, Tokens per sec: 163057.72294780504, Loss: 3.501999855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10185 , TFLOPS: 70.58197673497016, Tokens per sec: 174909.9135958574, Loss: 3.5060036182403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10186 , TFLOPS: 68.23556431559312, Tokens per sec: 169095.2451419751, Loss: 3.4763450622558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10187 , TFLOPS: 67.98041608848344, Tokens per sec: 168462.95972829981, Loss: 3.4921584129333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10188 , TFLOPS: 69.60923782229234, Tokens per sec: 172499.35941420455, Loss: 3.491547107696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10189 , TFLOPS: 68.64118180302046, Tokens per sec: 170100.40995827492, Loss: 3.5129096508026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10190 , TFLOPS: 68.02379538493388, Tokens per sec: 168570.45840352797, Loss: 3.514591932296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10191 , TFLOPS: 69.77435282214951, Tokens per sec: 172908.5325727694, Loss: 3.5126819610595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10192 , TFLOPS: 68.76255079662407, Tokens per sec: 170401.17569432303, Loss: 3.5073318481445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10193 , TFLOPS: 68.15105533473387, Tokens per sec: 168885.8225779731, Loss: 3.4971742630004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10194 , TFLOPS: 68.74205243588084, Tokens per sec: 170350.37849832044, Loss: 3.5154693126678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10195 , TFLOPS: 69.51222065252145, Tokens per sec: 172258.94018018877, Loss: 3.479771614074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10196 , TFLOPS: 67.89258175564824, Tokens per sec: 168245.29657578364, Loss: 3.504765033721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10197 , TFLOPS: 68.30614870758724, Tokens per sec: 169270.1610408492, Loss: 3.510669231414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10198 , TFLOPS: 70.61145741131716, Tokens per sec: 174982.96995940985, Loss: 3.5033726692199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10199 , TFLOPS: 67.98652251795113, Tokens per sec: 168478.09213319948, Loss: 3.48416805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10200 , TFLOPS: 67.9054187704747, Tokens per sec: 168277.10811263754, Loss: 3.505718946456909 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/optimizer.pt +[rank0]:[2024-08-25 18:17:32,222] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006425508006941527, 'preprocessing_with_comm': 0.0013259330007713288, 'state_converting': 0.9834856890083756, : 0.9926385269936873}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008200 + eval ppl=25.13986587524414, eval loss=3.224454879760742 +------------------------------------------------------------------ +iteration: 10201 , TFLOPS: 64.53470600624213, Tokens per sec: 159924.11056820542, Loss: 3.4998281002044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10202 , TFLOPS: 69.52101139352379, Tokens per sec: 172280.72460477264, Loss: 3.4781599044799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10203 , TFLOPS: 69.07723391881815, Tokens per sec: 171180.99513632758, Loss: 3.4834165573120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10204 , TFLOPS: 68.18416148448947, Tokens per sec: 168967.86326401093, Loss: 3.502902030944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10205 , TFLOPS: 70.58693283624923, Tokens per sec: 174922.19536078657, Loss: 3.506805419921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10206 , TFLOPS: 66.90014057556857, Tokens per sec: 165785.91800512513, Loss: 3.4978017807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10207 , TFLOPS: 67.97790679154471, Tokens per sec: 168456.74141406376, Loss: 3.4948177337646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10208 , TFLOPS: 70.52091009855053, Tokens per sec: 174758.5837437642, Loss: 3.516571521759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10209 , TFLOPS: 69.50513032587482, Tokens per sec: 172241.36956969445, Loss: 3.499067783355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10210 , TFLOPS: 67.84848541293776, Tokens per sec: 168136.02098093397, Loss: 3.514632225036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10211 , TFLOPS: 68.68591374714707, Tokens per sec: 170211.2606434511, Loss: 3.5311360359191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10212 , TFLOPS: 69.65344609211198, Tokens per sec: 172608.91237676053, Loss: 3.4914205074310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10213 , TFLOPS: 68.93066214111788, Tokens per sec: 170817.77412497206, Loss: 3.503486156463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10214 , TFLOPS: 67.11740885518263, Tokens per sec: 166324.33273608558, Loss: 3.4853014945983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10215 , TFLOPS: 70.59436598739123, Tokens per sec: 174940.6155281456, Loss: 3.4757065773010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10216 , TFLOPS: 68.26364439584962, Tokens per sec: 169164.83067412692, Loss: 3.520129919052124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10217 , TFLOPS: 68.72083369635715, Tokens per sec: 170297.79612434364, Loss: 3.48396635055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10218 , TFLOPS: 68.91024994005562, Tokens per sec: 170767.19044215023, Loss: 3.4831888675689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10219 , TFLOPS: 68.73562520691299, Tokens per sec: 170334.45111692036, Loss: 3.5044097900390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10220 , TFLOPS: 68.21163500097565, Tokens per sec: 169035.9456643214, Loss: 3.502811908721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10221 , TFLOPS: 68.79424294416359, Tokens per sec: 170479.7123271042, Loss: 3.496830463409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10222 , TFLOPS: 68.27300491346162, Tokens per sec: 169188.0270679156, Loss: 3.5000240802764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10223 , TFLOPS: 68.7916559129665, Tokens per sec: 170473.30137881284, Loss: 3.4897360801696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10224 , TFLOPS: 68.6418260389779, Tokens per sec: 170102.00644594588, Loss: 3.4878921508789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10225 , TFLOPS: 69.58943704091082, Tokens per sec: 172450.290896705, Loss: 3.492095947265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10226 , TFLOPS: 67.81324173392456, Tokens per sec: 168048.68326192748, Loss: 3.4792263507843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10227 , TFLOPS: 68.04801115623441, Tokens per sec: 168630.4677523981, Loss: 3.4971063137054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10228 , TFLOPS: 70.51507537613809, Tokens per sec: 174744.12465887843, Loss: 3.48564076423645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10229 , TFLOPS: 68.66360852473592, Tokens per sec: 170155.98584519315, Loss: 3.477219581604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10230 , TFLOPS: 68.77339424768368, Tokens per sec: 170428.0469605535, Loss: 3.4801955223083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10231 , TFLOPS: 68.4979595942056, Tokens per sec: 169745.48954760295, Loss: 3.4793570041656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10232 , TFLOPS: 70.6171720838789, Tokens per sec: 174997.13154754098, Loss: 3.4690771102905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10233 , TFLOPS: 67.82254417881377, Tokens per sec: 168071.73574511186, Loss: 3.4942188262939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10234 , TFLOPS: 68.05472574219228, Tokens per sec: 168647.1072360744, Loss: 3.490649461746216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10235 , TFLOPS: 70.59083455740456, Tokens per sec: 174931.86425561953, Loss: 3.5146982669830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10236 , TFLOPS: 67.20209211256693, Tokens per sec: 166534.18717651477, Loss: 3.4864792823791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10237 , TFLOPS: 68.15580432239643, Tokens per sec: 168897.59109254513, Loss: 3.504464864730835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10238 , TFLOPS: 69.59552062911739, Tokens per sec: 172465.36669844345, Loss: 3.4573118686676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10239 , TFLOPS: 68.86648274829619, Tokens per sec: 170658.7305196162, Loss: 3.4778456687927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10240 , TFLOPS: 67.83657643944147, Tokens per sec: 168106.5092327284, Loss: 3.492396593093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10241 , TFLOPS: 69.20601462294121, Tokens per sec: 171500.12790751018, Loss: 3.5114059448242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10242 , TFLOPS: 69.58067313809782, Tokens per sec: 172428.57298011082, Loss: 3.4893240928649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10243 , TFLOPS: 68.16920052605543, Tokens per sec: 168930.78836092577, Loss: 3.498514413833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10244 , TFLOPS: 68.12986016747074, Tokens per sec: 168833.29861865466, Loss: 3.46622371673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10245 , TFLOPS: 70.53341259400095, Tokens per sec: 174789.5663047539, Loss: 3.5160908699035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10246 , TFLOPS: 67.89844162371641, Tokens per sec: 168259.81797437434, Loss: 3.468738317489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10247 , TFLOPS: 67.82890500346153, Tokens per sec: 168087.49856929167, Loss: 3.4783012866973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10248 , TFLOPS: 70.60901724339587, Tokens per sec: 174976.9229544942, Loss: 3.5024073123931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10249 , TFLOPS: 69.71041609843967, Tokens per sec: 172750.09032820485, Loss: 3.490516185760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10250 , TFLOPS: 68.75437381082264, Tokens per sec: 170380.91222273774, Loss: 3.4928722381591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10251 , TFLOPS: 68.67158056124401, Tokens per sec: 170175.74142985255, Loss: 3.486464738845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10252 , TFLOPS: 70.55280502409722, Tokens per sec: 174837.622882785, Loss: 3.494986057281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10253 , TFLOPS: 67.82276711256414, Tokens per sec: 168072.2881995034, Loss: 3.4855031967163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10254 , TFLOPS: 67.8748176506352, Tokens per sec: 168201.27516668363, Loss: 3.502241849899292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10255 , TFLOPS: 69.68017253843064, Tokens per sec: 172675.1434548993, Loss: 3.522770643234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10256 , TFLOPS: 68.83409080170718, Tokens per sec: 170578.4597077011, Loss: 3.507291793823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10257 , TFLOPS: 68.8354752659576, Tokens per sec: 170581.890562625, Loss: 3.493058919906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10258 , TFLOPS: 68.04367749790272, Tokens per sec: 168619.72846966033, Loss: 3.49415922164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10259 , TFLOPS: 68.74285793453716, Tokens per sec: 170352.3746127238, Loss: 3.5075035095214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10260 , TFLOPS: 69.61333935574312, Tokens per sec: 172509.52346591678, Loss: 3.4837236404418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10261 , TFLOPS: 68.15102371803412, Tokens per sec: 168885.74422830826, Loss: 3.483757734298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10262 , TFLOPS: 68.69162344268116, Tokens per sec: 170225.40989795947, Loss: 3.4888155460357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10263 , TFLOPS: 68.89831792807085, Tokens per sec: 170737.6216020323, Loss: 3.5164835453033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10264 , TFLOPS: 68.05798493348081, Tokens per sec: 168655.18387111695, Loss: 3.4648001194000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10265 , TFLOPS: 70.58305259652523, Tokens per sec: 174912.57969930678, Loss: 3.5177838802337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10266 , TFLOPS: 66.8630452420965, Tokens per sec: 165693.99168239304, Loss: 3.513308525085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10267 , TFLOPS: 68.77117723486771, Tokens per sec: 170422.55295857138, Loss: 3.4855194091796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10268 , TFLOPS: 69.90607670380672, Tokens per sec: 173234.9588620994, Loss: 3.4885294437408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10269 , TFLOPS: 69.70482796338791, Tokens per sec: 172736.2423139627, Loss: 3.4836764335632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10270 , TFLOPS: 67.87093696640844, Tokens per sec: 168191.65840367705, Loss: 3.502197027206421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10271 , TFLOPS: 68.77529097997386, Tokens per sec: 170432.74727211086, Loss: 3.4853153228759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10272 , TFLOPS: 70.57276205985342, Tokens per sec: 174887.07861583782, Loss: 3.4987857341766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10273 , TFLOPS: 66.70470644920354, Tokens per sec: 165301.6107111463, Loss: 3.5040478706359863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10274 , TFLOPS: 67.91544855630612, Tokens per sec: 168301.96302679993, Loss: 3.5087504386901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10275 , TFLOPS: 70.58442899892322, Tokens per sec: 174915.99057607225, Loss: 3.4973254203796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10276 , TFLOPS: 68.92117034396557, Tokens per sec: 170794.25240602088, Loss: 3.4886913299560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10277 , TFLOPS: 68.52474795240113, Tokens per sec: 169811.87405019274, Loss: 3.504791259765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10278 , TFLOPS: 68.79855196630142, Tokens per sec: 170490.39055863972, Loss: 3.501410961151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10279 , TFLOPS: 69.51740384447352, Tokens per sec: 172271.78470082174, Loss: 3.4952971935272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10280 , TFLOPS: 68.1566797226818, Tokens per sec: 168899.7604308856, Loss: 3.5004358291625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10281 , TFLOPS: 68.00770256422325, Tokens per sec: 168530.5786210365, Loss: 3.493678092956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10282 , TFLOPS: 70.61965343323803, Tokens per sec: 175003.2806045961, Loss: 3.5122382640838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10283 , TFLOPS: 68.34745439146262, Tokens per sec: 169372.5210756891, Loss: 3.517045497894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10284 , TFLOPS: 68.75062328970463, Tokens per sec: 170371.6180182533, Loss: 3.517195701599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10285 , TFLOPS: 69.84370157941186, Tokens per sec: 173080.38643266173, Loss: 3.4960074424743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10286 , TFLOPS: 68.74971717484006, Tokens per sec: 170369.3725658021, Loss: 3.4947376251220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10287 , TFLOPS: 68.9336002938673, Tokens per sec: 170825.05519114938, Loss: 3.494472026824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10288 , TFLOPS: 68.2458602043008, Tokens per sec: 169120.7594883785, Loss: 3.5005388259887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10289 , TFLOPS: 69.64657934611354, Tokens per sec: 172591.8958237419, Loss: 3.490354299545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10290 , TFLOPS: 69.71122279798848, Tokens per sec: 172752.0894185521, Loss: 3.4916186332702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10291 , TFLOPS: 68.25499636833483, Tokens per sec: 169143.39990928635, Loss: 3.4810662269592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10292 , TFLOPS: 69.52038697240141, Tokens per sec: 172279.1772204457, Loss: 3.5071630477905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10293 , TFLOPS: 67.81897633258833, Tokens per sec: 168062.89422913452, Loss: 3.488558769226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10294 , TFLOPS: 67.90877846207874, Tokens per sec: 168285.43379853296, Loss: 3.459625244140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10295 , TFLOPS: 69.50099049531046, Tokens per sec: 172231.110613516, Loss: 3.4915614128112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10296 , TFLOPS: 68.75321708361572, Tokens per sec: 170378.04572529497, Loss: 3.4975404739379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10297 , TFLOPS: 68.80875871942891, Tokens per sec: 170515.6840172563, Loss: 3.473663330078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10298 , TFLOPS: 68.83687049916864, Tokens per sec: 170585.34810421953, Loss: 3.501166343688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10299 , TFLOPS: 69.64056248380999, Tokens per sec: 172576.98537614162, Loss: 3.494131088256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10300 , TFLOPS: 68.12637778749836, Tokens per sec: 168824.66889746726, Loss: 3.495516538619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10301 , TFLOPS: 67.76299518077911, Tokens per sec: 167924.16676811827, Loss: 3.501844882965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10302 , TFLOPS: 70.54418813142094, Tokens per sec: 174816.26927350406, Loss: 3.481463670730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10303 , TFLOPS: 68.6880548028589, Tokens per sec: 170216.56641536174, Loss: 3.469761371612549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10304 , TFLOPS: 68.34066634162495, Tokens per sec: 169355.6995404259, Loss: 3.5052061080932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10305 , TFLOPS: 69.63411910556387, Tokens per sec: 172561.017975051, Loss: 3.492281436920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10306 , TFLOPS: 68.74408326656788, Tokens per sec: 170355.41112047742, Loss: 3.5022542476654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10307 , TFLOPS: 68.60879893116292, Tokens per sec: 170020.1616345441, Loss: 3.500901699066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10308 , TFLOPS: 68.83865212096322, Tokens per sec: 170589.76315928838, Loss: 3.512782335281372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10309 , TFLOPS: 69.08212844252509, Tokens per sec: 171193.12430524963, Loss: 3.4749131202697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10310 , TFLOPS: 68.63781312605389, Tokens per sec: 170092.06200566134, Loss: 3.5049524307250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10311 , TFLOPS: 68.1522631162881, Tokens per sec: 168888.8155937126, Loss: 3.48165225982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10312 , TFLOPS: 70.52551885564188, Tokens per sec: 174770.00475153307, Loss: 3.482897996902466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10313 , TFLOPS: 67.85047643230492, Tokens per sec: 168140.95494625464, Loss: 3.483100652694702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10314 , TFLOPS: 67.64205465318727, Tokens per sec: 167624.4628180468, Loss: 3.5082616806030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10315 , TFLOPS: 69.87073662973171, Tokens per sec: 173147.38226551097, Loss: 3.4994938373565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10316 , TFLOPS: 68.90587514236111, Tokens per sec: 170756.3492115395, Loss: 3.4846506118774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10317 , TFLOPS: 67.78645543286011, Tokens per sec: 167982.3038571351, Loss: 3.50972580909729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10318 , TFLOPS: 68.88620308863142, Tokens per sec: 170707.5996953417, Loss: 3.495917558670044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10319 , TFLOPS: 70.61450468894095, Tokens per sec: 174990.5214490469, Loss: 3.49153733253479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10320 , TFLOPS: 67.94748012769861, Tokens per sec: 168381.3407892794, Loss: 3.5068745613098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10321 , TFLOPS: 66.9505581962693, Tokens per sec: 165910.85842318094, Loss: 3.4935860633850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10322 , TFLOPS: 70.53036716213258, Tokens per sec: 174782.01938910206, Loss: 3.5185720920562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10323 , TFLOPS: 68.7705707397135, Tokens per sec: 170421.0499967683, Loss: 3.5056347846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10324 , TFLOPS: 68.87184265004687, Tokens per sec: 170672.01294659628, Loss: 3.515803337097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10325 , TFLOPS: 66.788770778846, Tokens per sec: 165509.9313804507, Loss: 3.5174713134765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10326 , TFLOPS: 68.77116404071438, Tokens per sec: 170422.52026200597, Loss: 3.4750185012817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10327 , TFLOPS: 69.67534702743843, Tokens per sec: 172663.1853070867, Loss: 3.517615795135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10328 , TFLOPS: 68.1245310016544, Tokens per sec: 168820.0923587057, Loss: 3.4925057888031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10329 , TFLOPS: 69.62004233017448, Tokens per sec: 172526.13417495126, Loss: 3.4827914237976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10330 , TFLOPS: 68.87695516913588, Tokens per sec: 170684.68233208932, Loss: 3.489025354385376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10331 , TFLOPS: 67.95932588098016, Tokens per sec: 168410.69587082855, Loss: 3.5054101943969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10332 , TFLOPS: 69.47956627011317, Tokens per sec: 172178.01902340385, Loss: 3.4970641136169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10333 , TFLOPS: 68.86235845430511, Tokens per sec: 170648.51006477774, Loss: 3.45951509475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10334 , TFLOPS: 68.05703749441223, Tokens per sec: 168652.836012736, Loss: 3.485152244567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10335 , TFLOPS: 69.67322660212815, Tokens per sec: 172657.93065958368, Loss: 3.506983518600464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10336 , TFLOPS: 68.35977912252342, Tokens per sec: 169403.06311693895, Loss: 3.519211769104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10337 , TFLOPS: 68.77647064314144, Tokens per sec: 170435.67060739073, Loss: 3.4609994888305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10338 , TFLOPS: 68.4930671262481, Tokens per sec: 169733.36547305447, Loss: 3.4856057167053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10339 , TFLOPS: 70.56618324927699, Tokens per sec: 174870.77559851366, Loss: 3.5029783248901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10340 , TFLOPS: 67.13009823711262, Tokens per sec: 166355.77842235888, Loss: 3.4760074615478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10341 , TFLOPS: 67.87376413478252, Tokens per sec: 168198.66443834634, Loss: 3.4834794998168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10342 , TFLOPS: 70.55403270695895, Tokens per sec: 174840.66521615678, Loss: 3.4675140380859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10343 , TFLOPS: 68.65921150025909, Tokens per sec: 170145.08953416735, Loss: 3.510918140411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10344 , TFLOPS: 67.80823345539804, Tokens per sec: 168036.27219603135, Loss: 3.502957344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10345 , TFLOPS: 68.90525417156753, Tokens per sec: 170754.8103775074, Loss: 3.4823601245880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10346 , TFLOPS: 69.6473034475112, Tokens per sec: 172593.69022677167, Loss: 3.4853627681732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10347 , TFLOPS: 68.24645022197632, Tokens per sec: 169122.22161717425, Loss: 3.4827492237091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10348 , TFLOPS: 67.78862244618448, Tokens per sec: 167987.67395486913, Loss: 3.488077402114868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10349 , TFLOPS: 70.53142948009925, Tokens per sec: 174784.65193004746, Loss: 3.521070718765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10350 , TFLOPS: 68.62473672415419, Tokens per sec: 170059.6572412693, Loss: 3.5158028602600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10351 , TFLOPS: 67.80164554828002, Tokens per sec: 168019.946636475, Loss: 3.499225616455078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10352 , TFLOPS: 70.02905659386997, Tokens per sec: 173539.71657702938, Loss: 3.4767608642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10353 , TFLOPS: 68.76246860654523, Tokens per sec: 170400.97201825367, Loss: 3.4721949100494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10354 , TFLOPS: 67.78352304462344, Tokens per sec: 167975.0370760554, Loss: 3.4707322120666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10355 , TFLOPS: 69.52675226326711, Tokens per sec: 172294.9511124056, Loss: 3.5015852451324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10356 , TFLOPS: 67.85980968390344, Tokens per sec: 168164.0837718583, Loss: 3.4919486045837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10357 , TFLOPS: 68.70707893241386, Tokens per sec: 170263.71030407984, Loss: 3.4814453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10358 , TFLOPS: 67.85396289096838, Tokens per sec: 168149.5947748878, Loss: 3.5107038021087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10359 , TFLOPS: 70.53430084021622, Tokens per sec: 174791.76747672376, Loss: 3.471942186355591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10360 , TFLOPS: 67.91547042362555, Tokens per sec: 168302.01721642635, Loss: 3.4800028800964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10361 , TFLOPS: 67.82626971894531, Tokens per sec: 168080.96804395993, Loss: 3.505009651184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10362 , TFLOPS: 69.57029145863818, Tokens per sec: 172402.8460347734, Loss: 3.505728244781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10363 , TFLOPS: 68.74618442458937, Tokens per sec: 170360.61802151633, Loss: 3.508608102798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10364 , TFLOPS: 68.73737837085584, Tokens per sec: 170338.79565029772, Loss: 3.48746919631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10365 , TFLOPS: 68.8617079136304, Tokens per sec: 170646.89795332286, Loss: 3.49662446975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10366 , TFLOPS: 69.5791298772388, Tokens per sec: 172424.7486096979, Loss: 3.5119264125823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10367 , TFLOPS: 68.0375632455787, Tokens per sec: 168604.5766788609, Loss: 3.5022404193878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10368 , TFLOPS: 67.8977266458239, Tokens per sec: 168258.04618039535, Loss: 3.48854660987854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10369 , TFLOPS: 69.5262795572807, Tokens per sec: 172293.7796948984, Loss: 3.522213935852051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10370 , TFLOPS: 69.61399846371035, Tokens per sec: 172511.15680806633, Loss: 3.4996330738067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10371 , TFLOPS: 68.20419850464951, Tokens per sec: 169017.51720722753, Loss: 3.4999396800994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10372 , TFLOPS: 69.72339941082853, Tokens per sec: 172782.26440653438, Loss: 3.512765407562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10373 , TFLOPS: 67.85564760794043, Tokens per sec: 168153.76968913144, Loss: 3.490567207336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10374 , TFLOPS: 68.68858381831639, Tokens per sec: 170217.87737394127, Loss: 3.47441029548645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10375 , TFLOPS: 69.13870352512782, Tokens per sec: 171333.32359219904, Loss: 3.4949235916137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10376 , TFLOPS: 68.85212373344895, Tokens per sec: 170623.1472990486, Loss: 3.5187337398529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10377 , TFLOPS: 68.58695573774897, Tokens per sec: 169966.03179503794, Loss: 3.5093226432800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10378 , TFLOPS: 68.12497249640715, Tokens per sec: 168821.18643133767, Loss: 3.5010552406311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10379 , TFLOPS: 70.55760029695544, Tokens per sec: 174849.50609717058, Loss: 3.501445770263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10380 , TFLOPS: 67.9471321966331, Tokens per sec: 168380.4785777713, Loss: 3.5105319023132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10381 , TFLOPS: 67.08932182489339, Tokens per sec: 166254.72998098703, Loss: 3.458726406097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10382 , TFLOPS: 70.53098741998816, Tokens per sec: 174783.55645639528, Loss: 3.504084587097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10383 , TFLOPS: 68.75388525462573, Tokens per sec: 170379.7015266629, Loss: 3.5196027755737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10384 , TFLOPS: 67.718382957442, Tokens per sec: 167813.61276424685, Loss: 3.5045855045318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10385 , TFLOPS: 68.65962148663469, Tokens per sec: 170146.10552556982, Loss: 3.518980026245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10386 , TFLOPS: 70.56257380653469, Tokens per sec: 174861.8310017853, Loss: 3.501828908920288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10387 , TFLOPS: 67.90100271560195, Tokens per sec: 168266.16464219426, Loss: 3.4869940280914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10388 , TFLOPS: 67.21809806212362, Tokens per sec: 166573.85168271713, Loss: 3.503049850463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10389 , TFLOPS: 70.64179407419064, Tokens per sec: 175058.14755186962, Loss: 3.48097562789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10390 , TFLOPS: 69.61430193503246, Tokens per sec: 172511.90884343264, Loss: 3.5224766731262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10391 , TFLOPS: 67.66025644339123, Tokens per sec: 167669.5688592649, Loss: 3.5088815689086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10392 , TFLOPS: 69.04595731811487, Tokens per sec: 171103.48827438283, Loss: 3.5156123638153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10393 , TFLOPS: 68.69318803114085, Tokens per sec: 170229.2871205162, Loss: 3.4797821044921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10394 , TFLOPS: 69.70039149576525, Tokens per sec: 172725.24825847647, Loss: 3.4850592613220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10395 , TFLOPS: 67.34290953532182, Tokens per sec: 166883.14826242108, Loss: 3.4943878650665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10396 , TFLOPS: 69.67226418047188, Tokens per sec: 172655.54567269995, Loss: 3.483147144317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10397 , TFLOPS: 68.95553858462947, Tokens per sec: 170879.42069236026, Loss: 3.498643398284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10398 , TFLOPS: 67.84146605404153, Tokens per sec: 168118.62623634285, Loss: 3.506277084350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10399 , TFLOPS: 69.51060893210636, Tokens per sec: 172254.94615945456, Loss: 3.504605293273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10400 , TFLOPS: 68.00281035810343, Tokens per sec: 168518.45519535057, Loss: 3.4995765686035156 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/model.pt +[rank0]:[2024-08-25 18:43:18,511] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006330465999781154, 'preprocessing_with_comm': 0.001471128998673521, 'state_converting': 0.9917973730043741, : 1.0009711489983601}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008400 + eval ppl=24.48037338256836, eval loss=3.197871685028076 +------------------------------------------------------------------ +iteration: 10401 , TFLOPS: 63.98242133969929, Tokens per sec: 158555.48832533383, Loss: 3.496762990951538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10402 , TFLOPS: 69.19429091867354, Tokens per sec: 171471.07527686228, Loss: 3.4884397983551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10403 , TFLOPS: 69.5870013828574, Tokens per sec: 172444.25506773873, Loss: 3.493210554122925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10404 , TFLOPS: 68.09359906637121, Tokens per sec: 168743.43961563997, Loss: 3.505143880844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10405 , TFLOPS: 68.74495152849386, Tokens per sec: 170357.5627691769, Loss: 3.506735324859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10406 , TFLOPS: 69.66213695227422, Tokens per sec: 172630.44928561986, Loss: 3.503230571746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10407 , TFLOPS: 67.85632503341239, Tokens per sec: 168155.44842407556, Loss: 3.484848737716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10408 , TFLOPS: 68.06111627391205, Tokens per sec: 168662.9436776512, Loss: 3.4851889610290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10409 , TFLOPS: 69.61105995135377, Tokens per sec: 172503.87485074284, Loss: 3.481614112854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10410 , TFLOPS: 68.73138777168288, Tokens per sec: 170323.95028562815, Loss: 3.4993793964385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10411 , TFLOPS: 68.63004354488614, Tokens per sec: 170072.80812763763, Loss: 3.503218173980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10412 , TFLOPS: 68.95073753734593, Tokens per sec: 170867.52316831262, Loss: 3.482424020767212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10413 , TFLOPS: 69.61099127049725, Tokens per sec: 172503.7046520146, Loss: 3.5086100101470947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10414 , TFLOPS: 67.95270591513663, Tokens per sec: 168394.29086621886, Loss: 3.4828672409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10415 , TFLOPS: 68.29646224177841, Tokens per sec: 169246.15691152262, Loss: 3.486931324005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10416 , TFLOPS: 69.5685307814564, Tokens per sec: 172398.48288276236, Loss: 3.5009944438934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10417 , TFLOPS: 67.99422957174376, Tokens per sec: 168497.19106149967, Loss: 3.5076053142547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10418 , TFLOPS: 67.8814053205016, Tokens per sec: 168217.6001383043, Loss: 3.5027549266815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10419 , TFLOPS: 70.57113071729616, Tokens per sec: 174883.03596927374, Loss: 3.4734115600585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10420 , TFLOPS: 67.78754212911157, Tokens per sec: 167984.99681015575, Loss: 3.5010552406311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10421 , TFLOPS: 67.92488436266086, Tokens per sec: 168325.34599438685, Loss: 3.5242161750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10422 , TFLOPS: 69.93864341308291, Tokens per sec: 173315.66275520498, Loss: 3.486072063446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10423 , TFLOPS: 68.88633628736058, Tokens per sec: 170707.92977646558, Loss: 3.4889187812805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10424 , TFLOPS: 68.21683964813177, Tokens per sec: 169048.84335331345, Loss: 3.485957622528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10425 , TFLOPS: 68.7924956740715, Tokens per sec: 170475.38239933833, Loss: 3.5073060989379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10426 , TFLOPS: 70.61500195278875, Tokens per sec: 174991.7537236399, Loss: 3.4931347370147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10427 , TFLOPS: 67.90291713189355, Tokens per sec: 168270.90877665524, Loss: 3.4935903549194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10428 , TFLOPS: 67.79682782652266, Tokens per sec: 168008.0077912444, Loss: 3.4776721000671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10429 , TFLOPS: 70.63367633825777, Tokens per sec: 175038.03090798555, Loss: 3.5011465549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10430 , TFLOPS: 68.83677055334067, Tokens per sec: 170585.10042744793, Loss: 3.49434232711792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10431 , TFLOPS: 68.7078639149056, Tokens per sec: 170265.65557716732, Loss: 3.4897093772888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10432 , TFLOPS: 68.70268328383584, Tokens per sec: 170252.81740268282, Loss: 3.488642930984497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10433 , TFLOPS: 69.60494383254964, Tokens per sec: 172488.7184346001, Loss: 3.483253002166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10434 , TFLOPS: 68.93552671520209, Tokens per sec: 170829.8290754298, Loss: 3.4899871349334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10435 , TFLOPS: 68.05615137187371, Tokens per sec: 168650.6401034711, Loss: 3.5055055618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10436 , TFLOPS: 69.54411365885002, Tokens per sec: 172337.97456317567, Loss: 3.486009120941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10437 , TFLOPS: 68.04439126755354, Tokens per sec: 168621.49726948325, Loss: 3.4994187355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10438 , TFLOPS: 68.84229567682851, Tokens per sec: 170598.7922920934, Loss: 3.506157398223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10439 , TFLOPS: 69.01151757525332, Tokens per sec: 171018.14279772106, Loss: 3.5006537437438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10440 , TFLOPS: 68.35665513769754, Tokens per sec: 169395.3215384051, Loss: 3.489450216293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10441 , TFLOPS: 69.22579037692958, Tokens per sec: 171549.1344043725, Loss: 3.508242130279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10442 , TFLOPS: 68.99834051860951, Tokens per sec: 170985.48859978508, Loss: 3.505539655685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10443 , TFLOPS: 68.62985283289588, Tokens per sec: 170072.33552231712, Loss: 3.512549638748169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10444 , TFLOPS: 69.03280326516506, Tokens per sec: 171070.89108213465, Loss: 3.480001211166382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10445 , TFLOPS: 68.11413977386, Tokens per sec: 168794.34175154293, Loss: 3.486745595932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10446 , TFLOPS: 70.65756968529266, Tokens per sec: 175097.24125400744, Loss: 3.5035643577575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10447 , TFLOPS: 67.0973387425342, Tokens per sec: 166274.59678603735, Loss: 3.508728504180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10448 , TFLOPS: 67.9884852393809, Tokens per sec: 168482.95597311406, Loss: 3.48104190826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10449 , TFLOPS: 70.58596394098649, Tokens per sec: 174919.79433159286, Loss: 3.4990665912628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10450 , TFLOPS: 68.63619626508138, Tokens per sec: 170088.05524605955, Loss: 3.5024523735046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10451 , TFLOPS: 67.68391246790738, Tokens per sec: 167728.19109394215, Loss: 3.5175914764404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10452 , TFLOPS: 68.89948971985146, Tokens per sec: 170740.5254311482, Loss: 3.4920504093170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10453 , TFLOPS: 70.56694941682444, Tokens per sec: 174872.67424609658, Loss: 3.480654239654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10454 , TFLOPS: 67.33417246437953, Tokens per sec: 166861.4968381583, Loss: 3.501264810562134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10455 , TFLOPS: 67.98210081428397, Tokens per sec: 168467.13466442135, Loss: 3.4900331497192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10456 , TFLOPS: 70.58361217341954, Tokens per sec: 174913.96639249337, Loss: 3.513791084289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10457 , TFLOPS: 68.63678162609253, Tokens per sec: 170089.505835128, Loss: 3.524559497833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10458 , TFLOPS: 67.3085044520601, Tokens per sec: 166797.88867606575, Loss: 3.4847800731658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10459 , TFLOPS: 69.74313003681725, Tokens per sec: 172831.15907124232, Loss: 3.5075626373291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10460 , TFLOPS: 69.59656905237621, Tokens per sec: 172467.9648067722, Loss: 3.5107743740081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10461 , TFLOPS: 68.20274234594535, Tokens per sec: 169013.90868554948, Loss: 3.491220712661743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10462 , TFLOPS: 68.72692868452133, Tokens per sec: 170312.9001764328, Loss: 3.4857215881347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10463 , TFLOPS: 69.2215360148881, Tokens per sec: 171538.59162657705, Loss: 3.493422508239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10464 , TFLOPS: 68.70024122156482, Tokens per sec: 170246.7657033599, Loss: 3.4786088466644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10465 , TFLOPS: 67.1015469343831, Tokens per sec: 166285.02514900948, Loss: 3.518306016921997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10466 , TFLOPS: 70.59852045957449, Tokens per sec: 174950.91076786863, Loss: 3.509394884109497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10467 , TFLOPS: 68.02279324300547, Tokens per sec: 168567.97498543435, Loss: 3.5189905166625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10468 , TFLOPS: 68.78271964309381, Tokens per sec: 170451.15631765692, Loss: 3.4931936264038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10469 , TFLOPS: 68.88834327608383, Tokens per sec: 170712.90331560987, Loss: 3.486469030380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10470 , TFLOPS: 68.67618129219373, Tokens per sec: 170187.14254795833, Loss: 3.498680830001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10471 , TFLOPS: 68.61936014406845, Tokens per sec: 170046.3334835377, Loss: 3.4979782104492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10472 , TFLOPS: 68.8908649211029, Tokens per sec: 170719.15222974925, Loss: 3.4792590141296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10473 , TFLOPS: 69.6507455005853, Tokens per sec: 172602.2200134627, Loss: 3.469379186630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10474 , TFLOPS: 67.93487949624834, Tokens per sec: 168350.1150364715, Loss: 3.494067668914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10475 , TFLOPS: 67.95911481073983, Tokens per sec: 168410.17281552168, Loss: 3.504058837890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10476 , TFLOPS: 69.53484742152561, Tokens per sec: 172315.01180632543, Loss: 3.51314640045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10477 , TFLOPS: 68.6955688292647, Tokens per sec: 170235.18700053368, Loss: 3.4865646362304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10478 , TFLOPS: 68.75461085921135, Tokens per sec: 170381.499654758, Loss: 3.472174882888794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10479 , TFLOPS: 69.13258506162585, Tokens per sec: 171318.16136563674, Loss: 3.4860970973968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10480 , TFLOPS: 68.6977547985376, Tokens per sec: 170240.60407319633, Loss: 3.4834656715393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10481 , TFLOPS: 68.92300476576717, Tokens per sec: 170798.7983053235, Loss: 3.497378349304199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10482 , TFLOPS: 68.09806550522018, Tokens per sec: 168754.50794312658, Loss: 3.497407913208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10483 , TFLOPS: 70.58904558257662, Tokens per sec: 174927.430978924, Loss: 3.497819423675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10484 , TFLOPS: 67.07135330849471, Tokens per sec: 166210.20201795563, Loss: 3.489961624145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10485 , TFLOPS: 68.0275797275296, Tokens per sec: 168579.8364213599, Loss: 3.520033836364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10486 , TFLOPS: 70.61617980962588, Tokens per sec: 174994.67258263397, Loss: 3.491633653640747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10487 , TFLOPS: 68.80681114591256, Tokens per sec: 170510.8577155399, Loss: 3.4700493812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10488 , TFLOPS: 68.15763644000596, Tokens per sec: 168902.13128180592, Loss: 3.507260799407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10489 , TFLOPS: 69.60113187694157, Tokens per sec: 172479.27198868175, Loss: 3.4487032890319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10490 , TFLOPS: 69.63027597733988, Tokens per sec: 172551.49427995645, Loss: 3.4896204471588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10491 , TFLOPS: 67.31086038996213, Tokens per sec: 166803.72694971494, Loss: 3.492889404296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10492 , TFLOPS: 68.9079322383221, Tokens per sec: 170761.44692193862, Loss: 3.4771816730499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10493 , TFLOPS: 70.51480769097982, Tokens per sec: 174743.46130556831, Loss: 3.475342035293579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10494 , TFLOPS: 67.20194746001539, Tokens per sec: 166533.82871155726, Loss: 3.482431411743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10495 , TFLOPS: 66.96376549870567, Tokens per sec: 165943.58757352273, Loss: 3.5127501487731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10496 , TFLOPS: 70.58064498118888, Tokens per sec: 174906.61336328258, Loss: 3.5058813095092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10497 , TFLOPS: 68.82965468192006, Tokens per sec: 170567.4665141897, Loss: 3.4917471408843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10498 , TFLOPS: 68.75247550903626, Tokens per sec: 170376.20802179587, Loss: 3.4944229125976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10499 , TFLOPS: 68.76459945989251, Tokens per sec: 170406.2525075811, Loss: 3.5136830806732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10500 , TFLOPS: 69.58642003362317, Tokens per sec: 172442.8144202954, Loss: 3.5149660110473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10501 , TFLOPS: 68.86795525812914, Tokens per sec: 170662.3795611926, Loss: 3.4857420921325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10502 , TFLOPS: 67.02593153762483, Tokens per sec: 166097.6418661189, Loss: 3.506383180618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10503 , TFLOPS: 70.57895308568843, Tokens per sec: 174902.4206598548, Loss: 3.503657579421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10504 , TFLOPS: 67.88293939726573, Tokens per sec: 168221.40174951835, Loss: 3.5115737915039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10505 , TFLOPS: 68.77188519514748, Tokens per sec: 170424.3073621329, Loss: 3.4995381832122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10506 , TFLOPS: 69.03099746658631, Tokens per sec: 171066.41611433137, Loss: 3.5141072273254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10507 , TFLOPS: 68.74669072472771, Tokens per sec: 170361.8726890327, Loss: 3.5126607418060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10508 , TFLOPS: 68.02634949247681, Tokens per sec: 168576.78776337756, Loss: 3.48146390914917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10509 , TFLOPS: 68.68344321445292, Tokens per sec: 170205.13839128337, Loss: 3.5266661643981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10510 , TFLOPS: 69.51611830430636, Tokens per sec: 172268.59899067358, Loss: 3.492319345474243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10511 , TFLOPS: 67.87536023202793, Tokens per sec: 168202.6197431451, Loss: 3.4965710639953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10512 , TFLOPS: 68.76451091469066, Tokens per sec: 170406.03308281687, Loss: 3.4728715419769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10513 , TFLOPS: 69.49491231095521, Tokens per sec: 172216.04820311643, Loss: 3.4914519786834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10514 , TFLOPS: 68.03838511622601, Tokens per sec: 168606.61336486146, Loss: 3.491478681564331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10515 , TFLOPS: 68.1148281630424, Tokens per sec: 168796.04765576927, Loss: 3.4918437004089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10516 , TFLOPS: 70.61475721443381, Tokens per sec: 174991.14723503622, Loss: 3.4907119274139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10517 , TFLOPS: 67.84538726544712, Tokens per sec: 168128.343430163, Loss: 3.4994285106658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10518 , TFLOPS: 68.74506120531035, Tokens per sec: 170357.8345604101, Loss: 3.488157272338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10519 , TFLOPS: 68.70970244380203, Tokens per sec: 170270.21165430336, Loss: 3.4845495223999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10520 , TFLOPS: 70.5728455583694, Tokens per sec: 174887.28553435855, Loss: 3.485247850418091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10521 , TFLOPS: 67.31434720857592, Tokens per sec: 166812.3676703447, Loss: 3.490814685821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10522 , TFLOPS: 68.01916326944813, Tokens per sec: 168558.97951108465, Loss: 3.5083065032958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10523 , TFLOPS: 70.56246699038752, Tokens per sec: 174861.56629960585, Loss: 3.458604097366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10524 , TFLOPS: 67.87810365883692, Tokens per sec: 168209.4182569912, Loss: 3.5157320499420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10525 , TFLOPS: 67.7860806817681, Tokens per sec: 167981.37518264688, Loss: 3.5007262229919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10526 , TFLOPS: 69.67616641332833, Tokens per sec: 172665.21583558465, Loss: 3.529750347137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10527 , TFLOPS: 69.52964894514103, Tokens per sec: 172302.12940918503, Loss: 3.505099058151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10528 , TFLOPS: 67.18383466470591, Tokens per sec: 166488.94320949126, Loss: 3.4983372688293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10529 , TFLOPS: 69.12613478251828, Tokens per sec: 171302.17686345128, Loss: 3.4971160888671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10530 , TFLOPS: 69.63514982487855, Tokens per sec: 172563.5722110559, Loss: 3.4873197078704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10531 , TFLOPS: 68.65371862637927, Tokens per sec: 170131.47758760318, Loss: 3.51230525970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10532 , TFLOPS: 67.23527619977624, Tokens per sec: 166616.4210001484, Loss: 3.492554187774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10533 , TFLOPS: 70.54540143343695, Tokens per sec: 174819.275969556, Loss: 3.4974169731140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10534 , TFLOPS: 67.84094367781196, Tokens per sec: 168117.33173050152, Loss: 3.5007057189941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10535 , TFLOPS: 67.83709326361796, Tokens per sec: 168107.7899799706, Loss: 3.500037670135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10536 , TFLOPS: 69.36125328158765, Tokens per sec: 171884.82640458603, Loss: 3.494999885559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10537 , TFLOPS: 68.56564942513882, Tokens per sec: 169913.2324053059, Loss: 3.506164073944092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10538 , TFLOPS: 68.48964232990969, Tokens per sec: 169724.87845045538, Loss: 3.499730348587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10539 , TFLOPS: 68.56878815527187, Tokens per sec: 169921.0105243353, Loss: 3.479149103164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10540 , TFLOPS: 70.58862181087196, Tokens per sec: 174926.3808259576, Loss: 3.487602710723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10541 , TFLOPS: 67.95300229661862, Tokens per sec: 168395.02533218032, Loss: 3.4898247718811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10542 , TFLOPS: 67.9356758743256, Tokens per sec: 168352.08854907524, Loss: 3.50054931640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10543 , TFLOPS: 69.5833722764484, Tokens per sec: 172435.26174227818, Loss: 3.4972264766693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10544 , TFLOPS: 67.40565009162042, Tokens per sec: 167038.62627237182, Loss: 3.503509998321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10545 , TFLOPS: 67.82859751169312, Tokens per sec: 168086.73657081655, Loss: 3.4890663623809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10546 , TFLOPS: 67.98274726357626, Tokens per sec: 168468.73663697994, Loss: 3.502671718597412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10547 , TFLOPS: 69.55992071801468, Tokens per sec: 172377.146197077, Loss: 3.510371685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10548 , TFLOPS: 68.6638308953255, Tokens per sec: 170156.5369040103, Loss: 3.5039944648742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10549 , TFLOPS: 67.76059915276554, Tokens per sec: 167918.22914675638, Loss: 3.5069656372070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10550 , TFLOPS: 68.50657843337731, Tokens per sec: 169766.84798051397, Loss: 3.506145477294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10551 , TFLOPS: 68.35388837125171, Tokens per sec: 169388.46518636783, Loss: 3.4955289363861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10552 , TFLOPS: 67.83182366017324, Tokens per sec: 168094.73132213985, Loss: 3.511667251586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10553 , TFLOPS: 70.49685754613762, Tokens per sec: 174698.97886927763, Loss: 3.4695355892181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10554 , TFLOPS: 66.91362493210404, Tokens per sec: 165819.3337260445, Loss: 3.5063235759735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10555 , TFLOPS: 68.65127512976618, Tokens per sec: 170125.42233382253, Loss: 3.5092532634735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10556 , TFLOPS: 69.93670756192279, Tokens per sec: 173310.8655027786, Loss: 3.5041189193725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10557 , TFLOPS: 68.76557244486631, Tokens per sec: 170408.66367152933, Loss: 3.495100498199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10558 , TFLOPS: 67.55404753991088, Tokens per sec: 167406.37149656404, Loss: 3.5022246837615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10559 , TFLOPS: 68.84290807463512, Tokens per sec: 170600.30988131932, Loss: 3.489964246749878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10560 , TFLOPS: 70.60326262299878, Tokens per sec: 174962.66237122647, Loss: 3.4947474002838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10561 , TFLOPS: 66.98767751055381, Tokens per sec: 166002.844173008, Loss: 3.4903225898742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10562 , TFLOPS: 67.87835387535031, Tokens per sec: 168210.038321075, Loss: 3.4870107173919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10563 , TFLOPS: 70.59098852918666, Tokens per sec: 174932.24581465696, Loss: 3.5130984783172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10564 , TFLOPS: 68.70105558246489, Tokens per sec: 170248.7837793795, Loss: 3.518575429916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10565 , TFLOPS: 68.58087384118208, Tokens per sec: 169950.960185369, Loss: 3.477786064147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10566 , TFLOPS: 68.79461983416057, Tokens per sec: 170480.64630203458, Loss: 3.479034900665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10567 , TFLOPS: 69.32979813725159, Tokens per sec: 171806.87709185106, Loss: 3.4803361892700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10568 , TFLOPS: 68.78319523172188, Tokens per sec: 170452.33487866767, Loss: 3.5035276412963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10569 , TFLOPS: 67.70741563809187, Tokens per sec: 167786.43453874727, Loss: 3.47232985496521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10570 , TFLOPS: 70.57209536752454, Tokens per sec: 174885.42647880467, Loss: 3.505178928375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10571 , TFLOPS: 68.06673886497052, Tokens per sec: 168676.87707767362, Loss: 3.5255286693573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10572 , TFLOPS: 68.81794555774847, Tokens per sec: 170538.45001462597, Loss: 3.47292160987854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10573 , TFLOPS: 68.86601526936444, Tokens per sec: 170657.57205532622, Loss: 3.498257637023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10574 , TFLOPS: 68.65920941664353, Tokens per sec: 170145.08437073848, Loss: 3.479337215423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10575 , TFLOPS: 68.095622382339, Tokens per sec: 168748.45361549483, Loss: 3.493570327758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10576 , TFLOPS: 68.77367614934688, Tokens per sec: 170428.74554392893, Loss: 3.484870195388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10577 , TFLOPS: 69.60597187645412, Tokens per sec: 172491.2660406438, Loss: 3.481771945953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10578 , TFLOPS: 68.1760867591015, Tokens per sec: 168947.8532049951, Loss: 3.4841909408569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10579 , TFLOPS: 68.95974561732483, Tokens per sec: 170889.84618282225, Loss: 3.4836208820343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10580 , TFLOPS: 69.5861106318699, Tokens per sec: 172442.04768866743, Loss: 3.5011675357818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10581 , TFLOPS: 67.48800487363711, Tokens per sec: 167242.71049433734, Loss: 3.478665351867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10582 , TFLOPS: 67.7628971928587, Tokens per sec: 167923.92394325728, Loss: 3.5037975311279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10583 , TFLOPS: 68.98333910164558, Tokens per sec: 170948.31343600596, Loss: 3.4910237789154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10584 , TFLOPS: 68.71724029409118, Tokens per sec: 170288.89127768265, Loss: 3.490164041519165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10585 , TFLOPS: 68.67232562928642, Tokens per sec: 170177.58779053763, Loss: 3.497332811355591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10586 , TFLOPS: 68.77753270343204, Tokens per sec: 170438.30250978647, Loss: 3.5024030208587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10587 , TFLOPS: 69.50784239543788, Tokens per sec: 172248.09037683083, Loss: 3.4936230182647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10588 , TFLOPS: 68.03120708529461, Tokens per sec: 168588.82541348806, Loss: 3.4827871322631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10589 , TFLOPS: 67.77560433702745, Tokens per sec: 167955.41364631488, Loss: 3.4986484050750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10590 , TFLOPS: 70.57700335941645, Tokens per sec: 174897.58902337277, Loss: 3.47747802734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10591 , TFLOPS: 67.76181518925982, Tokens per sec: 167921.2426191467, Loss: 3.488388776779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10592 , TFLOPS: 67.80144617107419, Tokens per sec: 168019.45255779577, Loss: 3.481060743331909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10593 , TFLOPS: 69.73224899419452, Tokens per sec: 172804.19464897737, Loss: 3.47428822517395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10594 , TFLOPS: 69.57544377020727, Tokens per sec: 172415.61403041557, Loss: 3.4842092990875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10595 , TFLOPS: 67.39057922988275, Tokens per sec: 167001.2790167934, Loss: 3.516871690750122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10596 , TFLOPS: 69.06851702640228, Tokens per sec: 171159.39371667604, Loss: 3.500501871109009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10597 , TFLOPS: 69.55509516172786, Tokens per sec: 172365.18793701936, Loss: 3.4989871978759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10598 , TFLOPS: 68.61448882352877, Tokens per sec: 170034.2618146201, Loss: 3.507291316986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10599 , TFLOPS: 68.2484662151361, Tokens per sec: 169127.21747030417, Loss: 3.480297088623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10600 , TFLOPS: 70.57501445388283, Tokens per sec: 174892.6602963644, Loss: 3.485797166824341 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/optimizer.pt +[rank0]:[2024-08-25 19:09:06,101] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006341362997773103, 'preprocessing_with_comm': 0.001276814000448212, 'state_converting': 0.9769691859983141, : 0.9859503519983264}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600, took 4.56s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008600 + eval ppl=23.77022933959961, eval loss=3.168433904647827 +------------------------------------------------------------------ +iteration: 10601 , TFLOPS: 62.47121852859358, Tokens per sec: 154810.56128668154, Loss: 3.4979143142700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10602 , TFLOPS: 68.3438561268895, Tokens per sec: 169363.6041796953, Loss: 3.487882137298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10603 , TFLOPS: 68.18842822827644, Tokens per sec: 168978.4367251357, Loss: 3.491687774658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10604 , TFLOPS: 68.7549769487381, Tokens per sec: 170382.40686493338, Loss: 3.4981489181518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10605 , TFLOPS: 68.71715573280784, Tokens per sec: 170288.68172550728, Loss: 3.483018636703491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10606 , TFLOPS: 68.6977822624421, Tokens per sec: 170240.67213177707, Loss: 3.5063281059265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10607 , TFLOPS: 70.49609375384128, Tokens per sec: 174697.08610782865, Loss: 3.5264153480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10608 , TFLOPS: 67.82598805783161, Tokens per sec: 168080.27005669265, Loss: 3.4826385974884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10609 , TFLOPS: 68.06587096265115, Tokens per sec: 168674.72632011888, Loss: 3.515021324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10610 , TFLOPS: 69.6155771223867, Tokens per sec: 172515.0688981699, Loss: 3.4750728607177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10611 , TFLOPS: 69.55920630101733, Tokens per sec: 172375.37579305802, Loss: 3.5067248344421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10612 , TFLOPS: 68.9070809662159, Tokens per sec: 170759.3373758841, Loss: 3.4946517944335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10613 , TFLOPS: 68.77321772866976, Tokens per sec: 170427.60952699182, Loss: 3.503471612930298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10614 , TFLOPS: 68.58694912702923, Tokens per sec: 169966.0154129462, Loss: 3.5030150413513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10615 , TFLOPS: 69.09487572570005, Tokens per sec: 171224.71347718636, Loss: 3.4698054790496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10616 , TFLOPS: 67.86760829193257, Tokens per sec: 168183.4095816421, Loss: 3.497920036315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10617 , TFLOPS: 69.63602039027634, Tokens per sec: 172565.72956801235, Loss: 3.495696544647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10618 , TFLOPS: 68.1942918994624, Tokens per sec: 168992.9675482712, Loss: 3.502711772918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10619 , TFLOPS: 68.76836420781922, Tokens per sec: 170415.5819676647, Loss: 3.4705758094787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10620 , TFLOPS: 69.92172011788992, Tokens per sec: 173273.72496546188, Loss: 3.49527645111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10621 , TFLOPS: 66.90621500116099, Tokens per sec: 165800.97110089633, Loss: 3.4929041862487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10622 , TFLOPS: 69.57092288321124, Tokens per sec: 172404.4107744226, Loss: 3.513946294784546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10623 , TFLOPS: 68.85022546503659, Tokens per sec: 170618.44318081127, Loss: 3.4695658683776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10624 , TFLOPS: 68.42246965805624, Tokens per sec: 169558.4171699219, Loss: 3.48305082321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10625 , TFLOPS: 68.66040863507506, Tokens per sec: 170148.0561661166, Loss: 3.4928903579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10626 , TFLOPS: 68.85848669485756, Tokens per sec: 170638.91541835724, Loss: 3.4998788833618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10627 , TFLOPS: 70.57415606717585, Tokens per sec: 174890.5331195454, Loss: 3.4937615394592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10628 , TFLOPS: 67.00059686379333, Tokens per sec: 166034.85975351976, Loss: 3.5207269191741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10629 , TFLOPS: 68.72721521047163, Tokens per sec: 170313.6102193011, Loss: 3.4889087677001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10630 , TFLOPS: 69.99403584261027, Tokens per sec: 173452.93129755362, Loss: 3.474801778793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10631 , TFLOPS: 69.62903893185569, Tokens per sec: 172548.4287449748, Loss: 3.4907314777374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10632 , TFLOPS: 67.69865775758669, Tokens per sec: 167764.73154609522, Loss: 3.5162644386291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10633 , TFLOPS: 68.79449689425752, Tokens per sec: 170480.34164341187, Loss: 3.496025562286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10634 , TFLOPS: 70.49985954841004, Tokens per sec: 174706.41816160653, Loss: 3.487959861755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10635 , TFLOPS: 68.25084170436877, Tokens per sec: 169133.10419430444, Loss: 3.4965975284576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10636 , TFLOPS: 67.85968844472444, Tokens per sec: 168163.78332781716, Loss: 3.5003175735473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10637 , TFLOPS: 70.6242447048505, Tokens per sec: 175014.6582814221, Loss: 3.4856157302856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10638 , TFLOPS: 68.02480737138403, Tokens per sec: 168572.96621743098, Loss: 3.5017013549804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10639 , TFLOPS: 67.89081713911689, Tokens per sec: 168240.92366163043, Loss: 3.5207364559173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10640 , TFLOPS: 69.67041513676104, Tokens per sec: 172650.96353869553, Loss: 3.4750044345855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10641 , TFLOPS: 68.64575467933427, Tokens per sec: 170111.74204952447, Loss: 3.485769033432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10642 , TFLOPS: 69.00782511681564, Tokens per sec: 171008.9924789546, Loss: 3.495859146118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10643 , TFLOPS: 67.89515367222879, Tokens per sec: 168251.67006839023, Loss: 3.517442464828491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10644 , TFLOPS: 69.6176955138369, Tokens per sec: 172520.31850554363, Loss: 3.4958181381225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10645 , TFLOPS: 69.12917860436313, Tokens per sec: 171309.71978928748, Loss: 3.4951164722442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10646 , TFLOPS: 68.39707247406915, Tokens per sec: 169495.4801502711, Loss: 3.4909133911132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10647 , TFLOPS: 69.47954051175994, Tokens per sec: 172177.95519136707, Loss: 3.4951584339141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10648 , TFLOPS: 67.88677496196412, Tokens per sec: 168230.9067013633, Loss: 3.5190486907958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10649 , TFLOPS: 68.28164593383264, Tokens per sec: 169209.44046828194, Loss: 3.50978422164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10650 , TFLOPS: 69.6088869599971, Tokens per sec: 172498.48993878625, Loss: 3.498910665512085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10651 , TFLOPS: 68.67845323735575, Tokens per sec: 170192.7726783452, Loss: 3.464883804321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10652 , TFLOPS: 68.77763237915295, Tokens per sec: 170438.54951720306, Loss: 3.5007145404815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10653 , TFLOPS: 68.8872440442085, Tokens per sec: 170710.17929793242, Loss: 3.501396656036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10654 , TFLOPS: 69.50343294655868, Tokens per sec: 172237.1632767673, Loss: 3.478299617767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10655 , TFLOPS: 67.83268265785314, Tokens per sec: 168096.86001301705, Loss: 3.4879417419433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10656 , TFLOPS: 67.89383760400769, Tokens per sec: 168248.4087063615, Loss: 3.513667583465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10657 , TFLOPS: 70.60435021579325, Tokens per sec: 174965.35754597938, Loss: 3.5124993324279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10658 , TFLOPS: 67.10817014846103, Tokens per sec: 166301.4382031019, Loss: 3.503636121749878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10659 , TFLOPS: 68.85211898782363, Tokens per sec: 170623.1355388663, Loss: 3.4917852878570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10660 , TFLOPS: 69.55245134925622, Tokens per sec: 172358.63627847462, Loss: 3.479893684387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10661 , TFLOPS: 68.49946092105317, Tokens per sec: 169749.21000091603, Loss: 3.500899314880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10662 , TFLOPS: 69.64274244583261, Tokens per sec: 172582.3875621763, Loss: 3.5012149810791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10663 , TFLOPS: 68.18479051382973, Tokens per sec: 168969.4220680094, Loss: 3.500324010848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10664 , TFLOPS: 69.6729750626489, Tokens per sec: 172657.30731704473, Loss: 3.503110408782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10665 , TFLOPS: 67.76870931335264, Tokens per sec: 167938.32701809917, Loss: 3.4940688610076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10666 , TFLOPS: 68.0626182169497, Tokens per sec: 168666.66565795118, Loss: 3.4888243675231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10667 , TFLOPS: 70.55955618272205, Tokens per sec: 174854.35299755886, Loss: 3.4977946281433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10668 , TFLOPS: 67.47972373112144, Tokens per sec: 167222.18891094037, Loss: 3.511606216430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10669 , TFLOPS: 68.7437361981543, Tokens per sec: 170354.55104671582, Loss: 3.488123893737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10670 , TFLOPS: 69.09682583463166, Tokens per sec: 171229.5460619411, Loss: 3.4759578704833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10671 , TFLOPS: 69.59161215695111, Tokens per sec: 172455.68107386166, Loss: 3.4774231910705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10672 , TFLOPS: 68.72978400478519, Tokens per sec: 170319.9759745862, Loss: 3.500998020172119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10673 , TFLOPS: 67.90407723038577, Tokens per sec: 168273.78362851433, Loss: 3.5024046897888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10674 , TFLOPS: 70.57206578237566, Tokens per sec: 174885.35316354665, Loss: 3.513615369796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10675 , TFLOPS: 67.72429953641279, Tokens per sec: 167828.2746987014, Loss: 3.495651960372925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10676 , TFLOPS: 66.92731589772639, Tokens per sec: 165853.26144704106, Loss: 3.507889747619629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10677 , TFLOPS: 70.57212357927584, Tokens per sec: 174885.49639063215, Loss: 3.482942819595337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10678 , TFLOPS: 68.8224483305522, Tokens per sec: 170549.608381655, Loss: 3.4899401664733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10679 , TFLOPS: 69.06629225303146, Tokens per sec: 171153.88048318413, Loss: 3.5165326595306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10680 , TFLOPS: 68.07095658201244, Tokens per sec: 168687.3290451234, Loss: 3.50382137298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10681 , TFLOPS: 69.59163349791247, Tokens per sec: 172455.7339591148, Loss: 3.478008270263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10682 , TFLOPS: 69.15846961509376, Tokens per sec: 171382.30614054925, Loss: 3.518054485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10683 , TFLOPS: 67.84433877516288, Tokens per sec: 168125.7451557379, Loss: 3.5112271308898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10684 , TFLOPS: 69.38433419418602, Tokens per sec: 171942.0234485772, Loss: 3.5060834884643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10685 , TFLOPS: 68.02596297672326, Tokens per sec: 168575.82993476203, Loss: 3.4982705116271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10686 , TFLOPS: 68.91355359475733, Tokens per sec: 170775.37726242864, Loss: 3.5049750804901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10687 , TFLOPS: 69.81561168983653, Tokens per sec: 173010.77659193732, Loss: 3.4963431358337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10688 , TFLOPS: 66.93139286582125, Tokens per sec: 165863.36462309523, Loss: 3.4852330684661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10689 , TFLOPS: 69.6098421586259, Tokens per sec: 172500.85702621227, Loss: 3.484858512878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10690 , TFLOPS: 68.787796380707, Tokens per sec: 170463.73703271116, Loss: 3.489142417907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10691 , TFLOPS: 66.82930673546183, Tokens per sec: 165610.38394634897, Loss: 3.5139288902282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10692 , TFLOPS: 69.52904541579187, Tokens per sec: 172300.63379697356, Loss: 3.5025277137756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10693 , TFLOPS: 68.2653770951502, Tokens per sec: 169169.12449386631, Loss: 3.517786979675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10694 , TFLOPS: 70.55456022261848, Tokens per sec: 174841.97245807174, Loss: 3.4962756633758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10695 , TFLOPS: 67.76864549502999, Tokens per sec: 167938.16886926547, Loss: 3.5116968154907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10696 , TFLOPS: 68.61611228298811, Tokens per sec: 170038.28492599915, Loss: 3.490496873855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10697 , TFLOPS: 69.94029736662281, Tokens per sec: 173319.7614342749, Loss: 3.4928789138793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10698 , TFLOPS: 67.72870646409886, Tokens per sec: 167839.19555097067, Loss: 3.493724822998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10699 , TFLOPS: 68.7650623386719, Tokens per sec: 170407.39957218675, Loss: 3.487828254699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10700 , TFLOPS: 68.91895586005168, Tokens per sec: 170788.7646709367, Loss: 3.4996354579925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10701 , TFLOPS: 70.64631251830158, Tokens per sec: 175069.34475412426, Loss: 3.519254207611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10702 , TFLOPS: 67.28779856626919, Tokens per sec: 166746.57721013422, Loss: 3.5125935077667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10703 , TFLOPS: 67.80422849400192, Tokens per sec: 168026.3474605088, Loss: 3.497391700744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10704 , TFLOPS: 70.64607197463197, Tokens per sec: 175068.74866041288, Loss: 3.476670503616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10705 , TFLOPS: 67.6702902024289, Tokens per sec: 167694.43361947007, Loss: 3.487429618835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10706 , TFLOPS: 68.66246926663455, Tokens per sec: 170153.16263811835, Loss: 3.4793999195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10707 , TFLOPS: 69.69392247327178, Tokens per sec: 172709.217308116, Loss: 3.480555772781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10708 , TFLOPS: 67.94900289043649, Tokens per sec: 168385.11436309002, Loss: 3.5051636695861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10709 , TFLOPS: 69.59072244038323, Tokens per sec: 172453.47625819608, Loss: 3.494112253189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10710 , TFLOPS: 68.74171288538753, Tokens per sec: 170349.53705479347, Loss: 3.507089614868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10711 , TFLOPS: 69.51910738320218, Tokens per sec: 172276.00625745166, Loss: 3.477415084838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10712 , TFLOPS: 69.575393226476, Tokens per sec: 172415.48877748163, Loss: 3.545607805252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10713 , TFLOPS: 67.00645188042783, Tokens per sec: 166049.36912972335, Loss: 3.491751194000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10714 , TFLOPS: 70.56599411525109, Tokens per sec: 174870.30690356262, Loss: 3.485944986343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10715 , TFLOPS: 67.92628408651699, Tokens per sec: 168328.81466429465, Loss: 3.4920337200164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10716 , TFLOPS: 69.56750840476904, Tokens per sec: 172395.94932070698, Loss: 3.497422695159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10717 , TFLOPS: 68.13913728668173, Tokens per sec: 168856.28834201852, Loss: 3.5070433616638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10718 , TFLOPS: 68.6780256804627, Tokens per sec: 170191.71314526492, Loss: 3.479186773300171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10719 , TFLOPS: 68.7705409040076, Tokens per sec: 170420.9760606024, Loss: 3.5075197219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10720 , TFLOPS: 68.8502846752326, Tokens per sec: 170618.58991019952, Loss: 3.484743356704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10721 , TFLOPS: 69.44368168955847, Tokens per sec: 172089.09307977644, Loss: 3.502681255340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10722 , TFLOPS: 67.86152414361243, Tokens per sec: 168168.33239188054, Loss: 3.5033249855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10723 , TFLOPS: 67.90703797500088, Tokens per sec: 168281.1206798235, Loss: 3.4901626110076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10724 , TFLOPS: 70.58995206796268, Tokens per sec: 174929.67734956832, Loss: 3.5112557411193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10725 , TFLOPS: 67.016322805774, Tokens per sec: 166073.8303701034, Loss: 3.4930930137634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10726 , TFLOPS: 68.78318415986405, Tokens per sec: 170452.3074413843, Loss: 3.4806041717529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10727 , TFLOPS: 69.72269366938575, Tokens per sec: 172780.5155014953, Loss: 3.509214162826538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10728 , TFLOPS: 67.79319277033974, Tokens per sec: 167998.99972158865, Loss: 3.5019962787628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10729 , TFLOPS: 69.9225024507311, Tokens per sec: 173275.66367242343, Loss: 3.490412712097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10730 , TFLOPS: 68.24221877864014, Tokens per sec: 169111.73563445522, Loss: 3.508708953857422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10731 , TFLOPS: 70.41326417054682, Tokens per sec: 174491.82527599315, Loss: 3.5140085220336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10732 , TFLOPS: 67.69682769131872, Tokens per sec: 167760.19644028475, Loss: 3.510352611541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10733 , TFLOPS: 67.43116867526305, Tokens per sec: 167101.86413374232, Loss: 3.475649356842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10734 , TFLOPS: 70.20016082042709, Tokens per sec: 173963.7316991245, Loss: 3.502507448196411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10735 , TFLOPS: 66.11259430350978, Tokens per sec: 163834.2915875797, Loss: 3.5011394023895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10736 , TFLOPS: 69.04566161821654, Tokens per sec: 171102.75549746075, Loss: 3.5024237632751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10737 , TFLOPS: 67.91384437033176, Tokens per sec: 168297.9876772398, Loss: 3.501077890396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10738 , TFLOPS: 68.08722301067927, Tokens per sec: 168727.63904725484, Loss: 3.511110782623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10739 , TFLOPS: 67.90232258886994, Tokens per sec: 168269.43543354838, Loss: 3.4845752716064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10740 , TFLOPS: 67.98959441847414, Tokens per sec: 168485.70464109327, Loss: 3.485894203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10741 , TFLOPS: 70.20489997115243, Tokens per sec: 173975.47583668245, Loss: 3.5236616134643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10742 , TFLOPS: 67.29534072920315, Tokens per sec: 166765.2675207224, Loss: 3.484355926513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10743 , TFLOPS: 66.81232457252, Tokens per sec: 165568.30027583943, Loss: 3.475494384765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10744 , TFLOPS: 70.48889796883901, Tokens per sec: 174679.25415991186, Loss: 3.5085883140563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10745 , TFLOPS: 68.67314310544727, Tokens per sec: 170179.61358651664, Loss: 3.4634151458740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10746 , TFLOPS: 68.91737859304233, Tokens per sec: 170784.85602953722, Loss: 3.494123697280884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10747 , TFLOPS: 68.72990194874765, Tokens per sec: 170320.26825271777, Loss: 3.50457763671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10748 , TFLOPS: 69.55315693777588, Tokens per sec: 172360.38480455335, Loss: 3.52087664604187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10749 , TFLOPS: 69.08567257386463, Tokens per sec: 171201.90705312733, Loss: 3.4862449169158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10750 , TFLOPS: 66.80345086310642, Tokens per sec: 165546.31024639276, Loss: 3.525932788848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10751 , TFLOPS: 70.51381801885083, Tokens per sec: 174741.00878901128, Loss: 3.5137667655944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10752 , TFLOPS: 68.54144236237845, Tokens per sec: 169853.24463715762, Loss: 3.487349271774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10753 , TFLOPS: 68.52711950125547, Tokens per sec: 169817.75100950146, Loss: 3.5002052783966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10754 , TFLOPS: 68.92120522730124, Tokens per sec: 170794.3388507694, Loss: 3.4909188747406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10755 , TFLOPS: 67.91496027148533, Tokens per sec: 168300.753003225, Loss: 3.4813337326049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10756 , TFLOPS: 69.53936289945723, Tokens per sec: 172326.20165806098, Loss: 3.5194311141967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10757 , TFLOPS: 68.08835522102372, Tokens per sec: 168730.44478921164, Loss: 3.5227303504943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10758 , TFLOPS: 67.06602704539263, Tokens per sec: 166197.00295125233, Loss: 3.4931836128234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10759 , TFLOPS: 68.02280526881184, Tokens per sec: 168568.0047867073, Loss: 3.486349582672119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10760 , TFLOPS: 67.9760468032518, Tokens per sec: 168452.1321581793, Loss: 3.4869608879089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10761 , TFLOPS: 70.27505588034087, Tokens per sec: 174149.32990796276, Loss: 3.485405683517456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10762 , TFLOPS: 66.2895895493735, Tokens per sec: 164272.90530446594, Loss: 3.490403413772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10763 , TFLOPS: 68.18353255441284, Tokens per sec: 168966.3047059987, Loss: 3.5059525966644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10764 , TFLOPS: 69.41719590564024, Tokens per sec: 172023.45838957714, Loss: 3.477287769317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10765 , TFLOPS: 67.16133713435285, Tokens per sec: 166433.1918509688, Loss: 3.4879651069641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10766 , TFLOPS: 68.6274948772297, Tokens per sec: 170066.49224842605, Loss: 3.479159116744995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10767 , TFLOPS: 68.68934158051147, Tokens per sec: 170219.7551921351, Loss: 3.505129337310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10768 , TFLOPS: 70.53974759001207, Tokens per sec: 174805.2651227269, Loss: 3.506627321243286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10769 , TFLOPS: 67.01245051100655, Tokens per sec: 166064.23439710113, Loss: 3.4883079528808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10770 , TFLOPS: 67.77989765843078, Tokens per sec: 167966.05296970077, Loss: 3.4848060607910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10771 , TFLOPS: 70.55264757080617, Tokens per sec: 174837.2326961849, Loss: 3.510951519012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10772 , TFLOPS: 67.61694249961702, Tokens per sec: 167562.23213516764, Loss: 3.4867944717407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10773 , TFLOPS: 68.60356050252607, Tokens per sec: 170007.18023132143, Loss: 3.5059611797332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10774 , TFLOPS: 69.63414381556113, Tokens per sec: 172561.07920914626, Loss: 3.513038396835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10775 , TFLOPS: 67.78326727855469, Tokens per sec: 167974.4032595621, Loss: 3.48392391204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10776 , TFLOPS: 69.49674062015806, Tokens per sec: 172220.57895472596, Loss: 3.499244213104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10777 , TFLOPS: 68.58368907879304, Tokens per sec: 169957.93665429295, Loss: 3.4704790115356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10778 , TFLOPS: 68.22048885296618, Tokens per sec: 169057.88648488626, Loss: 3.5060319900512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10779 , TFLOPS: 69.63425241900914, Tokens per sec: 172561.3483404541, Loss: 3.5289547443389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10780 , TFLOPS: 66.79698508007644, Tokens per sec: 165530.2873237797, Loss: 3.524040460586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10781 , TFLOPS: 70.5746652375502, Tokens per sec: 174891.7948998228, Loss: 3.505166530609131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10782 , TFLOPS: 67.90047502460183, Tokens per sec: 168264.85696576603, Loss: 3.4940505027770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10783 , TFLOPS: 68.7342411778396, Tokens per sec: 170331.02134041316, Loss: 3.485461473464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10784 , TFLOPS: 68.60883725995775, Tokens per sec: 170020.25661751998, Loss: 3.488837718963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10785 , TFLOPS: 68.64371620606889, Tokens per sec: 170106.69048821647, Loss: 3.4961097240448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10786 , TFLOPS: 67.96450950078757, Tokens per sec: 168423.54145173414, Loss: 3.4890315532684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10787 , TFLOPS: 67.08744920081907, Tokens per sec: 166250.0894122443, Loss: 3.514700412750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10788 , TFLOPS: 70.23249921025699, Tokens per sec: 174043.8697914907, Loss: 3.498683452606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10789 , TFLOPS: 67.10055996940243, Tokens per sec: 166282.57934106424, Loss: 3.4957492351531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10790 , TFLOPS: 67.00656067668058, Tokens per sec: 166049.6387388226, Loss: 3.4940879344940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10791 , TFLOPS: 68.98299027839418, Tokens per sec: 170947.44901356293, Loss: 3.4942963123321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10792 , TFLOPS: 67.87353435692013, Tokens per sec: 168198.09502349122, Loss: 3.49969744682312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10793 , TFLOPS: 68.90251035911525, Tokens per sec: 170748.01090799476, Loss: 3.4985170364379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10794 , TFLOPS: 68.50112453314715, Tokens per sec: 169753.33261494694, Loss: 3.473585844039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10795 , TFLOPS: 66.77905354153556, Tokens per sec: 165485.85099595337, Loss: 3.516364812850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10796 , TFLOPS: 69.45056121370753, Tokens per sec: 172106.14129845993, Loss: 3.4824047088623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10797 , TFLOPS: 67.16717128248258, Tokens per sec: 166447.64951271875, Loss: 3.4703681468963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10798 , TFLOPS: 68.98834933692966, Tokens per sec: 170960.72935096288, Loss: 3.505279779434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10799 , TFLOPS: 67.99410499634469, Tokens per sec: 168496.88234993763, Loss: 3.503146171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10800 , TFLOPS: 66.95790499113409, Tokens per sec: 165929.06459017214, Loss: 3.4887585639953613 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/model.pt +[rank0]:[2024-08-25 19:34:57,913] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00912566899205558, 'preprocessing_with_comm': 0.002371141003095545, 'state_converting': 1.07703184899583, : 1.0905577990051825}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800, took 5.38s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0008800 + eval ppl=25.893909454345703, eval loss=3.2540078163146973 +------------------------------------------------------------------ +iteration: 10801 , TFLOPS: 66.63477341891304, Tokens per sec: 165128.3089583272, Loss: 3.524538516998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10802 , TFLOPS: 65.2416312574913, Tokens per sec: 161675.94921504598, Loss: 3.513967514038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10803 , TFLOPS: 68.67842086222758, Tokens per sec: 170192.6924492112, Loss: 3.4998319149017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10804 , TFLOPS: 69.88536177841472, Tokens per sec: 173183.62499504062, Loss: 3.498748302459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10805 , TFLOPS: 67.62402301703194, Tokens per sec: 167579.77843730518, Loss: 3.4973602294921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10806 , TFLOPS: 68.66701110121132, Tokens per sec: 170164.41780452395, Loss: 3.5146892070770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10807 , TFLOPS: 68.68856161954515, Tokens per sec: 170217.82236294076, Loss: 3.505330801010132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10808 , TFLOPS: 69.57439737286725, Tokens per sec: 172413.02094253685, Loss: 3.5162391662597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10809 , TFLOPS: 67.69175027805252, Tokens per sec: 167747.61405088787, Loss: 3.50618839263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10810 , TFLOPS: 67.82728706363694, Tokens per sec: 168083.48913617554, Loss: 3.48563551902771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10811 , TFLOPS: 70.55607624499541, Tokens per sec: 174845.72932852546, Loss: 3.504270553588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10812 , TFLOPS: 68.57040958605745, Tokens per sec: 169925.02860843728, Loss: 3.4987754821777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10813 , TFLOPS: 67.67168380371307, Tokens per sec: 167697.8871169701, Loss: 3.5021815299987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10814 , TFLOPS: 69.7083144919213, Tokens per sec: 172744.88231574127, Loss: 3.491488218307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10815 , TFLOPS: 69.52159628760667, Tokens per sec: 172282.17403674143, Loss: 3.500903606414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10816 , TFLOPS: 67.76744439306991, Tokens per sec: 167935.1924062961, Loss: 3.472101926803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10817 , TFLOPS: 69.43161526589792, Tokens per sec: 172059.19115271932, Loss: 3.5199074745178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10818 , TFLOPS: 69.47875939782777, Tokens per sec: 172176.01950499634, Loss: 3.473576068878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10819 , TFLOPS: 67.90490443195176, Tokens per sec: 168275.83352511862, Loss: 3.493856906890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10820 , TFLOPS: 67.84355799489416, Tokens per sec: 168123.810296222, Loss: 3.50360107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10821 , TFLOPS: 70.5899402279381, Tokens per sec: 174929.6480086831, Loss: 3.474482774734497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10822 , TFLOPS: 67.78303658560635, Tokens per sec: 167973.8315770232, Loss: 3.499070167541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10823 , TFLOPS: 69.64932485021934, Tokens per sec: 172598.69948535832, Loss: 3.4601352214813232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10824 , TFLOPS: 68.94544875809054, Tokens per sec: 170854.4169907106, Loss: 3.489804744720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10825 , TFLOPS: 68.76345467770125, Tokens per sec: 170403.41561120297, Loss: 3.4824163913726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10826 , TFLOPS: 68.87018511792863, Tokens per sec: 170667.90539941503, Loss: 3.470083713531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10827 , TFLOPS: 68.74458696505569, Tokens per sec: 170356.65934081716, Loss: 3.5023882389068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10828 , TFLOPS: 69.52735560651952, Tokens per sec: 172296.4462634488, Loss: 3.503028392791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10829 , TFLOPS: 68.04167380341964, Tokens per sec: 168614.76309400608, Loss: 3.4798097610473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10830 , TFLOPS: 67.8559120695845, Tokens per sec: 168154.42505421833, Loss: 3.4901046752929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10831 , TFLOPS: 70.54905226025167, Tokens per sec: 174828.32312057467, Loss: 3.508673667907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10832 , TFLOPS: 66.68103459273853, Tokens per sec: 165242.94924315574, Loss: 3.498150587081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10833 , TFLOPS: 67.81267345367262, Tokens per sec: 168047.2750008619, Loss: 3.49609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10834 , TFLOPS: 69.6519927831854, Tokens per sec: 172605.3109171453, Loss: 3.5124423503875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10835 , TFLOPS: 68.29495231281052, Tokens per sec: 169242.41514120775, Loss: 3.5152249336242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10836 , TFLOPS: 67.88487325893392, Tokens per sec: 168226.1940717646, Loss: 3.485511064529419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10837 , TFLOPS: 69.72395279511895, Tokens per sec: 172783.63575376634, Loss: 3.471728801727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10838 , TFLOPS: 69.53071956488066, Tokens per sec: 172304.78252283833, Loss: 3.506842613220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10839 , TFLOPS: 67.89509584091617, Tokens per sec: 168251.52675602693, Loss: 3.5130772590637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10840 , TFLOPS: 68.94708620215788, Tokens per sec: 170858.47475748888, Loss: 3.493989944458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10841 , TFLOPS: 69.85507086404071, Tokens per sec: 173108.56077240215, Loss: 3.4922995567321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10842 , TFLOPS: 68.48432211004352, Tokens per sec: 169711.69435955642, Loss: 3.4852828979492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10843 , TFLOPS: 69.00222111499602, Tokens per sec: 170995.1051451138, Loss: 3.5049326419830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10844 , TFLOPS: 69.67461867519269, Tokens per sec: 172661.38036998714, Loss: 3.524259567260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10845 , TFLOPS: 69.52098793223976, Tokens per sec: 172280.66646512633, Loss: 3.4826478958129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10846 , TFLOPS: 67.37756713365907, Tokens per sec: 166969.0336089504, Loss: 3.4943511486053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10847 , TFLOPS: 68.74509054596652, Tokens per sec: 170357.90726978824, Loss: 3.484557867050171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10848 , TFLOPS: 70.63263905657882, Tokens per sec: 175035.46040971912, Loss: 3.4907655715942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10849 , TFLOPS: 67.48965124071582, Tokens per sec: 167246.79037332072, Loss: 3.5089845657348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10850 , TFLOPS: 67.8564902326271, Tokens per sec: 168155.8578059278, Loss: 3.503525495529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10851 , TFLOPS: 70.61488858266178, Tokens per sec: 174991.47277997655, Loss: 3.4781384468078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10852 , TFLOPS: 68.91261109492517, Tokens per sec: 170773.04164401963, Loss: 3.4922780990600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10853 , TFLOPS: 68.83910112113017, Tokens per sec: 170590.87583116355, Loss: 3.4948439598083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10854 , TFLOPS: 68.14027194324963, Tokens per sec: 168859.10014598654, Loss: 3.501631736755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10855 , TFLOPS: 69.52586533660777, Tokens per sec: 172292.75321044071, Loss: 3.5018837451934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10856 , TFLOPS: 68.62574038456465, Tokens per sec: 170062.14442232877, Loss: 3.4937069416046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10857 , TFLOPS: 68.17011469005824, Tokens per sec: 168933.0537600282, Loss: 3.507310152053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10858 , TFLOPS: 69.6606039249078, Tokens per sec: 172626.65027492898, Loss: 3.4897797107696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10859 , TFLOPS: 69.77520710078998, Tokens per sec: 172910.64956934704, Loss: 3.4918372631073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10860 , TFLOPS: 68.23100749774225, Tokens per sec: 169083.9528453657, Loss: 3.489633083343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10861 , TFLOPS: 68.70563649369277, Tokens per sec: 170260.1357820312, Loss: 3.4989287853240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10862 , TFLOPS: 69.50787093992699, Tokens per sec: 172248.1611132193, Loss: 3.4911746978759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10863 , TFLOPS: 68.2004148422534, Tokens per sec: 169008.14087500513, Loss: 3.476565361022949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10864 , TFLOPS: 69.7124781589625, Tokens per sec: 172755.20034133515, Loss: 3.488119602203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10865 , TFLOPS: 68.64776798575109, Tokens per sec: 170116.7312446093, Loss: 3.4920730590820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10866 , TFLOPS: 67.9750041150976, Tokens per sec: 168449.54826207162, Loss: 3.503476142883301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10867 , TFLOPS: 68.95236381875812, Tokens per sec: 170871.5532728017, Loss: 3.497865915298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10868 , TFLOPS: 70.58502107334913, Tokens per sec: 174917.45780172205, Loss: 3.4934728145599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10869 , TFLOPS: 66.8005095847919, Tokens per sec: 165539.02143472087, Loss: 3.494737148284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10870 , TFLOPS: 68.87339560445747, Tokens per sec: 170675.86133869778, Loss: 3.5017037391662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10871 , TFLOPS: 69.94935332076624, Tokens per sec: 173342.203086238, Loss: 3.4964871406555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10872 , TFLOPS: 67.87697936113756, Tokens per sec: 168206.63212344056, Loss: 3.501016139984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10873 , TFLOPS: 68.8175906765311, Tokens per sec: 170537.57057987596, Loss: 3.490271806716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10874 , TFLOPS: 68.85181180128318, Tokens per sec: 170622.3742967797, Loss: 3.503688097000122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10875 , TFLOPS: 69.54521244967181, Tokens per sec: 172340.69748787384, Loss: 3.447059392929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10876 , TFLOPS: 68.56444822247917, Tokens per sec: 169910.25569279192, Loss: 3.502871036529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10877 , TFLOPS: 68.62326565753837, Tokens per sec: 170056.01177614395, Loss: 3.474795341491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10878 , TFLOPS: 70.59495044374073, Tokens per sec: 174942.06387536286, Loss: 3.5050435066223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10879 , TFLOPS: 68.10272949465316, Tokens per sec: 168766.06582272236, Loss: 3.5062739849090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10880 , TFLOPS: 68.0202803361192, Tokens per sec: 168561.74772535064, Loss: 3.5159192085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10881 , TFLOPS: 69.72726016315906, Tokens per sec: 172791.83177610638, Loss: 3.5201735496520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10882 , TFLOPS: 69.55977748074824, Tokens per sec: 172376.79123935182, Loss: 3.4892349243164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10883 , TFLOPS: 67.78539998891603, Tokens per sec: 167979.68835077493, Loss: 3.4953219890594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10884 , TFLOPS: 69.65589674127668, Tokens per sec: 172614.98535535196, Loss: 3.4815754890441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10885 , TFLOPS: 69.65617230563993, Tokens per sec: 172615.66823420004, Loss: 3.4830312728881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10886 , TFLOPS: 68.20914409903473, Tokens per sec: 169029.7729349163, Loss: 3.491119861602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10887 , TFLOPS: 67.99203982335628, Tokens per sec: 168491.7646237748, Loss: 3.4876115322113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10888 , TFLOPS: 70.5215393301471, Tokens per sec: 174760.14304897608, Loss: 3.5134758949279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10889 , TFLOPS: 68.7294063079917, Tokens per sec: 170319.04000032516, Loss: 3.4848103523254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10890 , TFLOPS: 69.01719711514076, Tokens per sec: 171032.2173232151, Loss: 3.484832525253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10891 , TFLOPS: 68.16206401893889, Tokens per sec: 168913.10331013985, Loss: 3.4864251613616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10892 , TFLOPS: 69.58537419327176, Tokens per sec: 172440.22271269583, Loss: 3.5058209896087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10893 , TFLOPS: 68.27513066711155, Tokens per sec: 169193.29491963124, Loss: 3.4922518730163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10894 , TFLOPS: 67.77676919113179, Tokens per sec: 167958.3002831047, Loss: 3.4765124320983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10895 , TFLOPS: 70.5497910935699, Tokens per sec: 174830.15403092644, Loss: 3.529698371887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10896 , TFLOPS: 68.02292848988608, Tokens per sec: 168568.31014210318, Loss: 3.4902188777923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10897 , TFLOPS: 68.73567754516144, Tokens per sec: 170334.5808168656, Loss: 3.4943459033966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10898 , TFLOPS: 69.577779097816, Tokens per sec: 172421.40122948762, Loss: 3.5285580158233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10899 , TFLOPS: 68.82571779476547, Tokens per sec: 170557.7104741367, Loss: 3.5008177757263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10900 , TFLOPS: 67.98372098059164, Tokens per sec: 168471.14961500783, Loss: 3.502702236175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10901 , TFLOPS: 69.68554766797217, Tokens per sec: 172688.46361802262, Loss: 3.4898717403411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10902 , TFLOPS: 68.65016832676397, Tokens per sec: 170122.67955405868, Loss: 3.50565767288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10903 , TFLOPS: 68.01945964545828, Tokens per sec: 168559.7139634863, Loss: 3.490298271179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10904 , TFLOPS: 69.77557613881059, Tokens per sec: 172911.564086215, Loss: 3.483996629714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10905 , TFLOPS: 69.58857915552545, Tokens per sec: 172448.16496221616, Loss: 3.4910616874694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10906 , TFLOPS: 67.77449126681609, Tokens per sec: 167952.65533571632, Loss: 3.512171506881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10907 , TFLOPS: 69.13927853719628, Tokens per sec: 171334.7485354475, Loss: 3.4924263954162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10908 , TFLOPS: 69.88363611043138, Tokens per sec: 173179.34859967986, Loss: 3.5025925636291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10909 , TFLOPS: 68.06306449210625, Tokens per sec: 168667.77157695036, Loss: 3.490936756134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10910 , TFLOPS: 68.76597303815069, Tokens per sec: 170409.656385817, Loss: 3.5177948474884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10911 , TFLOPS: 68.87434454928473, Tokens per sec: 170678.21292851464, Loss: 3.506986618041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10912 , TFLOPS: 69.51134701050891, Tokens per sec: 172256.7751990421, Loss: 3.489964008331299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10913 , TFLOPS: 66.91540190319621, Tokens per sec: 165823.7372561604, Loss: 3.500178813934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10914 , TFLOPS: 68.66370655262146, Tokens per sec: 170156.2287690922, Loss: 3.4934723377227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10915 , TFLOPS: 70.61900196204118, Tokens per sec: 175001.66618720483, Loss: 3.4712462425231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10916 , TFLOPS: 67.12835714283408, Tokens per sec: 166351.4637989393, Loss: 3.490602970123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10917 , TFLOPS: 68.67430749092175, Tokens per sec: 170182.49906199006, Loss: 3.481471061706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10918 , TFLOPS: 70.53784148324908, Tokens per sec: 174800.5415801932, Loss: 3.491133689880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10919 , TFLOPS: 68.7598027719458, Tokens per sec: 170394.36578645036, Loss: 3.499919891357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10920 , TFLOPS: 67.8702060019712, Tokens per sec: 168189.84699327918, Loss: 3.4884493350982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10921 , TFLOPS: 69.74459050478511, Tokens per sec: 172834.7782717505, Loss: 3.4934303760528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10922 , TFLOPS: 69.68784625349967, Tokens per sec: 172694.15976616988, Loss: 3.5047926902770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10923 , TFLOPS: 68.84635335089678, Tokens per sec: 170608.84765542424, Loss: 3.4813966751098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10924 , TFLOPS: 68.03726600347748, Tokens per sec: 168603.84008019016, Loss: 3.5184719562530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10925 , TFLOPS: 69.58102035062228, Tokens per sec: 172429.43341099503, Loss: 3.4774107933044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10926 , TFLOPS: 69.65790545459042, Tokens per sec: 172619.96316822147, Loss: 3.50468111038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10927 , TFLOPS: 68.19543641369177, Tokens per sec: 168995.80378061027, Loss: 3.508683204650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10928 , TFLOPS: 68.76485609390772, Tokens per sec: 170406.8884749414, Loss: 3.496910333633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10929 , TFLOPS: 69.68193781633698, Tokens per sec: 172679.51800801264, Loss: 3.4834656715393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10930 , TFLOPS: 68.26092189740332, Tokens per sec: 169158.084023069, Loss: 3.4999356269836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10931 , TFLOPS: 68.03471362023707, Tokens per sec: 168597.51499335218, Loss: 3.5007741451263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10932 , TFLOPS: 69.52973097060895, Tokens per sec: 172302.33267733039, Loss: 3.4915308952331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10933 , TFLOPS: 67.7864173925, Tokens per sec: 167982.20958893228, Loss: 3.5186522006988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10934 , TFLOPS: 68.80519904630479, Tokens per sec: 170506.8627551244, Loss: 3.5268630981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10935 , TFLOPS: 69.57602710269865, Tokens per sec: 172417.05959258857, Loss: 3.492612361907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10936 , TFLOPS: 67.97636969376498, Tokens per sec: 168452.9323164403, Loss: 3.4775872230529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10937 , TFLOPS: 68.65472365111417, Tokens per sec: 170133.96814960887, Loss: 3.4917194843292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10938 , TFLOPS: 70.58918709485992, Tokens per sec: 174927.78166195092, Loss: 3.5008256435394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10939 , TFLOPS: 67.68300612308236, Tokens per sec: 167725.94507162363, Loss: 3.4945478439331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10940 , TFLOPS: 68.75642588005229, Tokens per sec: 170385.9974763429, Loss: 3.4926986694335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10941 , TFLOPS: 69.17850558398568, Tokens per sec: 171431.9574786076, Loss: 3.5069518089294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10942 , TFLOPS: 69.36588000115178, Tokens per sec: 171896.29192534115, Loss: 3.5066897869110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10943 , TFLOPS: 68.03917185642824, Tokens per sec: 168608.5629937492, Loss: 3.487224817276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10944 , TFLOPS: 68.70167926149217, Tokens per sec: 170250.32932471304, Loss: 3.465684175491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10945 , TFLOPS: 70.54663172607738, Tokens per sec: 174822.32477024692, Loss: 3.4875969886779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10946 , TFLOPS: 68.65819065573284, Tokens per sec: 170142.55976897597, Loss: 3.5041561126708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10947 , TFLOPS: 67.5120739964194, Tokens per sec: 167302.35643202465, Loss: 3.5021629333496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10948 , TFLOPS: 69.71201585745357, Tokens per sec: 172754.0547072693, Loss: 3.462517499923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10949 , TFLOPS: 69.6412870555655, Tokens per sec: 172578.7809447699, Loss: 3.494447708129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10950 , TFLOPS: 67.12229964596074, Tokens per sec: 166336.4526543988, Loss: 3.4811160564422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10951 , TFLOPS: 68.69392715077892, Tokens per sec: 170231.1187404001, Loss: 3.495816707611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10952 , TFLOPS: 69.30485538894165, Tokens per sec: 171745.06621386847, Loss: 3.4957034587860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10953 , TFLOPS: 67.68252654807237, Tokens per sec: 167724.75663191933, Loss: 3.502957820892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10954 , TFLOPS: 68.35668259450281, Tokens per sec: 169395.38957939314, Loss: 3.475658416748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10955 , TFLOPS: 70.51719158099068, Tokens per sec: 174749.36884762236, Loss: 3.4742379188537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10956 , TFLOPS: 68.08413476978754, Tokens per sec: 168719.9860461273, Loss: 3.4929378032684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10957 , TFLOPS: 68.65943071699083, Tokens per sec: 170145.63277737732, Loss: 3.5147316455841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10958 , TFLOPS: 69.71497095954297, Tokens per sec: 172761.3777757782, Loss: 3.5155389308929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10959 , TFLOPS: 68.69208471569516, Tokens per sec: 170226.55298330155, Loss: 3.4892830848693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10960 , TFLOPS: 68.77753054124955, Tokens per sec: 170438.29715166005, Loss: 3.499674081802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10961 , TFLOPS: 67.93117810044403, Tokens per sec: 168340.942569926, Loss: 3.4862096309661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10962 , TFLOPS: 70.55729780773056, Tokens per sec: 174848.7564955494, Loss: 3.4940688610076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10963 , TFLOPS: 68.33614239148733, Tokens per sec: 169344.48869363064, Loss: 3.4734835624694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10964 , TFLOPS: 68.69327686512125, Tokens per sec: 170229.5072609055, Loss: 3.5186448097229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10965 , TFLOPS: 69.49761761719995, Tokens per sec: 172222.75225000497, Loss: 3.50228214263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10966 , TFLOPS: 68.62969311668041, Tokens per sec: 170071.939727941, Loss: 3.4950833320617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10967 , TFLOPS: 67.93134432654448, Tokens per sec: 168341.35449651422, Loss: 3.4880595207214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10968 , TFLOPS: 68.7400024383267, Tokens per sec: 170345.2983785554, Loss: 3.514918565750122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10969 , TFLOPS: 69.68564742465792, Tokens per sec: 172688.710826079, Loss: 3.49981951713562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10970 , TFLOPS: 68.04613409752524, Tokens per sec: 168625.81619414172, Loss: 3.5118257999420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10971 , TFLOPS: 68.78915919519056, Tokens per sec: 170467.11423712788, Loss: 3.505610466003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10972 , TFLOPS: 68.48481810239132, Tokens per sec: 169712.92348323233, Loss: 3.5072383880615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10973 , TFLOPS: 68.7020856543781, Tokens per sec: 170251.33641105224, Loss: 3.5071237087249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10974 , TFLOPS: 68.08382664059584, Tokens per sec: 168719.22246804705, Loss: 3.488248109817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10975 , TFLOPS: 69.89227367173838, Tokens per sec: 173200.75342810454, Loss: 3.505265235900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10976 , TFLOPS: 67.93588454999455, Tokens per sec: 168352.60567037042, Loss: 3.4997482299804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10977 , TFLOPS: 69.03644602309824, Tokens per sec: 171079.91823757597, Loss: 3.4876112937927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10978 , TFLOPS: 69.6137404776384, Tokens per sec: 172510.5174901605, Loss: 3.5096123218536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10979 , TFLOPS: 69.60798582905065, Tokens per sec: 172496.2568370332, Loss: 3.4919543266296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10980 , TFLOPS: 67.27171820914876, Tokens per sec: 166706.72831379226, Loss: 3.49129056930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10981 , TFLOPS: 68.90891680865356, Tokens per sec: 170763.88679567937, Loss: 3.5118205547332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10982 , TFLOPS: 70.57360114628025, Tokens per sec: 174889.15796443654, Loss: 3.5034215450286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10983 , TFLOPS: 66.98479683037951, Tokens per sec: 165995.70553020292, Loss: 3.4730112552642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10984 , TFLOPS: 68.7178679059735, Tokens per sec: 170290.44656906417, Loss: 3.514193534851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10985 , TFLOPS: 70.60785542658563, Tokens per sec: 174974.04384445373, Loss: 3.47713041305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10986 , TFLOPS: 68.78855851405518, Tokens per sec: 170465.6256831039, Loss: 3.490262985229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10987 , TFLOPS: 67.84035237531643, Tokens per sec: 168115.86641778023, Loss: 3.5045108795166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10988 , TFLOPS: 69.76096174935044, Tokens per sec: 172875.34801922485, Loss: 3.519289970397949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10989 , TFLOPS: 69.58865253018664, Tokens per sec: 172448.3467927095, Loss: 3.5033440589904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10990 , TFLOPS: 67.77224312190486, Tokens per sec: 167947.08418497271, Loss: 3.502357006072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10991 , TFLOPS: 68.0279384827922, Tokens per sec: 168580.7254564208, Loss: 3.4744694232940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10992 , TFLOPS: 70.52111802201641, Tokens per sec: 174759.09900101734, Loss: 3.4913723468780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10993 , TFLOPS: 69.00469155599022, Tokens per sec: 171001.227170039, Loss: 3.485469102859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10994 , TFLOPS: 67.33941181742892, Tokens per sec: 166874.48053217705, Loss: 3.4691364765167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10995 , TFLOPS: 69.58734882304928, Tokens per sec: 172445.1160628079, Loss: 3.5056145191192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10996 , TFLOPS: 69.54365327728027, Tokens per sec: 172336.8336869306, Loss: 3.496081829071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10997 , TFLOPS: 68.14063849193664, Tokens per sec: 168860.00849401162, Loss: 3.4984047412872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10998 , TFLOPS: 68.68887808981535, Tokens per sec: 170218.60661113236, Loss: 3.4997148513793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10999 , TFLOPS: 69.56948668281919, Tokens per sec: 172400.85171164057, Loss: 3.5098016262054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11000 , TFLOPS: 68.00379203084694, Tokens per sec: 168520.8878885492, Loss: 3.490664005279541 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/model.pt +[rank0]:[2024-08-25 20:00:44,247] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006319870997685939, 'preprocessing_with_comm': 0.0014034510095370933, 'state_converting': 0.9684566100040684, : 0.9775539919937728}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000, took 4.20s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009000 + eval ppl=25.472572326660156, eval loss=3.2376022338867188 +------------------------------------------------------------------ +iteration: 11001 , TFLOPS: 62.46570589046898, Tokens per sec: 154796.90036214158, Loss: 3.4998950958251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11002 , TFLOPS: 68.93910287576219, Tokens per sec: 170838.69119524432, Loss: 3.4746923446655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11003 , TFLOPS: 68.7782946988171, Tokens per sec: 170440.19081829136, Loss: 3.4742043018341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11004 , TFLOPS: 68.69745474600619, Tokens per sec: 170239.86050996996, Loss: 3.4831418991088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11005 , TFLOPS: 69.48638914059036, Tokens per sec: 172194.9268480476, Loss: 3.506457567214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11006 , TFLOPS: 68.14349264737356, Tokens per sec: 168867.08140559547, Loss: 3.4918158054351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11007 , TFLOPS: 68.87150904622128, Tokens per sec: 170671.18623956695, Loss: 3.486330032348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11008 , TFLOPS: 68.90678125292423, Tokens per sec: 170758.5946533312, Loss: 3.4733240604400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11009 , TFLOPS: 68.58000234570464, Tokens per sec: 169948.80052357272, Loss: 3.4869587421417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11010 , TFLOPS: 67.8126638647445, Tokens per sec: 168047.25123844174, Loss: 3.481398820877075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11011 , TFLOPS: 68.7859290223424, Tokens per sec: 170459.10951297721, Loss: 3.48313045501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11012 , TFLOPS: 69.67697469521083, Tokens per sec: 172667.2188471291, Loss: 3.4941868782043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11013 , TFLOPS: 67.9039514435058, Tokens per sec: 168273.4719147692, Loss: 3.5081892013549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11014 , TFLOPS: 68.13630886449205, Tokens per sec: 168849.27920025607, Loss: 3.4924447536468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11015 , TFLOPS: 70.59925290161046, Tokens per sec: 174952.7258399189, Loss: 3.498253345489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11016 , TFLOPS: 67.44265827309198, Tokens per sec: 167130.33662284608, Loss: 3.498145580291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11017 , TFLOPS: 69.58040204790194, Tokens per sec: 172427.90118874237, Loss: 3.510629892349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11018 , TFLOPS: 69.16435047370074, Tokens per sec: 171396.87955600736, Loss: 3.4837098121643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11019 , TFLOPS: 68.84538727267184, Tokens per sec: 170606.45360716054, Loss: 3.486809253692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11020 , TFLOPS: 67.64120171240084, Tokens per sec: 167622.34913681884, Loss: 3.5026044845581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11021 , TFLOPS: 68.81949551895866, Tokens per sec: 170542.29098924715, Loss: 3.4916534423828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11022 , TFLOPS: 70.57847441288706, Tokens per sec: 174901.23445592285, Loss: 3.497274160385132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11023 , TFLOPS: 68.08719468416842, Tokens per sec: 168727.56885104053, Loss: 3.494065284729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11024 , TFLOPS: 67.74375398953065, Tokens per sec: 167876.48497661238, Loss: 3.487907886505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11025 , TFLOPS: 70.59118934565018, Tokens per sec: 174932.7434599753, Loss: 3.5089027881622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11026 , TFLOPS: 68.90132230631129, Tokens per sec: 170745.06678227172, Loss: 3.485597610473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11027 , TFLOPS: 68.7497976472696, Tokens per sec: 170369.57198534728, Loss: 3.4905474185943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11028 , TFLOPS: 67.90131084696597, Tokens per sec: 168266.9282256578, Loss: 3.5009846687316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11029 , TFLOPS: 69.54219035148468, Tokens per sec: 172333.2083956545, Loss: 3.491532325744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11030 , TFLOPS: 67.82123821308382, Tokens per sec: 168068.499418169, Loss: 3.4752566814422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11031 , TFLOPS: 67.74330904873152, Tokens per sec: 167875.38236429787, Loss: 3.5037896633148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11032 , TFLOPS: 69.61065062319821, Tokens per sec: 172502.86049048213, Loss: 3.4901931285858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11033 , TFLOPS: 68.81550168984913, Tokens per sec: 170532.39384075714, Loss: 3.4946188926696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11034 , TFLOPS: 68.7313173269587, Tokens per sec: 170323.77571584153, Loss: 3.483107328414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11035 , TFLOPS: 68.96895887576466, Tokens per sec: 170912.67765216038, Loss: 3.501904249191284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11036 , TFLOPS: 68.69922703689679, Tokens per sec: 170244.25244203082, Loss: 3.4727017879486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11037 , TFLOPS: 68.99749451945674, Tokens per sec: 170983.39212069, Loss: 3.51059889793396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11038 , TFLOPS: 68.96978100363177, Tokens per sec: 170914.71497557993, Loss: 3.5096771717071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11039 , TFLOPS: 68.44810316341447, Tokens per sec: 169621.9398199633, Loss: 3.525810718536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11040 , TFLOPS: 68.25459568752093, Tokens per sec: 169142.4069780909, Loss: 3.4827685356140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11041 , TFLOPS: 68.83689663355469, Tokens per sec: 170585.41286810715, Loss: 3.485928535461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11042 , TFLOPS: 70.56636776597384, Tokens per sec: 174871.23285121462, Loss: 3.4709057807922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11043 , TFLOPS: 66.721413007766, Tokens per sec: 165343.01140364344, Loss: 3.5038695335388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11044 , TFLOPS: 68.79575945919613, Tokens per sec: 170483.4704184128, Loss: 3.5150203704833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11045 , TFLOPS: 69.88634247653906, Tokens per sec: 173186.05527302553, Loss: 3.4860410690307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11046 , TFLOPS: 67.80195586968708, Tokens per sec: 168020.7156471064, Loss: 3.489283561706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11047 , TFLOPS: 68.748406566348, Tokens per sec: 170366.12473358333, Loss: 3.499163866043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11048 , TFLOPS: 68.83860313439912, Tokens per sec: 170589.64176518624, Loss: 3.494241952896118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11049 , TFLOPS: 69.57989399159902, Tokens per sec: 172426.6421692567, Loss: 3.5168519020080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11050 , TFLOPS: 68.60212726933602, Tokens per sec: 170003.62852159285, Loss: 3.4949989318847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11051 , TFLOPS: 68.2099969232313, Tokens per sec: 169031.88632722176, Loss: 3.4906277656555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11052 , TFLOPS: 70.6069980957427, Tokens per sec: 174971.91928418214, Loss: 3.4938907623291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11053 , TFLOPS: 66.2713557837653, Tokens per sec: 164227.7200246737, Loss: 3.47772479057312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11054 , TFLOPS: 68.06411804747019, Tokens per sec: 168670.3824032009, Loss: 3.483959197998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11055 , TFLOPS: 69.53279631265495, Tokens per sec: 172309.92893259565, Loss: 3.4853157997131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11056 , TFLOPS: 69.59621417368963, Tokens per sec: 172467.08537829368, Loss: 3.508838415145874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11057 , TFLOPS: 65.75678017961852, Tokens per sec: 162952.5449924157, Loss: 3.5070486068725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11058 , TFLOPS: 68.79133879006044, Tokens per sec: 170472.51551331763, Loss: 3.5037035942077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11059 , TFLOPS: 69.60303851627359, Tokens per sec: 172483.99685098013, Loss: 3.491581439971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11060 , TFLOPS: 67.98683716178924, Tokens per sec: 168478.8718552912, Loss: 3.483985424041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11061 , TFLOPS: 67.90007827758976, Tokens per sec: 168263.8737829648, Loss: 3.496272087097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11062 , TFLOPS: 70.57196735676395, Tokens per sec: 174885.10925403834, Loss: 3.4886929988861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11063 , TFLOPS: 67.96395114120492, Tokens per sec: 168422.15777517998, Loss: 3.516319990158081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11064 , TFLOPS: 67.86726687680502, Tokens per sec: 168182.56351734605, Loss: 3.5151331424713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11065 , TFLOPS: 69.64549628692471, Tokens per sec: 172589.21188376317, Loss: 3.5123250484466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11066 , TFLOPS: 68.8012693462395, Tokens per sec: 170497.1245254704, Loss: 3.4801530838012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11067 , TFLOPS: 68.77594022848216, Tokens per sec: 170434.35618143503, Loss: 3.4763290882110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11068 , TFLOPS: 68.01182405273427, Tokens per sec: 168540.79212358626, Loss: 3.4828083515167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11069 , TFLOPS: 69.56543718801964, Tokens per sec: 172390.81661743863, Loss: 3.5060935020446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11070 , TFLOPS: 68.85517940750964, Tokens per sec: 170630.71959598133, Loss: 3.4935481548309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11071 , TFLOPS: 67.92331241765764, Tokens per sec: 168321.45054150483, Loss: 3.4821085929870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11072 , TFLOPS: 69.60675503891595, Tokens per sec: 172493.20680349667, Loss: 3.499096632003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11073 , TFLOPS: 68.71597169085473, Tokens per sec: 170285.74753911453, Loss: 3.4657416343688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11074 , TFLOPS: 67.86584080652523, Tokens per sec: 168179.0295580981, Loss: 3.5091376304626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11075 , TFLOPS: 69.64499437979727, Tokens per sec: 172587.96810261172, Loss: 3.49630069732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11076 , TFLOPS: 68.56191286801969, Tokens per sec: 169903.97280514895, Loss: 3.5259387493133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11077 , TFLOPS: 67.90146003247072, Tokens per sec: 168267.29792377268, Loss: 3.472777843475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11078 , TFLOPS: 68.78683208808634, Tokens per sec: 170461.3474093716, Loss: 3.4860870838165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11079 , TFLOPS: 68.67085470402687, Tokens per sec: 170173.94267570868, Loss: 3.4907572269439697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11080 , TFLOPS: 67.91637365681909, Tokens per sec: 168304.25552777929, Loss: 3.4884836673736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11081 , TFLOPS: 68.80974585942862, Tokens per sec: 170518.130258918, Loss: 3.5083060264587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11082 , TFLOPS: 70.6093144320156, Tokens per sec: 174977.65942063188, Loss: 3.478834390640259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11083 , TFLOPS: 67.093982768703, Tokens per sec: 166266.28031319258, Loss: 3.508837938308716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11084 , TFLOPS: 69.67423743622639, Tokens per sec: 172660.4356178317, Loss: 3.5071616172790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11085 , TFLOPS: 69.01750367392233, Tokens per sec: 171032.97700964607, Loss: 3.4734766483306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11086 , TFLOPS: 68.68066033783094, Tokens per sec: 170198.24211645493, Loss: 3.4843392372131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11087 , TFLOPS: 67.83992006448335, Tokens per sec: 168114.7951039129, Loss: 3.4786081314086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11088 , TFLOPS: 68.88315399020037, Tokens per sec: 170700.04369354362, Loss: 3.5062239170074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11089 , TFLOPS: 70.54767779176302, Tokens per sec: 174824.9170362496, Loss: 3.4967312812805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11090 , TFLOPS: 68.2806709729615, Tokens per sec: 169207.02440784246, Loss: 3.506511688232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11091 , TFLOPS: 66.93314347394887, Tokens per sec: 165867.70282288088, Loss: 3.4880683422088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11092 , TFLOPS: 70.57561646499548, Tokens per sec: 174894.15214621855, Loss: 3.50266432762146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11093 , TFLOPS: 68.62028329088011, Tokens per sec: 170048.62114302997, Loss: 3.505540609359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11094 , TFLOPS: 68.54832739890888, Tokens per sec: 169870.30651612944, Loss: 3.5039494037628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11095 , TFLOPS: 69.09109763983348, Tokens per sec: 171215.35096421847, Loss: 3.4869229793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11096 , TFLOPS: 69.45280263093272, Tokens per sec: 172111.69577725648, Loss: 3.490126609802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11097 , TFLOPS: 68.09575723583788, Tokens per sec: 168748.7877973204, Loss: 3.4883201122283936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11098 , TFLOPS: 67.75590259818433, Tokens per sec: 167906.59056713598, Loss: 3.4855825901031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11099 , TFLOPS: 69.62597538119239, Tokens per sec: 172540.83692895318, Loss: 3.5038328170776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11100 , TFLOPS: 68.74708035009746, Tokens per sec: 170362.8382236197, Loss: 3.4941117763519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11101 , TFLOPS: 68.27026105080282, Tokens per sec: 169181.2274739856, Loss: 3.4990129470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11102 , TFLOPS: 69.90127015268332, Tokens per sec: 173223.0476989303, Loss: 3.4940176010131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11103 , TFLOPS: 68.83987394242367, Tokens per sec: 170592.7909674621, Loss: 3.5117831230163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11104 , TFLOPS: 69.09347427020452, Tokens per sec: 171221.24051608538, Loss: 3.51204252243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11105 , TFLOPS: 67.86909615589849, Tokens per sec: 168187.0966724514, Loss: 3.4818429946899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11106 , TFLOPS: 69.53599861612062, Tokens per sec: 172317.86459334643, Loss: 3.4999468326568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11107 , TFLOPS: 68.14187519711409, Tokens per sec: 168863.07318567566, Loss: 3.461333990097046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11108 , TFLOPS: 68.84823798519223, Tokens per sec: 170613.51798681798, Loss: 3.4942057132720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11109 , TFLOPS: 69.5777833497538, Tokens per sec: 172421.41176625784, Loss: 3.4939563274383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11110 , TFLOPS: 67.95304987375155, Tokens per sec: 168395.14323355677, Loss: 3.495039224624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11111 , TFLOPS: 68.81977383571598, Tokens per sec: 170542.98068883092, Loss: 3.497865676879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11112 , TFLOPS: 69.86452744592847, Tokens per sec: 173131.99522404853, Loss: 3.5091450214385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11113 , TFLOPS: 67.08651343367796, Tokens per sec: 166247.77047818538, Loss: 3.5011885166168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11114 , TFLOPS: 68.68082207329002, Tokens per sec: 170198.64291473912, Loss: 3.5054445266723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11115 , TFLOPS: 68.9395297306515, Tokens per sec: 170839.748988682, Loss: 3.488347291946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11116 , TFLOPS: 68.58491403820751, Tokens per sec: 169960.97223866245, Loss: 3.538581371307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11117 , TFLOPS: 68.90805152105548, Tokens per sec: 170761.74251769204, Loss: 3.496521472930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11118 , TFLOPS: 68.27445890570628, Tokens per sec: 169191.6302208683, Loss: 3.4939467906951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11119 , TFLOPS: 69.66969953947755, Tokens per sec: 172649.1902097975, Loss: 3.5231947898864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11120 , TFLOPS: 68.5478357490846, Tokens per sec: 169869.08815370512, Loss: 3.5132622718811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11121 , TFLOPS: 68.11102414437467, Tokens per sec: 168786.62087846335, Loss: 3.508791446685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11122 , TFLOPS: 69.60466457367292, Tokens per sec: 172488.0264003407, Loss: 3.5018460750579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11123 , TFLOPS: 69.53776666141162, Tokens per sec: 172322.24600434382, Loss: 3.5168778896331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11124 , TFLOPS: 67.78507083889639, Tokens per sec: 167978.87268076738, Loss: 3.4894673824310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11125 , TFLOPS: 68.90180128501171, Tokens per sec: 170746.25374425523, Loss: 3.5036113262176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11126 , TFLOPS: 69.35330043130807, Tokens per sec: 171865.11836551444, Loss: 3.503413438796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11127 , TFLOPS: 67.000189035356, Tokens per sec: 166033.84910972603, Loss: 3.4900717735290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11128 , TFLOPS: 68.94216190304748, Tokens per sec: 170846.27180183696, Loss: 3.499814987182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11129 , TFLOPS: 70.55720025059772, Tokens per sec: 174848.51473822742, Loss: 3.510049343109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11130 , TFLOPS: 67.3622295633695, Tokens per sec: 166931.0254202, Loss: 3.515566349029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11131 , TFLOPS: 69.41320116844747, Tokens per sec: 172013.5589907568, Loss: 3.4865317344665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11132 , TFLOPS: 68.94053594123592, Tokens per sec: 170842.24248935343, Loss: 3.4826314449310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11133 , TFLOPS: 68.645182729325, Tokens per sec: 170110.324694396, Loss: 3.4910829067230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11134 , TFLOPS: 68.7015325387106, Tokens per sec: 170249.96572949743, Loss: 3.529082775115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11135 , TFLOPS: 67.91480478890949, Tokens per sec: 168300.36770027422, Loss: 3.4988675117492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11136 , TFLOPS: 69.57022699042287, Tokens per sec: 172402.68627543407, Loss: 3.508882761001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11137 , TFLOPS: 68.80696749341152, Tokens per sec: 170511.2451618654, Loss: 3.4838781356811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11138 , TFLOPS: 67.73702143049799, Tokens per sec: 167859.80095367687, Loss: 3.5150654315948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11139 , TFLOPS: 70.60850482755072, Tokens per sec: 174975.65313158266, Loss: 3.4945244789123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11140 , TFLOPS: 69.07997182609066, Tokens per sec: 171187.7799721542, Loss: 3.4928066730499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11141 , TFLOPS: 67.50597501889445, Tokens per sec: 167287.24249385993, Loss: 3.4835596084594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11142 , TFLOPS: 68.69931063589263, Tokens per sec: 170244.4596095517, Loss: 3.486454963684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11143 , TFLOPS: 69.5496541907148, Tokens per sec: 172351.70461147657, Loss: 3.4742960929870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11144 , TFLOPS: 68.03725450591914, Tokens per sec: 168603.81158797402, Loss: 3.490156412124634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11145 , TFLOPS: 68.9693326511747, Tokens per sec: 170913.6039088012, Loss: 3.508449077606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11146 , TFLOPS: 68.62060641630347, Tokens per sec: 170049.42188342428, Loss: 3.523784637451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11147 , TFLOPS: 67.89330952797563, Tokens per sec: 168247.1000757817, Loss: 3.489348888397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11148 , TFLOPS: 68.74535829955958, Tokens per sec: 170358.57079268736, Loss: 3.5184714794158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11149 , TFLOPS: 70.56899731512968, Tokens per sec: 174877.74916369157, Loss: 3.500859498977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11150 , TFLOPS: 67.86114310841475, Tokens per sec: 168167.38814468618, Loss: 3.502735137939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11151 , TFLOPS: 69.01694711429711, Tokens per sec: 171031.59779358466, Loss: 3.4936020374298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11152 , TFLOPS: 68.81048192669515, Tokens per sec: 170519.954314689, Loss: 3.4962449073791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11153 , TFLOPS: 67.94247077431709, Tokens per sec: 168368.92705977417, Loss: 3.492086410522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11154 , TFLOPS: 68.09258102092967, Tokens per sec: 168740.9167868889, Loss: 3.506157159805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11155 , TFLOPS: 68.76723614607876, Tokens per sec: 170412.78650640557, Loss: 3.5029380321502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11156 , TFLOPS: 70.59129209805319, Tokens per sec: 174932.99809174906, Loss: 3.505214214324951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11157 , TFLOPS: 67.98434837730484, Tokens per sec: 168472.7043731766, Loss: 3.498403787612915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11158 , TFLOPS: 68.03603419399595, Tokens per sec: 168600.7875205999, Loss: 3.4719483852386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11159 , TFLOPS: 70.60031360533381, Tokens per sec: 174955.35438059154, Loss: 3.5101521015167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11160 , TFLOPS: 68.72940025884512, Tokens per sec: 170319.02500987359, Loss: 3.5029473304748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11161 , TFLOPS: 67.82044302140217, Tokens per sec: 168066.5288455843, Loss: 3.472959280014038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11162 , TFLOPS: 69.14403878310814, Tokens per sec: 171346.5449492015, Loss: 3.4979279041290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11163 , TFLOPS: 69.56526666169613, Tokens per sec: 172390.39403442392, Loss: 3.487497329711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11164 , TFLOPS: 67.98771953321827, Tokens per sec: 168481.05846889355, Loss: 3.520343542098999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11165 , TFLOPS: 67.95404273999195, Tokens per sec: 168397.6036654737, Loss: 3.483311176300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11166 , TFLOPS: 69.58977524869672, Tokens per sec: 172451.12901285512, Loss: 3.49637770652771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11167 , TFLOPS: 68.88606519384761, Tokens per sec: 170707.25797687707, Loss: 3.51151180267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11168 , TFLOPS: 68.11681075314462, Tokens per sec: 168800.96073244285, Loss: 3.505753517150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11169 , TFLOPS: 69.92072171278438, Tokens per sec: 173271.25080762696, Loss: 3.512634515762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11170 , TFLOPS: 68.78473263611318, Tokens per sec: 170456.14473610837, Loss: 3.493892192840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11171 , TFLOPS: 68.742932624741, Tokens per sec: 170352.55970327673, Loss: 3.5005528926849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11172 , TFLOPS: 68.71297426018569, Tokens per sec: 170278.3195757224, Loss: 3.481733798980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11173 , TFLOPS: 69.57454903612974, Tokens per sec: 172413.39678080837, Loss: 3.4847464561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11174 , TFLOPS: 67.88066741161879, Tokens per sec: 168215.77151880536, Loss: 3.465294361114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11175 , TFLOPS: 67.74468425701568, Tokens per sec: 167878.79028191773, Loss: 3.5200488567352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11176 , TFLOPS: 70.57764181547883, Tokens per sec: 174899.17118782704, Loss: 3.5172066688537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11177 , TFLOPS: 67.87723553908226, Tokens per sec: 168207.26696060787, Loss: 3.4844961166381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11178 , TFLOPS: 68.76880287680379, Tokens per sec: 170416.6690377319, Loss: 3.4957594871520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11179 , TFLOPS: 69.13500509099573, Tokens per sec: 171324.1584650036, Loss: 3.470985174179077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11180 , TFLOPS: 68.42036726768416, Tokens per sec: 169553.20721498245, Loss: 3.4978671073913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11181 , TFLOPS: 68.75312112147866, Tokens per sec: 170377.80792054822, Loss: 3.515772581100464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11182 , TFLOPS: 68.74293497230441, Tokens per sec: 170352.56552079748, Loss: 3.481659412384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11183 , TFLOPS: 68.06856492833258, Tokens per sec: 168681.4022638423, Loss: 3.5107476711273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11184 , TFLOPS: 68.80405774045717, Tokens per sec: 170504.03447350874, Loss: 3.515455961227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11185 , TFLOPS: 68.22143422905008, Tokens per sec: 169060.22923096392, Loss: 3.4799861907958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11186 , TFLOPS: 69.6864239149552, Tokens per sec: 172690.63505457313, Loss: 3.487273693084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11187 , TFLOPS: 68.70882579854771, Tokens per sec: 170268.0392307927, Loss: 3.4572219848632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11188 , TFLOPS: 68.18834629832355, Tokens per sec: 168978.2336936871, Loss: 3.460204601287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11189 , TFLOPS: 69.70497343220087, Tokens per sec: 172736.6028017059, Loss: 3.4858593940734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11190 , TFLOPS: 68.51059501384974, Tokens per sec: 169776.8015093875, Loss: 3.486992359161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11191 , TFLOPS: 68.26403664516441, Tokens per sec: 169165.80271113876, Loss: 3.4893991947174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11192 , TFLOPS: 68.90249543759427, Tokens per sec: 170747.97393082202, Loss: 3.5005931854248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11193 , TFLOPS: 69.55062514218295, Tokens per sec: 172354.11073617375, Loss: 3.488348960876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11194 , TFLOPS: 66.96887662329291, Tokens per sec: 165956.25350328677, Loss: 3.5069305896759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11195 , TFLOPS: 68.74742138190362, Tokens per sec: 170363.68333800315, Loss: 3.5025196075439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11196 , TFLOPS: 70.53207743853001, Tokens per sec: 174786.2576424174, Loss: 3.4884085655212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11197 , TFLOPS: 67.89140209585987, Tokens per sec: 168242.37324887785, Loss: 3.490366220474243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11198 , TFLOPS: 67.61465669712206, Tokens per sec: 167556.56766478316, Loss: 3.509526014328003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11199 , TFLOPS: 69.69275608081027, Tokens per sec: 172706.32685910768, Loss: 3.5039162635803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11200 , TFLOPS: 68.82061131339901, Tokens per sec: 170545.05605078526, Loss: 3.4984500408172607 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/optimizer.pt +[rank0]:[2024-08-25 20:26:32,013] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063376459875144064, 'preprocessing_with_comm': 0.00127388299733866, 'state_converting': 1.0028424459887901, : 1.0118371459975606}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009200 + eval ppl=23.784027099609375, eval loss=3.1690142154693604 +------------------------------------------------------------------ +iteration: 11201 , TFLOPS: 62.46151113578347, Tokens per sec: 154786.5052979405, Loss: 3.4990317821502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11202 , TFLOPS: 68.70208611791017, Tokens per sec: 170251.33755973578, Loss: 3.4897899627685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11203 , TFLOPS: 68.72654298103794, Tokens per sec: 170311.94436071225, Loss: 3.486447334289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11204 , TFLOPS: 68.69875516704528, Tokens per sec: 170243.08309655802, Loss: 3.4894330501556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11205 , TFLOPS: 67.68801778787446, Tokens per sec: 167738.3645290589, Loss: 3.5011415481567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11206 , TFLOPS: 70.44606988848021, Tokens per sec: 174573.1214588802, Loss: 3.4875471591949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11207 , TFLOPS: 68.8093714238648, Tokens per sec: 170517.2023663434, Loss: 3.489744186401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11208 , TFLOPS: 68.12767923416096, Tokens per sec: 168827.8940256634, Loss: 3.4937338829040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11209 , TFLOPS: 68.89734710046919, Tokens per sec: 170735.2157842899, Loss: 3.511880874633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11210 , TFLOPS: 68.73111976114134, Tokens per sec: 170323.28612598244, Loss: 3.482367753982544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11211 , TFLOPS: 68.57337947453193, Tokens per sec: 169932.38831923725, Loss: 3.5034031867980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11212 , TFLOPS: 67.93170768949496, Tokens per sec: 168342.25494993324, Loss: 3.50689435005188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11213 , TFLOPS: 68.73681149696044, Tokens per sec: 170337.39087433866, Loss: 3.498110294342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11214 , TFLOPS: 69.76287978451522, Tokens per sec: 172880.10112165217, Loss: 3.4996836185455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11215 , TFLOPS: 68.20613618929917, Tokens per sec: 169022.31900324297, Loss: 3.5204131603240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11216 , TFLOPS: 69.49077273081092, Tokens per sec: 172205.78986750645, Loss: 3.501488208770752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11217 , TFLOPS: 69.87293820963887, Tokens per sec: 173152.83802304498, Loss: 3.4905619621276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11218 , TFLOPS: 68.7278491249832, Tokens per sec: 170315.18112929224, Loss: 3.4991719722747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11219 , TFLOPS: 69.56658423104777, Tokens per sec: 172393.65911641956, Loss: 3.49990177154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11220 , TFLOPS: 67.88612087481535, Tokens per sec: 168229.28580135497, Loss: 3.4904935359954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11221 , TFLOPS: 68.81004731364197, Tokens per sec: 170518.87729566667, Loss: 3.509638547897339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11222 , TFLOPS: 68.97722709321133, Tokens per sec: 170933.16720581256, Loss: 3.509673833847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11223 , TFLOPS: 68.7442655718251, Tokens per sec: 170355.86289298738, Loss: 3.491820812225342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11224 , TFLOPS: 67.72757259782178, Tokens per sec: 167836.38570543038, Loss: 3.513179302215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11225 , TFLOPS: 68.81589629071424, Tokens per sec: 170533.37170516973, Loss: 3.504465103149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11226 , TFLOPS: 70.56752517996792, Tokens per sec: 174874.10105059165, Loss: 3.493340015411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11227 , TFLOPS: 67.86718334169588, Tokens per sec: 168182.35650814342, Loss: 3.5197949409484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11228 , TFLOPS: 67.79148809826079, Tokens per sec: 167994.77535639185, Loss: 3.5038390159606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11229 , TFLOPS: 69.74359500185005, Tokens per sec: 172832.31130581372, Loss: 3.5308029651641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11230 , TFLOPS: 68.72079240587948, Tokens per sec: 170297.69380199147, Loss: 3.4822914600372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11231 , TFLOPS: 68.39756513565462, Tokens per sec: 169496.70101995103, Loss: 3.493631362915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11232 , TFLOPS: 68.83051804641124, Tokens per sec: 170569.60602650628, Loss: 3.50907039642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11233 , TFLOPS: 69.60607548678543, Tokens per sec: 172491.52279845838, Loss: 3.4825730323791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11234 , TFLOPS: 68.12023417620583, Tokens per sec: 168809.4443519097, Loss: 3.494683265686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11235 , TFLOPS: 67.92953480451814, Tokens per sec: 168336.87030159766, Loss: 3.4952633380889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11236 , TFLOPS: 70.56256354622234, Tokens per sec: 174861.805575601, Loss: 3.511413097381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11237 , TFLOPS: 69.53233374304348, Tokens per sec: 172308.7826341422, Loss: 3.5076892375946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11238 , TFLOPS: 67.7370574135217, Tokens per sec: 167859.89012357348, Loss: 3.496213674545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11239 , TFLOPS: 69.65180330417346, Tokens per sec: 172604.84136728084, Loss: 3.477672815322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11240 , TFLOPS: 68.87735413996144, Tokens per sec: 170685.6710257453, Loss: 3.480353593826294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11241 , TFLOPS: 68.76863297346293, Tokens per sec: 170416.2479985368, Loss: 3.501429557800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11242 , TFLOPS: 68.64368478036452, Tokens per sec: 170106.61261185922, Loss: 3.4826126098632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11243 , TFLOPS: 68.69310499968843, Tokens per sec: 170229.08135943042, Loss: 3.501392126083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11244 , TFLOPS: 69.74976534011952, Tokens per sec: 172847.60208376436, Loss: 3.493041515350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11245 , TFLOPS: 68.11322000393218, Tokens per sec: 168792.0624603408, Loss: 3.5005016326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11246 , TFLOPS: 69.4514185146329, Tokens per sec: 172108.2657845925, Loss: 3.478346347808838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11247 , TFLOPS: 68.85254422013072, Tokens per sec: 170624.1893113665, Loss: 3.47879958152771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11248 , TFLOPS: 68.70912483553423, Tokens per sec: 170268.78027738692, Loss: 3.485792875289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11249 , TFLOPS: 68.15954927949245, Tokens per sec: 168906.87150877013, Loss: 3.5268609523773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11250 , TFLOPS: 68.04403928247501, Tokens per sec: 168620.62501168431, Loss: 3.509359836578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11251 , TFLOPS: 68.79629869021382, Tokens per sec: 170484.8066922758, Loss: 3.4960975646972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11252 , TFLOPS: 68.74327121442697, Tokens per sec: 170353.3987658173, Loss: 3.479320526123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11253 , TFLOPS: 69.66705749521145, Tokens per sec: 172642.64293306088, Loss: 3.5146536827087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11254 , TFLOPS: 68.1868898416136, Tokens per sec: 168974.62443351792, Loss: 3.5047380924224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11255 , TFLOPS: 68.06098962470155, Tokens per sec: 168662.62982695602, Loss: 3.5132863521575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11256 , TFLOPS: 70.59008217701351, Tokens per sec: 174929.99977412898, Loss: 3.503333568572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11257 , TFLOPS: 68.57781543392147, Tokens per sec: 169943.38111526583, Loss: 3.510491371154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11258 , TFLOPS: 69.5366818876321, Tokens per sec: 172319.55781542044, Loss: 3.5068366527557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11259 , TFLOPS: 68.98615677291743, Tokens per sec: 170955.2959358097, Loss: 3.505920648574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11260 , TFLOPS: 68.71459695236433, Tokens per sec: 170282.34078569539, Loss: 3.50309157371521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11261 , TFLOPS: 67.69386596397173, Tokens per sec: 167752.85695365773, Loss: 3.4945878982543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11262 , TFLOPS: 68.8530087493468, Tokens per sec: 170625.34046593605, Loss: 3.5199062824249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11263 , TFLOPS: 68.82588458924154, Tokens per sec: 170558.1238092223, Loss: 3.485274314880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11264 , TFLOPS: 69.60974550172342, Tokens per sec: 172500.61749976026, Loss: 3.5016419887542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11265 , TFLOPS: 68.1855279804835, Tokens per sec: 168971.24959161613, Loss: 3.4897141456604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11266 , TFLOPS: 69.9125151291616, Tokens per sec: 173250.91398937991, Loss: 3.5080087184906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11267 , TFLOPS: 68.7781646248421, Tokens per sec: 170439.86848065248, Loss: 3.4929583072662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11268 , TFLOPS: 67.63354541206422, Tokens per sec: 167603.37598117357, Loss: 3.4988009929656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11269 , TFLOPS: 68.98957895944707, Tokens per sec: 170963.77649101513, Loss: 3.513925552368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11270 , TFLOPS: 68.76742524648702, Tokens per sec: 170413.2551180484, Loss: 3.492945909500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11271 , TFLOPS: 67.84968292860619, Tokens per sec: 168138.98855667753, Loss: 3.504816770553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11272 , TFLOPS: 68.70962212762359, Tokens per sec: 170270.01262196564, Loss: 3.49003529548645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11273 , TFLOPS: 70.56003907509945, Tokens per sec: 174855.54965806505, Loss: 3.470133066177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11274 , TFLOPS: 67.7644493532762, Tokens per sec: 167927.7703677564, Loss: 3.491008758544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11275 , TFLOPS: 68.47879083288761, Tokens per sec: 169697.9872454436, Loss: 3.491102933883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11276 , TFLOPS: 70.00210333194964, Tokens per sec: 173472.9234248439, Loss: 3.4896421432495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11277 , TFLOPS: 69.36588261853, Tokens per sec: 171896.29841149272, Loss: 3.4715328216552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11278 , TFLOPS: 69.56047086144714, Tokens per sec: 172378.50951310596, Loss: 3.503086805343628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11279 , TFLOPS: 67.92000998922228, Tokens per sec: 168313.26676004948, Loss: 3.4897119998931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11280 , TFLOPS: 68.69981529210816, Tokens per sec: 170245.71020324627, Loss: 3.5001955032348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11281 , TFLOPS: 69.66402007744016, Tokens per sec: 172635.11587722442, Loss: 3.514296531677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11282 , TFLOPS: 68.14839550259127, Tokens per sec: 168879.23122091906, Loss: 3.513864755630493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11283 , TFLOPS: 68.48965278478208, Tokens per sec: 169724.90435878088, Loss: 3.4902305603027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11284 , TFLOPS: 70.03906619804329, Tokens per sec: 173564.52147881978, Loss: 3.5171782970428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11285 , TFLOPS: 68.00036499069267, Tokens per sec: 168512.39530552726, Loss: 3.5018932819366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11286 , TFLOPS: 68.89250814625312, Tokens per sec: 170723.2243226877, Loss: 3.5057342052459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11287 , TFLOPS: 68.73390604130911, Tokens per sec: 170330.19083517228, Loss: 3.491455316543579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11288 , TFLOPS: 68.75198965812656, Tokens per sec: 170375.00402972076, Loss: 3.478602409362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11289 , TFLOPS: 68.96322575627245, Tokens per sec: 170898.47035050343, Loss: 3.5096020698547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11290 , TFLOPS: 67.68995534107474, Tokens per sec: 167743.16599932828, Loss: 3.4843320846557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11291 , TFLOPS: 68.52113667187821, Tokens per sec: 169802.92489924154, Loss: 3.488725423812866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11292 , TFLOPS: 68.77407054759387, Tokens per sec: 170429.72290623153, Loss: 3.4715349674224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11293 , TFLOPS: 70.57865782734449, Tokens per sec: 174901.68897715313, Loss: 3.5141115188598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11294 , TFLOPS: 67.74328644248318, Tokens per sec: 167875.32634352424, Loss: 3.483309030532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11295 , TFLOPS: 69.05322609104891, Tokens per sec: 171121.50109443482, Loss: 3.4910829067230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11296 , TFLOPS: 69.15960330362554, Tokens per sec: 171385.11554561707, Loss: 3.5042545795440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11297 , TFLOPS: 69.4806258103726, Tokens per sec: 172180.64468088472, Loss: 3.4785428047180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11298 , TFLOPS: 68.51027081371798, Tokens per sec: 169775.99810574742, Loss: 3.512212038040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11299 , TFLOPS: 68.72314879313426, Tokens per sec: 170303.53318918627, Loss: 3.489424228668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11300 , TFLOPS: 69.55692061670493, Tokens per sec: 172369.71161554297, Loss: 3.5020546913146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11301 , TFLOPS: 68.13961103282817, Tokens per sec: 168857.46233715685, Loss: 3.493522882461548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11302 , TFLOPS: 67.90385159297038, Tokens per sec: 168273.224474143, Loss: 3.5097904205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11303 , TFLOPS: 70.51360675764535, Tokens per sec: 174740.48526047173, Loss: 3.52248215675354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11304 , TFLOPS: 69.23448724152574, Tokens per sec: 171570.68619288047, Loss: 3.4830195903778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11305 , TFLOPS: 67.03843500715858, Tokens per sec: 166128.62684099446, Loss: 3.5122387409210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11306 , TFLOPS: 69.74133525395844, Tokens per sec: 172826.71140160653, Loss: 3.485285758972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11307 , TFLOPS: 68.64256308373272, Tokens per sec: 170103.83292404053, Loss: 3.503289222717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11308 , TFLOPS: 67.82021341326966, Tokens per sec: 168065.9598513385, Loss: 3.5264089107513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11309 , TFLOPS: 68.78964554478445, Tokens per sec: 170468.31946499736, Loss: 3.5093114376068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11310 , TFLOPS: 68.89968628209027, Tokens per sec: 170741.01253402908, Loss: 3.536996603012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11311 , TFLOPS: 69.63511281379608, Tokens per sec: 172563.48049351643, Loss: 3.474966049194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11312 , TFLOPS: 67.3366082968242, Tokens per sec: 166867.5330992852, Loss: 3.493283271789551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11313 , TFLOPS: 70.4379543467765, Tokens per sec: 174553.0102525377, Loss: 3.475436210632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11314 , TFLOPS: 68.61893036995058, Tokens per sec: 170045.26845593003, Loss: 3.490725517272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11315 , TFLOPS: 68.84975606703719, Tokens per sec: 170617.2799608603, Loss: 3.4895691871643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11316 , TFLOPS: 67.893264332965, Tokens per sec: 168246.9880775667, Loss: 3.4940760135650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11317 , TFLOPS: 69.64783671377982, Tokens per sec: 172595.01171932943, Loss: 3.4730746746063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11318 , TFLOPS: 69.4538819375588, Tokens per sec: 172114.37041797137, Loss: 3.5184364318847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11319 , TFLOPS: 68.74220868573339, Tokens per sec: 170350.7657026674, Loss: 3.5023040771484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11320 , TFLOPS: 69.50206374550544, Tokens per sec: 172233.7702457274, Loss: 3.470261573791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11321 , TFLOPS: 68.09988801167222, Tokens per sec: 168759.02431488043, Loss: 3.5084481239318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11322 , TFLOPS: 67.97801209693833, Tokens per sec: 168457.00237242947, Loss: 3.511786699295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11323 , TFLOPS: 69.67265442899858, Tokens per sec: 172656.51275153857, Loss: 3.504608154296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11324 , TFLOPS: 68.9392604894242, Tokens per sec: 170839.08177926138, Loss: 3.4930951595306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11325 , TFLOPS: 68.75699412209804, Tokens per sec: 170387.40564272928, Loss: 3.503506660461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11326 , TFLOPS: 68.92344727621752, Tokens per sec: 170799.894894966, Loss: 3.5011608600616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11327 , TFLOPS: 67.74684715260234, Tokens per sec: 167884.1501754441, Loss: 3.5094497203826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11328 , TFLOPS: 68.81679903435356, Tokens per sec: 170535.60880335342, Loss: 3.4878158569335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11329 , TFLOPS: 68.87339771895523, Tokens per sec: 170675.86657865616, Loss: 3.49849009513855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11330 , TFLOPS: 69.62868501306656, Tokens per sec: 172547.55169522797, Loss: 3.4883131980895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11331 , TFLOPS: 68.95264154990181, Tokens per sec: 170872.24152117042, Loss: 3.516540288925171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11332 , TFLOPS: 68.10398691834227, Tokens per sec: 168769.18185714065, Loss: 3.4814646244049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11333 , TFLOPS: 69.98226723796895, Tokens per sec: 173423.76739883362, Loss: 3.4761955738067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11334 , TFLOPS: 68.60709178484446, Tokens per sec: 170015.93113790927, Loss: 3.4777090549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11335 , TFLOPS: 67.86971484387317, Tokens per sec: 168188.62984940677, Loss: 3.5198233127593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11336 , TFLOPS: 68.65824732789498, Tokens per sec: 170142.70020883673, Loss: 3.49202823638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11337 , TFLOPS: 69.69096752253782, Tokens per sec: 172701.8946146826, Loss: 3.51519775390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11338 , TFLOPS: 68.2633730938763, Tokens per sec: 169164.15835795063, Loss: 3.468571186065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11339 , TFLOPS: 68.83068394247114, Tokens per sec: 170570.01713521767, Loss: 3.5063631534576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11340 , TFLOPS: 70.55179922378127, Tokens per sec: 174835.13039880406, Loss: 3.468864917755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11341 , TFLOPS: 67.8921583990622, Tokens per sec: 168244.24745152684, Loss: 3.5029842853546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11342 , TFLOPS: 67.81844863868913, Tokens per sec: 168061.586545522, Loss: 3.494255781173706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11343 , TFLOPS: 69.90122617633817, Tokens per sec: 173222.93872070263, Loss: 3.4852447509765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11344 , TFLOPS: 68.81359739050052, Tokens per sec: 170527.67477719538, Loss: 3.5206799507141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11345 , TFLOPS: 67.79227364411285, Tokens per sec: 167996.7220255483, Loss: 3.4753687381744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11346 , TFLOPS: 68.85064416461856, Tokens per sec: 170619.48076449896, Loss: 3.484708547592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11347 , TFLOPS: 68.64249355941718, Tokens per sec: 170103.66063512786, Loss: 3.492783546447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11348 , TFLOPS: 69.71234877735955, Tokens per sec: 172754.87971947045, Loss: 3.5011327266693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11349 , TFLOPS: 67.18369269949292, Tokens per sec: 166488.5914040547, Loss: 3.508132219314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11350 , TFLOPS: 69.63969898648894, Tokens per sec: 172574.84553465794, Loss: 3.4931092262268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11351 , TFLOPS: 69.52905927679427, Tokens per sec: 172300.6681460644, Loss: 3.506227731704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11352 , TFLOPS: 67.86817387399823, Tokens per sec: 168184.81115630484, Loss: 3.5003464221954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11353 , TFLOPS: 68.67778511522839, Tokens per sec: 170191.11699811387, Loss: 3.498757839202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11354 , TFLOPS: 68.62716088750821, Tokens per sec: 170065.6645851042, Loss: 3.5041708946228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11355 , TFLOPS: 68.80676165916036, Tokens per sec: 170510.73508191653, Loss: 3.5121047496795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11356 , TFLOPS: 67.8038642358201, Tokens per sec: 168025.44478860794, Loss: 3.4820423126220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11357 , TFLOPS: 68.77634921565638, Tokens per sec: 170435.36969670645, Loss: 3.4923479557037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11358 , TFLOPS: 69.60649616395122, Tokens per sec: 172492.56528281665, Loss: 3.50080943107605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11359 , TFLOPS: 68.92222033293683, Tokens per sec: 170796.8543943575, Loss: 3.4889988899230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11360 , TFLOPS: 69.54336343863226, Tokens per sec: 172336.11543483243, Loss: 3.4843544960021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11361 , TFLOPS: 68.03272702889268, Tokens per sec: 168592.59200115968, Loss: 3.48103666305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11362 , TFLOPS: 68.14368908587824, Tokens per sec: 168867.56820184956, Loss: 3.483367681503296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11363 , TFLOPS: 69.18247425645956, Tokens per sec: 171441.7922861832, Loss: 3.4949777126312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11364 , TFLOPS: 67.877683546144, Tokens per sec: 168208.37717145888, Loss: 3.5001652240753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11365 , TFLOPS: 68.84380251258885, Tokens per sec: 170602.52639709917, Loss: 3.4934511184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11366 , TFLOPS: 68.79813280035953, Tokens per sec: 170489.35181926077, Loss: 3.5196807384490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11367 , TFLOPS: 69.61115656609003, Tokens per sec: 172504.1142727023, Loss: 3.507133960723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11368 , TFLOPS: 68.1461268174217, Tokens per sec: 168873.60916915257, Loss: 3.487975597381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11369 , TFLOPS: 67.94212024395017, Tokens per sec: 168368.05840691083, Loss: 3.5015311241149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11370 , TFLOPS: 70.5973090104469, Tokens per sec: 174947.90866347833, Loss: 3.4944324493408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11371 , TFLOPS: 69.06566772966583, Tokens per sec: 171152.3328454872, Loss: 3.475048303604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11372 , TFLOPS: 67.77191005697462, Tokens per sec: 167946.25881338544, Loss: 3.4825758934020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11373 , TFLOPS: 69.07928932541596, Tokens per sec: 171186.08866029864, Loss: 3.4947521686553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11374 , TFLOPS: 68.87025142187414, Tokens per sec: 170668.069707896, Loss: 3.4914917945861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11375 , TFLOPS: 67.88778324130888, Tokens per sec: 168233.40532865073, Loss: 3.492039203643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11376 , TFLOPS: 68.7817568421541, Tokens per sec: 170448.770390867, Loss: 3.489736557006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11377 , TFLOPS: 69.56511677550078, Tokens per sec: 172390.02259992054, Loss: 3.4955475330352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11378 , TFLOPS: 70.60670986254917, Tokens per sec: 174971.20501057708, Loss: 3.492933750152588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11379 , TFLOPS: 67.12991175263735, Tokens per sec: 166355.31629328622, Loss: 3.4853241443634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11380 , TFLOPS: 70.57158581097175, Tokens per sec: 174884.16374153458, Loss: 3.4875104427337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11381 , TFLOPS: 68.86760274537029, Tokens per sec: 170661.50599574365, Loss: 3.494145393371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11382 , TFLOPS: 67.87221415824844, Tokens per sec: 168194.8234257509, Loss: 3.5205557346343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11383 , TFLOPS: 69.0894952093926, Tokens per sec: 171211.37996506368, Loss: 3.5011370182037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11384 , TFLOPS: 68.69943889636806, Tokens per sec: 170244.7774531389, Loss: 3.4964094161987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11385 , TFLOPS: 68.8070649474161, Tokens per sec: 170511.48666362415, Loss: 3.513594150543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11386 , TFLOPS: 67.72912852967775, Tokens per sec: 167840.24147596952, Loss: 3.471355438232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11387 , TFLOPS: 70.59075021631614, Tokens per sec: 174931.6552491115, Loss: 3.510249137878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11388 , TFLOPS: 68.33810077669962, Tokens per sec: 169349.34178792062, Loss: 3.504807949066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11389 , TFLOPS: 67.89272822120887, Tokens per sec: 168245.65953357742, Loss: 3.4867684841156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11390 , TFLOPS: 69.70516234723839, Tokens per sec: 172737.0709539796, Loss: 3.4875690937042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11391 , TFLOPS: 68.63143838082907, Tokens per sec: 170076.2646847581, Loss: 3.500253200531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11392 , TFLOPS: 68.82428933766231, Tokens per sec: 170554.17060007752, Loss: 3.516040802001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11393 , TFLOPS: 68.89661153145248, Tokens per sec: 170733.3929632369, Loss: 3.539836883544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11394 , TFLOPS: 67.93615481567879, Tokens per sec: 168353.2754185082, Loss: 3.497082233428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11395 , TFLOPS: 68.79300734047446, Tokens per sec: 170476.65036504954, Loss: 3.511019706726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11396 , TFLOPS: 68.64743776837466, Tokens per sec: 170115.91292957411, Loss: 3.4677181243896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11397 , TFLOPS: 68.62573854322765, Tokens per sec: 170062.13985929286, Loss: 3.493635654449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11398 , TFLOPS: 70.62397456032437, Tokens per sec: 175013.9888335275, Loss: 3.4817543029785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11399 , TFLOPS: 68.82986592387164, Tokens per sec: 170567.9899950159, Loss: 3.4971017837524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11400 , TFLOPS: 70.07082725516065, Tokens per sec: 173643.22887712586, Loss: 3.4914979934692383 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/optimizer.pt +[rank0]:[2024-08-25 20:52:17,943] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006328372997813858, 'preprocessing_with_comm': 0.0015438729897141457, 'state_converting': 0.9805629719921853, : 0.9898340340005234}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400, took 4.23s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009400 + eval ppl=23.810955047607422, eval loss=3.1701457500457764 +------------------------------------------------------------------ +iteration: 11401 , TFLOPS: 63.36955308383239, Tokens per sec: 157036.7332742839, Loss: 3.5029709339141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11402 , TFLOPS: 68.63074898491207, Tokens per sec: 170074.55628573254, Loss: 3.4595351219177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11403 , TFLOPS: 69.12153969090862, Tokens per sec: 171290.78972025143, Loss: 3.4912614822387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11404 , TFLOPS: 67.74177729796966, Tokens per sec: 167871.58651717374, Loss: 3.500258445739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11405 , TFLOPS: 68.71025841072732, Tokens per sec: 170271.58940158904, Loss: 3.493391752243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11406 , TFLOPS: 68.74182332830499, Tokens per sec: 170349.8107445092, Loss: 3.5029003620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11407 , TFLOPS: 70.53491017259402, Tokens per sec: 174793.27746947945, Loss: 3.5043246746063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11408 , TFLOPS: 67.02112827821723, Tokens per sec: 166085.7388601842, Loss: 3.486248731613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11409 , TFLOPS: 67.88276311121285, Tokens per sec: 168220.9648932598, Loss: 3.499711275100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11410 , TFLOPS: 70.45453217524786, Tokens per sec: 174594.0919376883, Loss: 3.510533094406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11411 , TFLOPS: 68.69418901542062, Tokens per sec: 170231.767669829, Loss: 3.504033327102661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11412 , TFLOPS: 68.53949057505157, Tokens per sec: 169848.40789315448, Loss: 3.5061733722686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11413 , TFLOPS: 68.90722575217556, Tokens per sec: 170759.69617144164, Loss: 3.491208553314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11414 , TFLOPS: 68.82834096115116, Tokens per sec: 170564.21096940606, Loss: 3.4845387935638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11415 , TFLOPS: 69.74627391984464, Tokens per sec: 172838.9499597115, Loss: 3.499508857727051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11416 , TFLOPS: 67.69826059560492, Tokens per sec: 167763.7473349534, Loss: 3.479656934738159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11417 , TFLOPS: 69.71266876675621, Tokens per sec: 172755.67268844516, Loss: 3.4788596630096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11418 , TFLOPS: 69.85110274585185, Tokens per sec: 173098.72733840582, Loss: 3.5006322860717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11419 , TFLOPS: 66.93404925927464, Tokens per sec: 165869.94745869865, Loss: 3.4891326427459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11420 , TFLOPS: 69.92178840663463, Tokens per sec: 173273.8941924939, Loss: 3.505725860595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11421 , TFLOPS: 68.79685468280792, Tokens per sec: 170486.18450317162, Loss: 3.506807804107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11422 , TFLOPS: 68.70919716091319, Tokens per sec: 170268.95950764333, Loss: 3.4974546432495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11423 , TFLOPS: 67.84040014257997, Tokens per sec: 168115.98479032126, Loss: 3.4850575923919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11424 , TFLOPS: 68.78813148892459, Tokens per sec: 170464.56746778957, Loss: 3.4795427322387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11425 , TFLOPS: 69.71625119616655, Tokens per sec: 172764.55034316116, Loss: 3.493436098098755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11426 , TFLOPS: 68.21377582194056, Tokens per sec: 169041.25085450333, Loss: 3.490975856781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11427 , TFLOPS: 69.5457814811806, Tokens per sec: 172342.1076106366, Loss: 3.4899137020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11428 , TFLOPS: 67.71572561121536, Tokens per sec: 167807.02756756637, Loss: 3.4721508026123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11429 , TFLOPS: 68.633402784433, Tokens per sec: 170081.13269328556, Loss: 3.4918899536132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11430 , TFLOPS: 69.98532904829305, Tokens per sec: 173431.354902103, Loss: 3.496718645095825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11431 , TFLOPS: 67.87910762514721, Tokens per sec: 168211.9061961041, Loss: 3.5173816680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11432 , TFLOPS: 68.83048477601292, Tokens per sec: 170569.5235787942, Loss: 3.4704577922821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11433 , TFLOPS: 68.79224570739767, Tokens per sec: 170474.76295438447, Loss: 3.5125155448913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11434 , TFLOPS: 68.62417090618608, Tokens per sec: 170058.25508201434, Loss: 3.4933078289031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11435 , TFLOPS: 68.89756644183885, Tokens per sec: 170735.75933636638, Loss: 3.495487928390503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11436 , TFLOPS: 67.97567511105726, Tokens per sec: 168451.21106397593, Loss: 3.5129024982452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11437 , TFLOPS: 70.5654290991936, Tokens per sec: 174868.9067315305, Loss: 3.454547882080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11438 , TFLOPS: 68.09692131411158, Tokens per sec: 168751.67251151643, Loss: 3.5037755966186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11439 , TFLOPS: 67.34938302019734, Tokens per sec: 166899.19027105582, Loss: 3.5018997192382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11440 , TFLOPS: 69.56142593805053, Tokens per sec: 172380.87629813948, Loss: 3.4906044006347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11441 , TFLOPS: 68.68423014288267, Tokens per sec: 170207.0884866196, Loss: 3.49527907371521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11442 , TFLOPS: 67.81029406956266, Tokens per sec: 168041.37862492667, Loss: 3.525235891342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11443 , TFLOPS: 68.95062653867144, Tokens per sec: 170867.24810136974, Loss: 3.4889049530029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11444 , TFLOPS: 68.75869500283974, Tokens per sec: 170391.62061257474, Loss: 3.495389223098755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11445 , TFLOPS: 69.71307219990803, Tokens per sec: 172756.67244023716, Loss: 3.512223243713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11446 , TFLOPS: 66.37813148838875, Tokens per sec: 164492.32198304546, Loss: 3.5170648097991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11447 , TFLOPS: 70.4871609756568, Tokens per sec: 174674.94969946076, Loss: 3.4937894344329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11448 , TFLOPS: 68.6673960833833, Tokens per sec: 170165.3718327554, Loss: 3.4636380672454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11449 , TFLOPS: 67.8950639562263, Tokens per sec: 168251.44774225305, Loss: 3.4650185108184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11450 , TFLOPS: 69.12824791555566, Tokens per sec: 171307.41343989878, Loss: 3.5190541744232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11451 , TFLOPS: 68.85413108412124, Tokens per sec: 170628.12173514246, Loss: 3.4740045070648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11452 , TFLOPS: 68.82194484135564, Tokens per sec: 170548.36067996203, Loss: 3.505110740661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11453 , TFLOPS: 67.75340342305869, Tokens per sec: 167900.39733586775, Loss: 3.4697587490081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11454 , TFLOPS: 70.55862739212708, Tokens per sec: 174852.05135214978, Loss: 3.5225720405578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11455 , TFLOPS: 67.63472724792088, Tokens per sec: 167606.3047006163, Loss: 3.5061984062194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11456 , TFLOPS: 67.78680789127094, Tokens per sec: 167983.17728790364, Loss: 3.4950647354125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11457 , TFLOPS: 70.54345957623862, Tokens per sec: 174814.46383350535, Loss: 3.488711357116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11458 , TFLOPS: 68.91054051681584, Tokens per sec: 170767.91052337174, Loss: 3.4815356731414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11459 , TFLOPS: 68.00552665817436, Tokens per sec: 168525.18648615154, Loss: 3.500141143798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11460 , TFLOPS: 68.82391664968921, Tokens per sec: 170553.2470382252, Loss: 3.4966230392456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11461 , TFLOPS: 68.67477728782988, Tokens per sec: 170183.66327048084, Loss: 3.505699872970581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11462 , TFLOPS: 69.59984449170508, Tokens per sec: 172476.08170624854, Loss: 3.4945623874664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11463 , TFLOPS: 68.1643771545379, Tokens per sec: 168918.8355149671, Loss: 3.5050578117370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11464 , TFLOPS: 68.14467034808115, Tokens per sec: 168869.99987768332, Loss: 3.497941017150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11465 , TFLOPS: 69.85903260265333, Tokens per sec: 173118.37839710576, Loss: 3.50488018989563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11466 , TFLOPS: 67.42594631710794, Tokens per sec: 167088.9225549437, Loss: 3.500178575515747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11467 , TFLOPS: 70.59241472244513, Tokens per sec: 174935.7800786596, Loss: 3.497096061706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11468 , TFLOPS: 67.84689441348625, Tokens per sec: 168132.07830903004, Loss: 3.494178056716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11469 , TFLOPS: 68.19193843849588, Tokens per sec: 168987.13541274105, Loss: 3.4896364212036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11470 , TFLOPS: 68.79552199286557, Tokens per sec: 170482.88195068657, Loss: 3.500100612640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11471 , TFLOPS: 68.66177250643914, Tokens per sec: 170151.43598959962, Loss: 3.504018783569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11472 , TFLOPS: 68.74932828735163, Tokens per sec: 170368.4088597663, Loss: 3.4673643112182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11473 , TFLOPS: 68.7875987672098, Tokens per sec: 170463.2473246962, Loss: 3.4626731872558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11474 , TFLOPS: 70.51126320864198, Tokens per sec: 174734.67768787817, Loss: 3.5017528533935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11475 , TFLOPS: 67.12725156127772, Tokens per sec: 166348.7240460528, Loss: 3.484785795211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11476 , TFLOPS: 67.86864009272395, Tokens per sec: 168185.9664976654, Loss: 3.5111186504364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11477 , TFLOPS: 70.56240585189339, Tokens per sec: 174861.4147916824, Loss: 3.4716529846191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11478 , TFLOPS: 68.67580126238417, Tokens per sec: 170186.20079222647, Loss: 3.499055862426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11479 , TFLOPS: 67.91056427372959, Tokens per sec: 168289.85923652715, Loss: 3.4946329593658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11480 , TFLOPS: 69.10330584814052, Tokens per sec: 171245.60424924848, Loss: 3.474815845489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11481 , TFLOPS: 68.86631287241028, Tokens per sec: 170658.3095484575, Loss: 3.4942407608032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11482 , TFLOPS: 68.5850064218122, Tokens per sec: 169961.20117541176, Loss: 3.4909627437591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11483 , TFLOPS: 67.96827081049561, Tokens per sec: 168432.86239153313, Loss: 3.4867911338806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11484 , TFLOPS: 70.55174239089484, Tokens per sec: 174834.9895606508, Loss: 3.5132033824920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11485 , TFLOPS: 68.86145348494121, Tokens per sec: 170646.26745100343, Loss: 3.5006866455078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11486 , TFLOPS: 66.62995449529271, Tokens per sec: 165116.36713474774, Loss: 3.498413324356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11487 , TFLOPS: 70.02448681943453, Tokens per sec: 173528.39217257616, Loss: 3.490093231201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11488 , TFLOPS: 68.86143742591322, Tokens per sec: 170646.22765496303, Loss: 3.4886093139648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11489 , TFLOPS: 68.59249967776081, Tokens per sec: 169979.77028910271, Loss: 3.4978690147399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11490 , TFLOPS: 67.66856901931601, Tokens per sec: 167690.16833811236, Loss: 3.496945381164551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11491 , TFLOPS: 68.75167572215575, Tokens per sec: 170374.22606180218, Loss: 3.485729217529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11492 , TFLOPS: 68.85980219354639, Tokens per sec: 170642.17536902233, Loss: 3.503214120864868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11493 , TFLOPS: 68.06975264318667, Tokens per sec: 168684.34555208834, Loss: 3.483013868331909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11494 , TFLOPS: 70.53754982056849, Tokens per sec: 174799.8188079414, Loss: 3.4913079738616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11495 , TFLOPS: 67.87764128724358, Tokens per sec: 168208.27244924844, Loss: 3.5081229209899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11496 , TFLOPS: 68.6758355682562, Tokens per sec: 170186.28580595646, Loss: 3.4818403720855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11497 , TFLOPS: 68.93000712344005, Tokens per sec: 170816.15091901025, Loss: 3.4948651790618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11498 , TFLOPS: 68.71217594500963, Tokens per sec: 170276.34126277445, Loss: 3.5040700435638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11499 , TFLOPS: 68.75435210520581, Tokens per sec: 170380.85843382814, Loss: 3.493694543838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11500 , TFLOPS: 68.8866046224626, Tokens per sec: 170708.59474040795, Loss: 3.5011792182922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11501 , TFLOPS: 68.78171626474182, Tokens per sec: 170448.6698355694, Loss: 3.4821319580078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11502 , TFLOPS: 68.96791092118268, Tokens per sec: 170910.08070526453, Loss: 3.4944541454315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11503 , TFLOPS: 67.85742505147861, Tokens per sec: 168158.17439002066, Loss: 3.5047287940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11504 , TFLOPS: 70.5886005786839, Tokens per sec: 174926.32821025664, Loss: 3.48109769821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11505 , TFLOPS: 68.74731400060215, Tokens per sec: 170363.41723530905, Loss: 3.5038511753082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11506 , TFLOPS: 67.82573160615664, Tokens per sec: 168079.63454119157, Loss: 3.4804718494415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11507 , TFLOPS: 69.6733032177919, Tokens per sec: 172658.12052163837, Loss: 3.5091307163238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11508 , TFLOPS: 68.48561614868491, Tokens per sec: 169714.9011298599, Loss: 3.4866437911987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11509 , TFLOPS: 68.75030991068186, Tokens per sec: 170370.84143051333, Loss: 3.514962673187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11510 , TFLOPS: 68.68684377371234, Tokens per sec: 170213.56535173074, Loss: 3.509539842605591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11511 , TFLOPS: 68.7005957849855, Tokens per sec: 170247.6443505747, Loss: 3.4887423515319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11512 , TFLOPS: 68.70244091430436, Tokens per sec: 170252.21678428462, Loss: 3.4821720123291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11513 , TFLOPS: 68.23831535478365, Tokens per sec: 169102.06252014122, Loss: 3.492490291595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11514 , TFLOPS: 70.60673535244226, Tokens per sec: 174971.26817734007, Loss: 3.506251811981201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11515 , TFLOPS: 68.7526682603117, Tokens per sec: 170376.68568068973, Loss: 3.497825860977173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11516 , TFLOPS: 67.60867598800233, Tokens per sec: 167541.74680875498, Loss: 3.5036864280700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11517 , TFLOPS: 69.68592117003064, Tokens per sec: 172689.38919726814, Loss: 3.4892196655273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11518 , TFLOPS: 68.78468800678829, Tokens per sec: 170456.03413972497, Loss: 3.4948954582214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11519 , TFLOPS: 68.74058040520472, Tokens per sec: 170346.7306441475, Loss: 3.4936575889587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11520 , TFLOPS: 67.88870286142058, Tokens per sec: 168235.68424859204, Loss: 3.5222458839416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11521 , TFLOPS: 70.56107509052605, Tokens per sec: 174858.11701841865, Loss: 3.4960060119628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11522 , TFLOPS: 68.09185167636805, Tokens per sec: 168739.10939072128, Loss: 3.4727933406829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11523 , TFLOPS: 67.18830333641468, Tokens per sec: 166500.01707025038, Loss: 3.4961142539978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11524 , TFLOPS: 70.58308919186149, Tokens per sec: 174912.67038658133, Loss: 3.4876275062561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11525 , TFLOPS: 68.795873741161, Tokens per sec: 170483.75362171093, Loss: 3.508725643157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11526 , TFLOPS: 68.65687721308973, Tokens per sec: 170139.30491341872, Loss: 3.5096004009246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11527 , TFLOPS: 68.11768825455995, Tokens per sec: 168803.13527761484, Loss: 3.4924278259277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11528 , TFLOPS: 68.65560808647372, Tokens per sec: 170136.15987785842, Loss: 3.4994373321533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11529 , TFLOPS: 69.64004519436851, Tokens per sec: 172575.7034759215, Loss: 3.5054612159729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11530 , TFLOPS: 67.37874338454705, Tokens per sec: 166971.94848822558, Loss: 3.4710190296173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11531 , TFLOPS: 68.80239414373258, Tokens per sec: 170499.91189756495, Loss: 3.469310760498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11532 , TFLOPS: 70.56556112115373, Tokens per sec: 174869.23389649115, Loss: 3.481196880340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11533 , TFLOPS: 67.30836072721634, Tokens per sec: 166797.5325100703, Loss: 3.4931979179382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11534 , TFLOPS: 69.45886381922872, Tokens per sec: 172126.71606954897, Loss: 3.5178399085998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11535 , TFLOPS: 68.70567619681815, Tokens per sec: 170260.23417074955, Loss: 3.4890830516815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11536 , TFLOPS: 68.77890627118913, Tokens per sec: 170441.70636199947, Loss: 3.4923760890960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11537 , TFLOPS: 69.2003142496404, Tokens per sec: 171486.0017545228, Loss: 3.4635627269744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11538 , TFLOPS: 67.68049128109352, Tokens per sec: 167719.7130161426, Loss: 3.5066890716552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11539 , TFLOPS: 68.81489478854607, Tokens per sec: 170530.88987247247, Loss: 3.5011074542999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11540 , TFLOPS: 68.7547038165719, Tokens per sec: 170381.73001333754, Loss: 3.5123746395111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11541 , TFLOPS: 70.58485494104112, Tokens per sec: 174917.0461075618, Loss: 3.5032689571380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11542 , TFLOPS: 67.56273361551462, Tokens per sec: 167427.8965487603, Loss: 3.501673460006714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11543 , TFLOPS: 68.16619470755492, Tokens per sec: 168923.33961156328, Loss: 3.498593807220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11544 , TFLOPS: 70.53696591946019, Tokens per sec: 174798.37183667303, Loss: 3.5128109455108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11545 , TFLOPS: 67.89426760713238, Tokens per sec: 168249.47430147324, Loss: 3.496227979660034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11546 , TFLOPS: 68.6861213808566, Tokens per sec: 170211.7751826561, Loss: 3.4929068088531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11547 , TFLOPS: 68.88194628816228, Tokens per sec: 170697.05087485389, Loss: 3.484781265258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11548 , TFLOPS: 68.65369660145126, Tokens per sec: 170131.4230074053, Loss: 3.466414451599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11549 , TFLOPS: 68.61551370440073, Tokens per sec: 170036.80158232074, Loss: 3.5108375549316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11550 , TFLOPS: 67.9329797691094, Tokens per sec: 168345.40730334897, Loss: 3.525012731552124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11551 , TFLOPS: 69.74815725967522, Tokens per sec: 172843.61708327866, Loss: 3.498883008956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11552 , TFLOPS: 69.82366180173995, Tokens per sec: 173030.725656027, Loss: 3.5063276290893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11553 , TFLOPS: 66.99509552795068, Tokens per sec: 166021.2268372795, Loss: 3.5118093490600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11554 , TFLOPS: 69.89990547999298, Tokens per sec: 173219.66588967253, Loss: 3.49033784866333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11555 , TFLOPS: 68.6892807932421, Tokens per sec: 170219.60455458533, Loss: 3.4913549423217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11556 , TFLOPS: 67.75144375140799, Tokens per sec: 167895.54105364165, Loss: 3.4973127841949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11557 , TFLOPS: 68.92638148477566, Tokens per sec: 170807.16618700282, Loss: 3.490215301513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11558 , TFLOPS: 68.86751093387662, Tokens per sec: 170661.27847674844, Loss: 3.506235122680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11559 , TFLOPS: 68.67844734859762, Tokens per sec: 170192.75808535385, Loss: 3.494786024093628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11560 , TFLOPS: 67.1512769090738, Tokens per sec: 166408.261504502, Loss: 3.4787471294403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11561 , TFLOPS: 70.54516824108077, Tokens per sec: 174818.69809320915, Loss: 3.4795141220092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11562 , TFLOPS: 68.68068597062009, Tokens per sec: 170198.3056373301, Loss: 3.5161776542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11563 , TFLOPS: 68.21380190713921, Tokens per sec: 169041.3154964991, Loss: 3.512300491333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11564 , TFLOPS: 67.26353648590995, Tokens per sec: 166686.45310231528, Loss: 3.5015783309936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11565 , TFLOPS: 68.69174021310809, Tokens per sec: 170225.6992679407, Loss: 3.5125393867492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11566 , TFLOPS: 68.72964090017285, Tokens per sec: 170319.62134559255, Loss: 3.5051610469818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11567 , TFLOPS: 67.70154646409067, Tokens per sec: 167771.8900790294, Loss: 3.4918625354766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11568 , TFLOPS: 69.68728069211872, Tokens per sec: 172692.75824276614, Loss: 3.494199275970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11569 , TFLOPS: 68.90663179634207, Tokens per sec: 170758.2242834566, Loss: 3.481968402862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11570 , TFLOPS: 67.93204565673528, Tokens per sec: 168343.09246998493, Loss: 3.509860038757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11571 , TFLOPS: 69.6370933083086, Tokens per sec: 172568.38837708792, Loss: 3.4988293647766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11572 , TFLOPS: 69.11848125599613, Tokens per sec: 171283.2105816239, Loss: 3.4965593814849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11573 , TFLOPS: 67.94594725710976, Tokens per sec: 168377.5421671006, Loss: 3.502687931060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11574 , TFLOPS: 69.72566479157722, Tokens per sec: 172787.8782695819, Loss: 3.4957175254821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11575 , TFLOPS: 67.73482972437412, Tokens per sec: 167854.36966446557, Loss: 3.5116708278656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11576 , TFLOPS: 68.713026608587, Tokens per sec: 170278.44930082752, Loss: 3.482135057449341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11577 , TFLOPS: 68.83162039052111, Tokens per sec: 170572.33775664377, Loss: 3.496582508087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11578 , TFLOPS: 68.95386810372456, Tokens per sec: 170875.2810566592, Loss: 3.4808847904205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11579 , TFLOPS: 68.74761544389047, Tokens per sec: 170364.16424498422, Loss: 3.4739484786987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11580 , TFLOPS: 68.2760046890914, Tokens per sec: 169195.46084237873, Loss: 3.507066011428833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11581 , TFLOPS: 70.59503357122932, Tokens per sec: 174942.26987443684, Loss: 3.5093233585357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11582 , TFLOPS: 67.69332733630822, Tokens per sec: 167751.52217497278, Loss: 3.5228843688964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11583 , TFLOPS: 67.8428403560489, Tokens per sec: 168122.03190810897, Loss: 3.4950153827667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11584 , TFLOPS: 69.72920800175943, Tokens per sec: 172796.65873473024, Loss: 3.500502586364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11585 , TFLOPS: 68.84692405639768, Tokens per sec: 170610.26192652388, Loss: 3.5046069622039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11586 , TFLOPS: 67.8418103479222, Tokens per sec: 168119.47943450627, Loss: 3.504640579223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11587 , TFLOPS: 68.81220497871868, Tokens per sec: 170524.2242274132, Loss: 3.506793737411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11588 , TFLOPS: 70.58201128717069, Tokens per sec: 174909.99922001653, Loss: 3.512005567550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11589 , TFLOPS: 68.0987854180451, Tokens per sec: 168756.29196641166, Loss: 3.509641647338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11590 , TFLOPS: 66.94676233783409, Tokens per sec: 165901.45186783015, Loss: 3.4970805644989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11591 , TFLOPS: 70.46960355938731, Tokens per sec: 174631.44048783588, Loss: 3.515350341796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11592 , TFLOPS: 68.84128031934398, Tokens per sec: 170596.27612439584, Loss: 3.4999849796295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11593 , TFLOPS: 68.03000170230892, Tokens per sec: 168585.83834166566, Loss: 3.5156428813934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11594 , TFLOPS: 68.59231590147448, Tokens per sec: 169979.31487122053, Loss: 3.485928535461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11595 , TFLOPS: 68.66475399241172, Tokens per sec: 170158.8244402775, Loss: 3.4660451412200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11596 , TFLOPS: 69.7278649801983, Tokens per sec: 172793.33057935574, Loss: 3.52830171585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11597 , TFLOPS: 67.310355070177, Tokens per sec: 166802.47471162182, Loss: 3.515562057495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11598 , TFLOPS: 68.77325427590985, Tokens per sec: 170427.70009507876, Loss: 3.4671223163604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11599 , TFLOPS: 70.54721631346942, Tokens per sec: 174823.773442202, Loss: 3.5092246532440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11600 , TFLOPS: 67.48225614492743, Tokens per sec: 167228.46451132005, Loss: 3.4763944149017334 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/optimizer.pt +[rank0]:[2024-08-25 21:18:05,888] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063538649992551655, 'preprocessing_with_comm': 0.0013103280070936307, 'state_converting': 0.9981884230073774, : 1.0072627549961908}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011600, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009600 + eval ppl=23.730653762817383, eval loss=3.1667675971984863 +------------------------------------------------------------------ +iteration: 11601 , TFLOPS: 63.74553145335259, Tokens per sec: 157968.44909139146, Loss: 3.494112730026245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11602 , TFLOPS: 68.62951978986342, Tokens per sec: 170071.510204995, Loss: 3.4986772537231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11603 , TFLOPS: 68.6884342730789, Tokens per sec: 170217.506784369, Loss: 3.5034241676330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11604 , TFLOPS: 68.00610725119304, Tokens per sec: 168526.6252596094, Loss: 3.4855520725250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11605 , TFLOPS: 68.88376438156827, Tokens per sec: 170701.55631059344, Loss: 3.4927892684936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11606 , TFLOPS: 69.64953156925837, Tokens per sec: 172599.211757909, Loss: 3.4817659854888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11607 , TFLOPS: 68.09222598859374, Tokens per sec: 168740.0369776504, Loss: 3.5026841163635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11608 , TFLOPS: 69.4137574390854, Tokens per sec: 172014.93749067586, Loss: 3.4769198894500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11609 , TFLOPS: 68.80737333553087, Tokens per sec: 170512.25088334436, Loss: 3.482475519180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11610 , TFLOPS: 68.08392572526365, Tokens per sec: 168719.468010769, Loss: 3.519843816757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11611 , TFLOPS: 69.59945684250924, Tokens per sec: 172475.12106883817, Loss: 3.4893641471862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11612 , TFLOPS: 67.72779212662418, Tokens per sec: 167836.92972198583, Loss: 3.4794182777404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11613 , TFLOPS: 68.82671682941866, Tokens per sec: 170560.18619206006, Loss: 3.5089335441589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11614 , TFLOPS: 68.85445728426293, Tokens per sec: 170628.93009502755, Loss: 3.4927010536193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11615 , TFLOPS: 69.73027088997871, Tokens per sec: 172799.29268882444, Loss: 3.51027250289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11616 , TFLOPS: 67.97470570893212, Tokens per sec: 168448.80877872137, Loss: 3.506049633026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11617 , TFLOPS: 67.88439181111116, Tokens per sec: 168225.0009910239, Loss: 3.5018727779388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11618 , TFLOPS: 70.54807036949883, Tokens per sec: 174825.88988712494, Loss: 3.4736952781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11619 , TFLOPS: 67.93890304570694, Tokens per sec: 168360.08583526046, Loss: 3.524451971054077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11620 , TFLOPS: 67.96052396265308, Tokens per sec: 168413.66484919362, Loss: 3.502840042114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11621 , TFLOPS: 70.00070765687688, Tokens per sec: 173469.46478826727, Loss: 3.4690377712249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11622 , TFLOPS: 68.78674473684785, Tokens per sec: 170461.13094338006, Loss: 3.50009822845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11623 , TFLOPS: 67.6444983702871, Tokens per sec: 167630.51861821781, Loss: 3.4831221103668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11624 , TFLOPS: 68.86019271698031, Tokens per sec: 170643.14312911127, Loss: 3.4896764755249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11625 , TFLOPS: 68.67879374018047, Tokens per sec: 170193.61648185435, Loss: 3.5241429805755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11626 , TFLOPS: 69.38133327159709, Tokens per sec: 171934.5868318229, Loss: 3.5092811584472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11627 , TFLOPS: 67.98620007217866, Tokens per sec: 168477.29307705496, Loss: 3.488309383392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11628 , TFLOPS: 70.55371468774726, Tokens per sec: 174839.87712951752, Loss: 3.504373788833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11629 , TFLOPS: 68.88469356281412, Tokens per sec: 170703.8589240784, Loss: 3.5141289234161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11630 , TFLOPS: 66.957316110197, Tokens per sec: 165927.60527833947, Loss: 3.5006513595581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11631 , TFLOPS: 69.74642837318976, Tokens per sec: 172839.3327121151, Loss: 3.4905195236206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11632 , TFLOPS: 68.66761919105649, Tokens per sec: 170165.9247181469, Loss: 3.4927914142608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11633 , TFLOPS: 68.68286430121407, Tokens per sec: 170203.70378050493, Loss: 3.514432430267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11634 , TFLOPS: 68.52445462438767, Tokens per sec: 169811.14715106267, Loss: 3.4998013973236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11635 , TFLOPS: 70.5938959817556, Tokens per sec: 174939.45080240513, Loss: 3.4685046672821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11636 , TFLOPS: 68.1642224058044, Tokens per sec: 168918.4520305585, Loss: 3.474879503250122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11637 , TFLOPS: 68.82617786412408, Tokens per sec: 170558.85057668813, Loss: 3.508983612060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11638 , TFLOPS: 69.1141416568341, Tokens per sec: 171272.45657685405, Loss: 3.4601385593414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11639 , TFLOPS: 68.78910123409601, Tokens per sec: 170466.9706031506, Loss: 3.5155391693115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11640 , TFLOPS: 68.79069393436802, Tokens per sec: 170470.91748987508, Loss: 3.4949228763580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11641 , TFLOPS: 68.0391217539103, Tokens per sec: 168608.43883419046, Loss: 3.4863505363464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11642 , TFLOPS: 68.7437233092469, Tokens per sec: 170354.51910658344, Loss: 3.4885783195495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11643 , TFLOPS: 69.68437286089296, Tokens per sec: 172685.5523166647, Loss: 3.48686146736145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11644 , TFLOPS: 68.25476527787328, Tokens per sec: 169142.82724166595, Loss: 3.489168405532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11645 , TFLOPS: 67.81650351432205, Tokens per sec: 168056.76631306726, Loss: 3.477250576019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11646 , TFLOPS: 70.58040523255085, Tokens per sec: 174906.01923974705, Loss: 3.4901015758514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11647 , TFLOPS: 67.2814177307556, Tokens per sec: 166730.76479682451, Loss: 3.4752044677734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11648 , TFLOPS: 70.53524996507426, Tokens per sec: 174794.1195126767, Loss: 3.4790029525756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11649 , TFLOPS: 67.73600212940552, Tokens per sec: 167857.27501328444, Loss: 3.4822793006896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11650 , TFLOPS: 67.98805681827227, Tokens per sec: 168481.89429841243, Loss: 3.4876232147216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11651 , TFLOPS: 69.65642352229935, Tokens per sec: 172616.2907767559, Loss: 3.5302574634552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11652 , TFLOPS: 68.7684106202511, Tokens per sec: 170415.69698278367, Loss: 3.4848861694335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11653 , TFLOPS: 67.80160204680737, Tokens per sec: 168019.83883503373, Loss: 3.506253957748413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11654 , TFLOPS: 68.78320980655904, Tokens per sec: 170452.37099671972, Loss: 3.5035953521728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11655 , TFLOPS: 70.60395538276457, Tokens per sec: 174964.37910623985, Loss: 3.483114719390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11656 , TFLOPS: 66.77769081065013, Tokens per sec: 165482.47399870207, Loss: 3.467827081680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11657 , TFLOPS: 68.76347462077766, Tokens per sec: 170403.46503234323, Loss: 3.489281177520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11658 , TFLOPS: 69.71875173081968, Tokens per sec: 172770.74694348822, Loss: 3.516683578491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11659 , TFLOPS: 68.84399494374017, Tokens per sec: 170603.00326269018, Loss: 3.503174304962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11660 , TFLOPS: 68.66845544559978, Tokens per sec: 170167.99704902622, Loss: 3.512845039367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11661 , TFLOPS: 68.74979925415306, Tokens per sec: 170369.57596738148, Loss: 3.484461784362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11662 , TFLOPS: 68.6757820917947, Tokens per sec: 170186.15328539393, Loss: 3.514469861984253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11663 , TFLOPS: 68.55176736023245, Tokens per sec: 169878.83111923223, Loss: 3.468200206756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11664 , TFLOPS: 67.91160464602966, Tokens per sec: 168292.4373936932, Loss: 3.496959686279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11665 , TFLOPS: 69.8054359935078, Tokens per sec: 172985.5600954892, Loss: 3.496347427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11666 , TFLOPS: 69.84589817572325, Tokens per sec: 173085.82984029644, Loss: 3.519087314605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11667 , TFLOPS: 67.84138519039351, Tokens per sec: 168118.4258473152, Loss: 3.488994598388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11668 , TFLOPS: 69.93459221534432, Tokens per sec: 173305.6234409325, Loss: 3.499762773513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11669 , TFLOPS: 67.90211629639644, Tokens per sec: 168268.92421807407, Loss: 3.4969542026519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11670 , TFLOPS: 68.00332741239082, Tokens per sec: 168519.73651283295, Loss: 3.485762119293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11671 , TFLOPS: 68.75685473500937, Tokens per sec: 170387.0602261688, Loss: 3.4927003383636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11672 , TFLOPS: 68.76106192935555, Tokens per sec: 170397.48611721938, Loss: 3.493290662765503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11673 , TFLOPS: 69.59987242544761, Tokens per sec: 172476.1509291397, Loss: 3.5059850215911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11674 , TFLOPS: 68.10943576532176, Tokens per sec: 168782.68470019585, Loss: 3.509105920791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11675 , TFLOPS: 69.4948830018876, Tokens per sec: 172215.9755720182, Loss: 3.4791040420532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11676 , TFLOPS: 67.72109860180365, Tokens per sec: 167820.3424301279, Loss: 3.486586093902588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11677 , TFLOPS: 68.91027757425553, Tokens per sec: 170767.25892274178, Loss: 3.5092177391052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11678 , TFLOPS: 68.78645872429666, Tokens per sec: 170460.42217277136, Loss: 3.5127692222595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11679 , TFLOPS: 68.86175640457249, Tokens per sec: 170647.01811921908, Loss: 3.488300323486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11680 , TFLOPS: 68.68997129411348, Tokens per sec: 170221.31569180955, Loss: 3.4812779426574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11681 , TFLOPS: 68.84659002234672, Tokens per sec: 170609.43415334873, Loss: 3.4871420860290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11682 , TFLOPS: 68.64961114207476, Tokens per sec: 170121.29878901984, Loss: 3.4991209506988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11683 , TFLOPS: 68.87198947407757, Tokens per sec: 170672.37679271834, Loss: 3.512139081954956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11684 , TFLOPS: 67.97408410167262, Tokens per sec: 168447.26836745665, Loss: 3.5076169967651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11685 , TFLOPS: 70.5290806782015, Tokens per sec: 174778.83134020297, Loss: 3.5049362182617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11686 , TFLOPS: 69.2186647673082, Tokens per sec: 171531.47635878003, Loss: 3.4973082542419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11687 , TFLOPS: 67.88554726402008, Tokens per sec: 168227.86433061582, Loss: 3.4784069061279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11688 , TFLOPS: 69.68808290268537, Tokens per sec: 172694.74620892107, Loss: 3.518035888671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11689 , TFLOPS: 68.68188032622453, Tokens per sec: 170201.26538208706, Loss: 3.5081303119659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11690 , TFLOPS: 67.75114472110253, Tokens per sec: 167894.80002360384, Loss: 3.5040388107299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11691 , TFLOPS: 69.07271358856579, Tokens per sec: 171169.7932600067, Loss: 3.4730851650238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11692 , TFLOPS: 68.80759955346379, Tokens per sec: 170512.81147630207, Loss: 3.495736837387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11693 , TFLOPS: 68.71170568079388, Tokens per sec: 170275.17589624395, Loss: 3.4900617599487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11694 , TFLOPS: 68.03597291386413, Tokens per sec: 168600.6356616827, Loss: 3.468343734741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11695 , TFLOPS: 70.57749020695326, Tokens per sec: 174898.79548519963, Loss: 3.4910295009613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11696 , TFLOPS: 68.66920117616961, Tokens per sec: 170169.84505152729, Loss: 3.501354694366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11697 , TFLOPS: 67.13165696233675, Tokens per sec: 166359.64111517204, Loss: 3.52826189994812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11698 , TFLOPS: 69.6217959201402, Tokens per sec: 172530.47976406023, Loss: 3.499608278274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11699 , TFLOPS: 68.85210008572342, Tokens per sec: 170623.08869737978, Loss: 3.49306058883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11700 , TFLOPS: 67.74914396529587, Tokens per sec: 167889.84193031347, Loss: 3.5224075317382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11701 , TFLOPS: 68.63271519969591, Tokens per sec: 170079.4287825633, Loss: 3.48474383354187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11702 , TFLOPS: 70.55257213043852, Tokens per sec: 174837.0457466434, Loss: 3.499941825866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11703 , TFLOPS: 68.02160378410284, Tokens per sec: 168565.0273752439, Loss: 3.499185085296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11704 , TFLOPS: 67.53324106913324, Tokens per sec: 167354.81076995507, Loss: 3.480879545211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11705 , TFLOPS: 69.91766741787856, Tokens per sec: 173263.68192839198, Loss: 3.505694627761841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11706 , TFLOPS: 68.88157738468126, Tokens per sec: 170696.13669138984, Loss: 3.4939939975738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11707 , TFLOPS: 68.66398744887158, Tokens per sec: 170156.92486094325, Loss: 3.485273838043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11708 , TFLOPS: 68.20343546174826, Tokens per sec: 169015.6263028621, Loss: 3.491605758666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11709 , TFLOPS: 68.69037034746027, Tokens per sec: 170222.30458996227, Loss: 3.504932165145874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11710 , TFLOPS: 69.56867493247591, Tokens per sec: 172398.84010486823, Loss: 3.4670791625976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11711 , TFLOPS: 68.2665545235149, Tokens per sec: 169172.0422910585, Loss: 3.4955825805664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11712 , TFLOPS: 68.70018253652032, Tokens per sec: 170246.6202753549, Loss: 3.48738431930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11713 , TFLOPS: 69.9154110561841, Tokens per sec: 173258.09041555435, Loss: 3.488342046737671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11714 , TFLOPS: 66.91778431356998, Tokens per sec: 165829.6411315108, Loss: 3.4837851524353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11715 , TFLOPS: 69.38633963354698, Tokens per sec: 171946.99314823103, Loss: 3.4979395866394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11716 , TFLOPS: 68.27776507455565, Tokens per sec: 169199.8232714816, Loss: 3.4862585067749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11717 , TFLOPS: 67.25899235901252, Tokens per sec: 166675.19225528653, Loss: 3.4996674060821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11718 , TFLOPS: 69.17380046747697, Tokens per sec: 171420.2976815876, Loss: 3.496933937072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11719 , TFLOPS: 66.894639047127, Tokens per sec: 165772.2846116035, Loss: 3.5148816108703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11720 , TFLOPS: 67.90975115865461, Tokens per sec: 168287.84424779934, Loss: 3.5102627277374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11721 , TFLOPS: 68.05090324225293, Tokens per sec: 168637.6346601416, Loss: 3.4865570068359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11722 , TFLOPS: 70.22921953336669, Tokens per sec: 174035.7423908703, Loss: 3.4769790172576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11723 , TFLOPS: 66.61277065645261, Tokens per sec: 165073.78368914587, Loss: 3.492187023162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11724 , TFLOPS: 67.86754828913595, Tokens per sec: 168183.26088810223, Loss: 3.4984662532806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11725 , TFLOPS: 69.31621614667293, Tokens per sec: 171773.21942301738, Loss: 3.4744150638580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11726 , TFLOPS: 68.63350362641569, Tokens per sec: 170081.38259082733, Loss: 3.4964935779571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11727 , TFLOPS: 67.41168048354156, Tokens per sec: 167053.57024785492, Loss: 3.4705443382263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11728 , TFLOPS: 68.68809764918781, Tokens per sec: 170216.6725932847, Loss: 3.4907052516937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11729 , TFLOPS: 68.74445446564924, Tokens per sec: 170356.33099269198, Loss: 3.476585865020752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11730 , TFLOPS: 68.54473541035351, Tokens per sec: 169861.405172799, Loss: 3.4878265857696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11731 , TFLOPS: 67.7739056508676, Tokens per sec: 167951.20411488507, Loss: 3.494023084640503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11732 , TFLOPS: 70.53789605706899, Tokens per sec: 174800.67682013076, Loss: 3.490215301513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11733 , TFLOPS: 68.86003728261161, Tokens per sec: 170642.7579456231, Loss: 3.486955165863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11734 , TFLOPS: 65.90358936718206, Tokens per sec: 163316.35433156524, Loss: 3.484508991241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11735 , TFLOPS: 70.53345352615788, Tokens per sec: 174789.66773914784, Loss: 3.4898552894592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11736 , TFLOPS: 68.75843845041429, Tokens per sec: 170390.98484740287, Loss: 3.5120067596435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11737 , TFLOPS: 67.56992490396352, Tokens per sec: 167445.71735372252, Loss: 3.493706226348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11738 , TFLOPS: 69.65066504955655, Tokens per sec: 172602.0206469511, Loss: 3.495887279510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11739 , TFLOPS: 68.63165233132536, Tokens per sec: 170076.79487765647, Loss: 3.5077521800994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11740 , TFLOPS: 68.76441071449972, Tokens per sec: 170405.78477570575, Loss: 3.517707109451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11741 , TFLOPS: 67.7606448218892, Tokens per sec: 167918.34231987566, Loss: 3.4776062965393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11742 , TFLOPS: 70.59545290132931, Tokens per sec: 174943.30902061763, Loss: 3.4671404361724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11743 , TFLOPS: 67.80219387222027, Tokens per sec: 168021.30544360183, Loss: 3.498340606689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11744 , TFLOPS: 68.61427692737695, Tokens per sec: 170033.73671261358, Loss: 3.497910737991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11745 , TFLOPS: 68.74580582464435, Tokens per sec: 170359.67980914633, Loss: 3.4833626747131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11746 , TFLOPS: 68.82358240050316, Tokens per sec: 170552.41873192159, Loss: 3.522075653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11747 , TFLOPS: 68.68165482709149, Tokens per sec: 170200.7065703946, Loss: 3.5173697471618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11748 , TFLOPS: 68.62452593897426, Tokens per sec: 170059.13489237355, Loss: 3.5144426822662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11749 , TFLOPS: 68.59746269330714, Tokens per sec: 169992.0691883478, Loss: 3.5332489013671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11750 , TFLOPS: 69.04517695442732, Tokens per sec: 171101.55444720094, Loss: 3.484117031097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11751 , TFLOPS: 68.018297113011, Tokens per sec: 168556.83308001843, Loss: 3.490738868713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11752 , TFLOPS: 69.48417128030779, Tokens per sec: 172189.4307459499, Loss: 3.485424757003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11753 , TFLOPS: 68.6486190937265, Tokens per sec: 170118.8403839297, Loss: 3.4974710941314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11754 , TFLOPS: 67.74249742902309, Tokens per sec: 167873.37108125314, Loss: 3.507779598236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11755 , TFLOPS: 69.86930505461754, Tokens per sec: 173143.83466467704, Loss: 3.5014760494232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11756 , TFLOPS: 68.63689790753169, Tokens per sec: 170089.79399334372, Loss: 3.509199619293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11757 , TFLOPS: 68.71056345268131, Tokens per sec: 170272.3453291542, Loss: 3.4985926151275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11758 , TFLOPS: 69.16418217906678, Tokens per sec: 171396.46250336524, Loss: 3.514582633972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11759 , TFLOPS: 68.71863273639188, Tokens per sec: 170292.34190309394, Loss: 3.5054736137390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11760 , TFLOPS: 68.56123892295851, Tokens per sec: 169902.30269504618, Loss: 3.5007126331329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11761 , TFLOPS: 68.1614511342314, Tokens per sec: 168911.58451431984, Loss: 3.4871628284454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11762 , TFLOPS: 70.5199494585357, Tokens per sec: 174756.20317198423, Loss: 3.515056848526001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11763 , TFLOPS: 68.80488609952207, Tokens per sec: 170506.08723852286, Loss: 3.4984869956970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11764 , TFLOPS: 67.03798522633483, Tokens per sec: 166127.51223456571, Loss: 3.5137455463409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11765 , TFLOPS: 69.78156587911666, Tokens per sec: 172926.4073225178, Loss: 3.4874725341796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11766 , TFLOPS: 68.72566081196298, Tokens per sec: 170309.75824856557, Loss: 3.526690721511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11767 , TFLOPS: 67.82405627248899, Tokens per sec: 168075.4828798099, Loss: 3.5357556343078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11768 , TFLOPS: 68.81186334864447, Tokens per sec: 170523.3776304556, Loss: 3.5053536891937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11769 , TFLOPS: 70.53108305344166, Tokens per sec: 174783.79344662794, Loss: 3.503815174102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11770 , TFLOPS: 68.07899636338303, Tokens per sec: 168707.25250901474, Loss: 3.507827043533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11771 , TFLOPS: 68.68763590413681, Tokens per sec: 170215.52833818292, Loss: 3.5193216800689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11772 , TFLOPS: 70.00696353766925, Tokens per sec: 173484.96755001292, Loss: 3.488748788833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11773 , TFLOPS: 68.79172908258376, Tokens per sec: 170473.48270118466, Loss: 3.470027208328247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11774 , TFLOPS: 67.6842084648955, Tokens per sec: 167728.92460708547, Loss: 3.484560966491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11775 , TFLOPS: 69.1091620917446, Tokens per sec: 171260.11666601786, Loss: 3.5053393840789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11776 , TFLOPS: 68.75752347687013, Tokens per sec: 170388.71744216778, Loss: 3.500843048095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11777 , TFLOPS: 69.73887075363278, Tokens per sec: 172820.60409831244, Loss: 3.486379384994507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11778 , TFLOPS: 67.42616530318423, Tokens per sec: 167089.4652265641, Loss: 3.4878430366516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11779 , TFLOPS: 68.73769890529843, Tokens per sec: 170339.58996995626, Loss: 3.5026257038116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11780 , TFLOPS: 70.51244151394432, Tokens per sec: 174737.59765821844, Loss: 3.4962098598480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11781 , TFLOPS: 67.22675916854979, Tokens per sec: 166595.3148585408, Loss: 3.5138370990753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11782 , TFLOPS: 69.47192438786247, Tokens per sec: 172159.0815973638, Loss: 3.4781196117401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11783 , TFLOPS: 68.74304644564289, Tokens per sec: 170352.84176401002, Loss: 3.529961109161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11784 , TFLOPS: 67.89488709367467, Tokens per sec: 168251.0094573671, Loss: 3.4786479473114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11785 , TFLOPS: 69.63328847269281, Tokens per sec: 172558.95957529478, Loss: 3.497586965560913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11786 , TFLOPS: 67.77906477051903, Tokens per sec: 167963.98898170525, Loss: 3.4961047172546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11787 , TFLOPS: 68.77708824087782, Tokens per sec: 170437.20108261553, Loss: 3.500309705734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11788 , TFLOPS: 68.75466313711452, Tokens per sec: 170381.62920516092, Loss: 3.484884738922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11789 , TFLOPS: 69.6477338767826, Tokens per sec: 172594.75687792187, Loss: 3.46502685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11790 , TFLOPS: 67.90209898354325, Tokens per sec: 168268.8813149167, Loss: 3.49691104888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11791 , TFLOPS: 68.9283936074342, Tokens per sec: 170812.1524486043, Loss: 3.480536937713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11792 , TFLOPS: 69.76388461731155, Tokens per sec: 172882.59120801295, Loss: 3.487009048461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11793 , TFLOPS: 67.83543748961918, Tokens per sec: 168103.6867896031, Loss: 3.4917914867401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11794 , TFLOPS: 68.63944584092533, Tokens per sec: 170096.1080529712, Loss: 3.494297504425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11795 , TFLOPS: 68.8452363186567, Tokens per sec: 170606.07952648206, Loss: 3.477982997894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11796 , TFLOPS: 68.5587515602721, Tokens per sec: 169896.13873630363, Loss: 3.4829156398773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11797 , TFLOPS: 68.79977369691882, Tokens per sec: 170493.41814169436, Loss: 3.5008511543273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11798 , TFLOPS: 67.99428033286293, Tokens per sec: 168497.3168531448, Loss: 3.498656749725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11799 , TFLOPS: 69.74249197297101, Tokens per sec: 172829.57787874277, Loss: 3.4829845428466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11800 , TFLOPS: 69.81844398270425, Tokens per sec: 173017.79532566635, Loss: 3.5184359550476074 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/optimizer.pt +[rank0]:[2024-08-25 21:43:54,160] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006361777006532066, 'preprocessing_with_comm': 0.0015202399954432622, 'state_converting': 1.0017831750010373, : 1.0110738450021017}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011800, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0009800 + eval ppl=24.72349739074707, eval loss=3.207754135131836 +------------------------------------------------------------------ +iteration: 11801 , TFLOPS: 64.04448516860403, Tokens per sec: 158709.2893302549, Loss: 3.474026679992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11802 , TFLOPS: 69.59303248463806, Tokens per sec: 172459.2008023319, Loss: 3.5155270099639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11803 , TFLOPS: 69.53715280795954, Tokens per sec: 172320.724807868, Loss: 3.4844138622283936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11804 , TFLOPS: 67.53455800336475, Tokens per sec: 167358.07427805304, Loss: 3.5207555294036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11805 , TFLOPS: 68.85338331050426, Tokens per sec: 170626.26866974556, Loss: 3.4929890632629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11806 , TFLOPS: 69.66562804817305, Tokens per sec: 172639.1006058331, Loss: 3.494946241378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11807 , TFLOPS: 68.74513566831041, Tokens per sec: 170358.01908792704, Loss: 3.4992353916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11808 , TFLOPS: 67.81786459102314, Tokens per sec: 168060.1392110675, Loss: 3.4945337772369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11809 , TFLOPS: 70.51919932104444, Tokens per sec: 174754.3442486464, Loss: 3.508892297744751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11810 , TFLOPS: 68.01712088380964, Tokens per sec: 168553.91825448512, Loss: 3.486694812774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11811 , TFLOPS: 68.75861517931895, Tokens per sec: 170391.4228010969, Loss: 3.48860502243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11812 , TFLOPS: 69.02289328345792, Tokens per sec: 171046.33305578973, Loss: 3.4805684089660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11813 , TFLOPS: 69.64262052359561, Tokens per sec: 172582.08542544217, Loss: 3.5076749324798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11814 , TFLOPS: 69.58973471859352, Tokens per sec: 172451.02857479465, Loss: 3.5118212699890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11815 , TFLOPS: 68.17309704729391, Tokens per sec: 168940.44436979198, Loss: 3.5251572132110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11816 , TFLOPS: 68.69348475735387, Tokens per sec: 170230.02244075917, Loss: 3.481799602508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11817 , TFLOPS: 68.72362635061053, Tokens per sec: 170304.71662921977, Loss: 3.489576816558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11818 , TFLOPS: 69.66793961186475, Tokens per sec: 172644.82891530092, Loss: 3.463327169418335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11819 , TFLOPS: 67.94767329251187, Tokens per sec: 168381.8194729655, Loss: 3.4985320568084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11820 , TFLOPS: 70.53480565806309, Tokens per sec: 174793.0184709586, Loss: 3.513711929321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11821 , TFLOPS: 68.09178447176289, Tokens per sec: 168738.94285030646, Loss: 3.476010322570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11822 , TFLOPS: 69.73406656559865, Tokens per sec: 172808.69879113883, Loss: 3.497917890548706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11823 , TFLOPS: 67.77703528549603, Tokens per sec: 167958.959694252, Loss: 3.4947669506073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11824 , TFLOPS: 68.66159397757276, Tokens per sec: 170150.9935754022, Loss: 3.503840208053589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11825 , TFLOPS: 68.86209810038426, Tokens per sec: 170647.86487908155, Loss: 3.492339611053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11826 , TFLOPS: 68.74966134254053, Tokens per sec: 170369.23420721345, Loss: 3.5058767795562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11827 , TFLOPS: 67.84816891822913, Tokens per sec: 168135.2366721812, Loss: 3.499685049057007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11828 , TFLOPS: 68.66617198744243, Tokens per sec: 170162.33838816863, Loss: 3.4880731105804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11829 , TFLOPS: 70.56768657469051, Tokens per sec: 174874.5010044933, Loss: 3.481940269470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11830 , TFLOPS: 67.09090769944234, Tokens per sec: 166258.6599528178, Loss: 3.482652425765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11831 , TFLOPS: 69.60361209961259, Tokens per sec: 172485.41825367959, Loss: 3.495229482650757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11832 , TFLOPS: 69.95979783497931, Tokens per sec: 173368.0857430162, Loss: 3.5153861045837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11833 , TFLOPS: 69.22489574294862, Tokens per sec: 171546.9174028156, Loss: 3.490089178085327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11834 , TFLOPS: 68.61129507502085, Tokens per sec: 170026.347353997, Loss: 3.4956679344177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11835 , TFLOPS: 68.83864707280485, Tokens per sec: 170589.7506493958, Loss: 3.5025641918182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11836 , TFLOPS: 68.71659531964191, Tokens per sec: 170287.29295994755, Loss: 3.4742722511291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11837 , TFLOPS: 68.7967113077355, Tokens per sec: 170485.82920394812, Loss: 3.5064353942871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11838 , TFLOPS: 68.95301589387583, Tokens per sec: 170873.1691867754, Loss: 3.5012731552124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11839 , TFLOPS: 69.58260087016805, Tokens per sec: 172433.35011253803, Loss: 3.4859085083007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11840 , TFLOPS: 70.01390147897388, Tokens per sec: 173502.16053284312, Loss: 3.4854159355163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11841 , TFLOPS: 67.01337513711528, Tokens per sec: 166066.52572245456, Loss: 3.5019474029541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11842 , TFLOPS: 69.80928731676121, Tokens per sec: 172995.10409876946, Loss: 3.4969632625579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11843 , TFLOPS: 68.74269200618616, Tokens per sec: 170351.96342399152, Loss: 3.510843276977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11844 , TFLOPS: 68.72569196485048, Tokens per sec: 170309.83544885257, Loss: 3.4917988777160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11845 , TFLOPS: 67.61374230193427, Tokens per sec: 167554.30169277886, Loss: 3.4876959323883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11846 , TFLOPS: 68.72496208169869, Tokens per sec: 170308.0267179992, Loss: 3.489201545715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11847 , TFLOPS: 68.75630107650458, Tokens per sec: 170385.68819940346, Loss: 3.4890315532684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11848 , TFLOPS: 69.63282326460738, Tokens per sec: 172557.80673841218, Loss: 3.4791109561920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11849 , TFLOPS: 68.80320646677221, Tokens per sec: 170501.9249235419, Loss: 3.538259983062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11850 , TFLOPS: 68.7480060796238, Tokens per sec: 170365.13228336352, Loss: 3.4888808727264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11851 , TFLOPS: 69.1024838710709, Tokens per sec: 171243.56729952176, Loss: 3.482987642288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11852 , TFLOPS: 68.84769016857734, Tokens per sec: 170612.16043689928, Loss: 3.491788864135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11853 , TFLOPS: 69.56645741054305, Tokens per sec: 172393.34484123843, Loss: 3.5014374256134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11854 , TFLOPS: 69.5160835918547, Tokens per sec: 172268.51296939442, Loss: 3.4750194549560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11855 , TFLOPS: 68.0422027558184, Tokens per sec: 168616.0738963191, Loss: 3.498880624771118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11856 , TFLOPS: 68.52119230228791, Tokens per sec: 169803.06275752498, Loss: 3.5144834518432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11857 , TFLOPS: 69.1779258879629, Tokens per sec: 171430.52092800432, Loss: 3.4839961528778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11858 , TFLOPS: 68.90415837264042, Tokens per sec: 170752.0948670538, Loss: 3.4980225563049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11859 , TFLOPS: 70.5485539600809, Tokens per sec: 174827.08827785918, Loss: 3.512572765350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11860 , TFLOPS: 68.96305332786332, Tokens per sec: 170898.04305391104, Loss: 3.5135393142700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11861 , TFLOPS: 67.90465890565082, Tokens per sec: 168275.22508389794, Loss: 3.4933907985687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11862 , TFLOPS: 69.73342606823526, Tokens per sec: 172807.11156811574, Loss: 3.477644681930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11863 , TFLOPS: 68.8254269038255, Tokens per sec: 170556.98961434318, Loss: 3.4885447025299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11864 , TFLOPS: 67.78221513599782, Tokens per sec: 167971.79593440308, Loss: 3.484616756439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11865 , TFLOPS: 68.81654810392487, Tokens per sec: 170534.98697010888, Loss: 3.505002737045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11866 , TFLOPS: 68.6593911474997, Tokens per sec: 170145.53471981938, Loss: 3.496307849884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11867 , TFLOPS: 68.72337628639121, Tokens per sec: 170304.09694253746, Loss: 3.502145290374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11868 , TFLOPS: 68.41996196394426, Tokens per sec: 169552.2028276671, Loss: 3.4949066638946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11869 , TFLOPS: 69.69148795392827, Tokens per sec: 172703.18430099834, Loss: 3.482454299926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11870 , TFLOPS: 68.87276453033311, Tokens per sec: 170674.29746749907, Loss: 3.5041630268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11871 , TFLOPS: 68.62624058026643, Tokens per sec: 170063.38396237884, Loss: 3.485358953475952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11872 , TFLOPS: 69.62357084894833, Tokens per sec: 172534.87823317133, Loss: 3.516991138458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11873 , TFLOPS: 69.51656132547862, Tokens per sec: 172269.69684594122, Loss: 3.507547378540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11874 , TFLOPS: 68.18448913880668, Tokens per sec: 168968.6752275031, Loss: 3.49674916267395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11875 , TFLOPS: 68.74125631875047, Tokens per sec: 170348.40563237187, Loss: 3.48926043510437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11876 , TFLOPS: 70.63317692669001, Tokens per sec: 175036.7933111059, Loss: 3.4858856201171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11877 , TFLOPS: 67.90133538745128, Tokens per sec: 168266.98903968374, Loss: 3.479712963104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11878 , TFLOPS: 67.47123546083581, Tokens per sec: 167201.1540421124, Loss: 3.526317596435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11879 , TFLOPS: 70.57618893542191, Tokens per sec: 174895.57079099806, Loss: 3.506012201309204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11880 , TFLOPS: 68.74813496821768, Tokens per sec: 170365.4516834974, Loss: 3.4945030212402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11881 , TFLOPS: 68.80227150704293, Tokens per sec: 170499.60799033844, Loss: 3.5054469108581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11882 , TFLOPS: 68.06170332011699, Tokens per sec: 168664.39844281552, Loss: 3.4978582859039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11883 , TFLOPS: 68.77504627530482, Tokens per sec: 170432.14086698453, Loss: 3.5289955139160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11884 , TFLOPS: 68.73165560590948, Tokens per sec: 170324.61400834605, Loss: 3.4879262447357178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11885 , TFLOPS: 68.95569392950225, Tokens per sec: 170879.80565406766, Loss: 3.498215675354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11886 , TFLOPS: 68.69340914060149, Tokens per sec: 170229.83505411688, Loss: 3.490297794342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11887 , TFLOPS: 69.89000611535353, Tokens per sec: 173195.13417359092, Loss: 3.494291305541992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11888 , TFLOPS: 68.2180675886039, Tokens per sec: 169051.8863250722, Loss: 3.5067360401153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11889 , TFLOPS: 68.6685046890243, Tokens per sec: 170168.1190796568, Loss: 3.469428300857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11890 , TFLOPS: 68.76127993949684, Tokens per sec: 170398.0263703653, Loss: 3.4807870388031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11891 , TFLOPS: 68.88244912317023, Tokens per sec: 170698.29695539543, Loss: 3.498549222946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11892 , TFLOPS: 69.55754024757299, Tokens per sec: 172371.24712909202, Loss: 3.497997760772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11893 , TFLOPS: 69.57655418521112, Tokens per sec: 172418.36576111746, Loss: 3.5378615856170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11894 , TFLOPS: 69.44647166949183, Tokens per sec: 172096.00695739296, Loss: 3.4910213947296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11895 , TFLOPS: 68.85785029663725, Tokens per sec: 170637.33835346226, Loss: 3.5192739963531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11896 , TFLOPS: 70.63050924575788, Tokens per sec: 175030.18250388678, Loss: 3.4916718006134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11897 , TFLOPS: 67.90612230893264, Tokens per sec: 168278.85155843748, Loss: 3.49931263923645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11898 , TFLOPS: 69.6940275108353, Tokens per sec: 172709.47760276927, Loss: 3.483388900756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11899 , TFLOPS: 69.81968677284362, Tokens per sec: 173020.87509653633, Loss: 3.508627414703369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11900 , TFLOPS: 68.72651503014119, Tokens per sec: 170311.87509531106, Loss: 3.5098979473114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11901 , TFLOPS: 67.88628845611883, Tokens per sec: 168229.70108628558, Loss: 3.5074105262756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11902 , TFLOPS: 68.86652213593572, Tokens per sec: 170658.82812652583, Loss: 3.5227019786834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11903 , TFLOPS: 68.78604944690474, Tokens per sec: 170459.40793830855, Loss: 3.4955554008483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11904 , TFLOPS: 68.62365378727084, Tokens per sec: 170056.97360437704, Loss: 3.492218255996704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11905 , TFLOPS: 68.24149444902557, Tokens per sec: 169109.94066587885, Loss: 3.479381561279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11906 , TFLOPS: 69.57909170946576, Tokens per sec: 172424.6540257518, Loss: 3.4820547103881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11907 , TFLOPS: 69.65454374918612, Tokens per sec: 172611.6324919072, Loss: 3.505927324295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11908 , TFLOPS: 66.76951463308914, Tokens per sec: 165462.21253002566, Loss: 3.4884393215179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11909 , TFLOPS: 69.61908967128656, Tokens per sec: 172523.773381283, Loss: 3.503492593765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11910 , TFLOPS: 68.79607420935541, Tokens per sec: 170484.25040398008, Loss: 3.4711196422576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11911 , TFLOPS: 67.88166197577098, Tokens per sec: 168218.23615833477, Loss: 3.4900450706481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11912 , TFLOPS: 69.79168153203756, Tokens per sec: 172951.47502478692, Loss: 3.4890737533569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11913 , TFLOPS: 69.59756431515095, Tokens per sec: 172470.43117756522, Loss: 3.4932637214660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11914 , TFLOPS: 69.5304236519628, Tokens per sec: 172304.04921803038, Loss: 3.4528443813323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11915 , TFLOPS: 68.61057587091479, Tokens per sec: 170024.5650869953, Loss: 3.505598545074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11916 , TFLOPS: 69.87848210311311, Tokens per sec: 173166.5764017844, Loss: 3.5043678283691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11917 , TFLOPS: 68.75131235513436, Tokens per sec: 170373.325598295, Loss: 3.498002529144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11918 , TFLOPS: 69.05880060178465, Tokens per sec: 171135.31534612106, Loss: 3.458991050720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11919 , TFLOPS: 68.81171715646222, Tokens per sec: 170523.01535012363, Loss: 3.502389907836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11920 , TFLOPS: 68.70697881025865, Tokens per sec: 170263.46219034985, Loss: 3.4979066848754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11921 , TFLOPS: 68.70621084231163, Tokens per sec: 170261.5590811785, Loss: 3.5081727504730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11922 , TFLOPS: 68.67431673721768, Tokens per sec: 170182.52197532993, Loss: 3.5282177925109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11923 , TFLOPS: 68.54124376600727, Tokens per sec: 169852.75249347262, Loss: 3.5081076622009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11924 , TFLOPS: 68.9614110832592, Tokens per sec: 170893.97339087384, Loss: 3.483754873275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11925 , TFLOPS: 68.05689291739199, Tokens per sec: 168652.47773495334, Loss: 3.5203824043273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11926 , TFLOPS: 69.60665851869668, Tokens per sec: 172492.96761576078, Loss: 3.504218339920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11927 , TFLOPS: 68.98873286941428, Tokens per sec: 170961.67978670934, Loss: 3.5147929191589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11928 , TFLOPS: 68.87605272820657, Tokens per sec: 170682.44598405442, Loss: 3.5056116580963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11929 , TFLOPS: 69.103783273721, Tokens per sec: 171246.78736242972, Loss: 3.4984583854675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11930 , TFLOPS: 68.52085715598194, Tokens per sec: 169802.2322280594, Loss: 3.5033717155456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11931 , TFLOPS: 68.65668152619803, Tokens per sec: 170138.81997974438, Loss: 3.5021514892578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11932 , TFLOPS: 68.9928141903433, Tokens per sec: 170971.79374956607, Loss: 3.481813430786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11933 , TFLOPS: 69.61342668634663, Tokens per sec: 172509.73988077257, Loss: 3.4942116737365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11934 , TFLOPS: 69.6676040542314, Tokens per sec: 172643.99736652078, Loss: 3.4817800521850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11935 , TFLOPS: 68.96157906611573, Tokens per sec: 170894.3896708972, Loss: 3.5011932849884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11936 , TFLOPS: 69.68367717285275, Tokens per sec: 172683.82832506465, Loss: 3.4906022548675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11937 , TFLOPS: 68.77242901892045, Tokens per sec: 170425.65501734946, Loss: 3.5159599781036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11938 , TFLOPS: 67.77377927001619, Tokens per sec: 167950.8909292133, Loss: 3.4961283206939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11939 , TFLOPS: 69.70724244337352, Tokens per sec: 172742.22566134395, Loss: 3.5015852451324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11940 , TFLOPS: 68.71898589305911, Tokens per sec: 170293.21706421874, Loss: 3.505084991455078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11941 , TFLOPS: 67.86507338908272, Tokens per sec: 168177.1278131376, Loss: 3.494462013244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11942 , TFLOPS: 68.77319243769713, Tokens per sec: 170427.5468531756, Loss: 3.4881534576416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11943 , TFLOPS: 70.49043759020401, Tokens per sec: 174683.06951125764, Loss: 3.4608895778656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11944 , TFLOPS: 68.27304362502812, Tokens per sec: 169188.1229994418, Loss: 3.4954936504364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11945 , TFLOPS: 67.82750696999693, Tokens per sec: 168084.03408836032, Loss: 3.4956562519073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11946 , TFLOPS: 69.92476543937457, Tokens per sec: 173281.2716075707, Loss: 3.513571262359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11947 , TFLOPS: 68.65306750820116, Tokens per sec: 170129.86404503122, Loss: 3.5090954303741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11948 , TFLOPS: 67.78977615994833, Tokens per sec: 167990.532984668, Loss: 3.484426498413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11949 , TFLOPS: 68.69972413227663, Tokens per sec: 170245.48429914165, Loss: 3.4928932189941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11950 , TFLOPS: 68.51279887470965, Tokens per sec: 169782.26291937366, Loss: 3.4913671016693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11951 , TFLOPS: 68.78927475115971, Tokens per sec: 170467.40059754896, Loss: 3.5283594131469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11952 , TFLOPS: 68.68140829361927, Tokens per sec: 170200.09563329254, Loss: 3.5003883838653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11953 , TFLOPS: 69.53734437630165, Tokens per sec: 172321.19953532275, Loss: 3.4967617988586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11954 , TFLOPS: 70.4595865481128, Tokens per sec: 174606.61723043275, Loss: 3.470637321472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11955 , TFLOPS: 68.4670446082107, Tokens per sec: 169668.87880674074, Loss: 3.492622137069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11956 , TFLOPS: 68.10795467548321, Tokens per sec: 168779.01439642042, Loss: 3.4895217418670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11957 , TFLOPS: 68.83698481031035, Tokens per sec: 170585.63137982113, Loss: 3.495819091796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11958 , TFLOPS: 68.43817691180385, Tokens per sec: 169597.34147500532, Loss: 3.505518674850464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11959 , TFLOPS: 69.67862355848055, Tokens per sec: 172671.30491194874, Loss: 3.5011754035949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11960 , TFLOPS: 67.77415394570554, Tokens per sec: 167951.8194168454, Loss: 3.5278491973876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11961 , TFLOPS: 68.10853689637237, Tokens per sec: 168780.4572039207, Loss: 3.491051435470581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11962 , TFLOPS: 68.80338841513567, Tokens per sec: 170502.3758116298, Loss: 3.4979372024536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11963 , TFLOPS: 69.53278129975011, Tokens per sec: 172309.89172896367, Loss: 3.5063929557800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11964 , TFLOPS: 68.73801856820175, Tokens per sec: 170340.38212984448, Loss: 3.486541271209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11965 , TFLOPS: 68.95635591095206, Tokens per sec: 170881.44611702356, Loss: 3.4827754497528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11966 , TFLOPS: 69.1590584256039, Tokens per sec: 171383.76527785618, Loss: 3.478459596633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11967 , TFLOPS: 67.94187030694079, Tokens per sec: 168367.43903546873, Loss: 3.494558811187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11968 , TFLOPS: 68.79846241268913, Tokens per sec: 170490.16863492335, Loss: 3.4997425079345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11969 , TFLOPS: 68.79741404558008, Tokens per sec: 170487.57066573994, Loss: 3.519110918045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11970 , TFLOPS: 68.65769539882069, Tokens per sec: 170141.33246779066, Loss: 3.5224337577819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11971 , TFLOPS: 68.67560625161633, Tokens per sec: 170185.71753406152, Loss: 3.4883055686950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11972 , TFLOPS: 69.0100929990734, Tokens per sec: 171014.61254101715, Loss: 3.4963457584381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11973 , TFLOPS: 70.54489979147743, Tokens per sec: 174818.03284551992, Loss: 3.485602617263794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11974 , TFLOPS: 69.78009344180634, Tokens per sec: 172922.75846066047, Loss: 3.4964730739593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11975 , TFLOPS: 67.87153936509495, Tokens per sec: 168193.1512139819, Loss: 3.504366159439087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11976 , TFLOPS: 69.68769128892015, Tokens per sec: 172693.77574687105, Loss: 3.496983051300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11977 , TFLOPS: 68.67865595279513, Tokens per sec: 170193.2750295349, Loss: 3.4951791763305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11978 , TFLOPS: 67.73033843736438, Tokens per sec: 167843.23976049933, Loss: 3.508978843688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11979 , TFLOPS: 69.49347590846922, Tokens per sec: 172212.4886395233, Loss: 3.511399984359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11980 , TFLOPS: 68.83971860875765, Tokens per sec: 170592.40603352632, Loss: 3.5114123821258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11981 , TFLOPS: 68.86651122520571, Tokens per sec: 170658.80108853496, Loss: 3.487910509109497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11982 , TFLOPS: 68.85014146251608, Tokens per sec: 170618.23501331182, Loss: 3.5125765800476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11983 , TFLOPS: 69.98039104745844, Tokens per sec: 173419.11799206914, Loss: 3.5000789165496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11984 , TFLOPS: 68.0363270973597, Tokens per sec: 168601.51336740126, Loss: 3.500979423522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11985 , TFLOPS: 67.6667284283549, Tokens per sec: 167685.60715095288, Loss: 3.4931979179382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11986 , TFLOPS: 68.73934579123598, Tokens per sec: 170343.67113472897, Loss: 3.5143940448760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11987 , TFLOPS: 68.78982831522809, Tokens per sec: 170468.7723902905, Loss: 3.4848878383636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11988 , TFLOPS: 68.75177983154542, Tokens per sec: 170374.48405633838, Loss: 3.464674234390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11989 , TFLOPS: 67.97667179773522, Tokens per sec: 168453.68096335803, Loss: 3.497314929962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11990 , TFLOPS: 69.61495545926725, Tokens per sec: 172513.52834847808, Loss: 3.4931368827819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11991 , TFLOPS: 68.95992078672936, Tokens per sec: 170890.28027190318, Loss: 3.5006680488586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11992 , TFLOPS: 68.7982022377956, Tokens per sec: 170489.52389287655, Loss: 3.4807074069976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11993 , TFLOPS: 69.55574516234013, Tokens per sec: 172366.79871014002, Loss: 3.514248847961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11994 , TFLOPS: 69.66625244073538, Tokens per sec: 172640.6479193851, Loss: 3.499781370162964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11995 , TFLOPS: 68.63586797435929, Tokens per sec: 170087.24170548597, Loss: 3.494821548461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11996 , TFLOPS: 69.72142775817458, Tokens per sec: 172777.37843406244, Loss: 3.501185655593872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11997 , TFLOPS: 67.39443382957437, Tokens per sec: 167010.83113944845, Loss: 3.5025813579559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11998 , TFLOPS: 67.81239320846093, Tokens per sec: 168046.5805223558, Loss: 3.4986891746520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11999 , TFLOPS: 68.87297684323904, Tokens per sec: 170674.82360226815, Loss: 3.520354986190796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12000 , TFLOPS: 67.91592233350366, Tokens per sec: 168303.1370988863, Loss: 3.4861130714416504 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/optimizer.pt +[rank0]:[2024-08-25 22:09:39,636] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.009381625000969507, 'preprocessing_with_comm': 0.00281269199331291, 'state_converting': 1.1037734189885668, : 1.117716824010131}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012000, took 5.65s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010000 + eval ppl=27.125335693359375, eval loss=3.3004682064056396 +------------------------------------------------------------------ +iteration: 12001 , TFLOPS: 66.00268568340697, Tokens per sec: 163561.92591953094, Loss: 3.4702303409576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12002 , TFLOPS: 67.93347194795344, Tokens per sec: 168346.62697674247, Loss: 3.519050359725952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12003 , TFLOPS: 67.45921353380426, Tokens per sec: 167171.36238260363, Loss: 3.511199474334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12004 , TFLOPS: 68.13802915100459, Tokens per sec: 168853.54225973933, Loss: 3.5061583518981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12005 , TFLOPS: 67.4691551884346, Tokens per sec: 167195.99889794132, Loss: 3.4983644485473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12006 , TFLOPS: 69.55293964301474, Tokens per sec: 172359.8463241982, Loss: 3.4981064796447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12007 , TFLOPS: 67.72786551985313, Tokens per sec: 167837.1115984921, Loss: 3.4904375076293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12008 , TFLOPS: 68.59283975518052, Tokens per sec: 169980.61303841136, Loss: 3.4803857803344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12009 , TFLOPS: 68.72128240065467, Tokens per sec: 170298.90806302178, Loss: 3.5128703117370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12010 , TFLOPS: 68.6180457053912, Tokens per sec: 170043.07615969804, Loss: 3.5022783279418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12011 , TFLOPS: 69.47932879196124, Tokens per sec: 172177.430526383, Loss: 3.4639952182769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12012 , TFLOPS: 68.12693553167229, Tokens per sec: 168826.05104897087, Loss: 3.4773406982421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12013 , TFLOPS: 68.57182634274658, Tokens per sec: 169928.53948758155, Loss: 3.5053162574768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12014 , TFLOPS: 69.62316503597721, Tokens per sec: 172533.8725839249, Loss: 3.478536367416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12015 , TFLOPS: 67.67633808298403, Tokens per sec: 167709.42093371472, Loss: 3.492865562438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12016 , TFLOPS: 69.62977814461395, Tokens per sec: 172550.2605956208, Loss: 3.4785616397857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12017 , TFLOPS: 69.53958699633941, Tokens per sec: 172326.7569948213, Loss: 3.5032196044921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12018 , TFLOPS: 67.63652298134282, Tokens per sec: 167610.75472585228, Loss: 3.492661952972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12019 , TFLOPS: 68.77722851403321, Tokens per sec: 170437.54869494692, Loss: 3.502053737640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12020 , TFLOPS: 69.59365435997533, Tokens per sec: 172460.7418779229, Loss: 3.481412887573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12021 , TFLOPS: 69.02216195098488, Tokens per sec: 171044.52073335863, Loss: 3.4960219860076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12022 , TFLOPS: 67.61795058651452, Tokens per sec: 167564.73028554957, Loss: 3.5032756328582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12023 , TFLOPS: 69.66635100562473, Tokens per sec: 172640.89217403874, Loss: 3.4930083751678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12024 , TFLOPS: 69.59808079980475, Tokens per sec: 172471.7110834328, Loss: 3.4714012145996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12025 , TFLOPS: 68.67196964742683, Tokens per sec: 170176.70562827494, Loss: 3.493513584136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12026 , TFLOPS: 68.03939459030232, Tokens per sec: 168609.1149528253, Loss: 3.5030298233032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12027 , TFLOPS: 68.77147614484836, Tokens per sec: 170423.29369043108, Loss: 3.502244472503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12028 , TFLOPS: 68.63564563348721, Tokens per sec: 170086.6907203121, Loss: 3.495023488998413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12029 , TFLOPS: 69.73645063738958, Tokens per sec: 172814.60678366394, Loss: 3.499495267868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12030 , TFLOPS: 67.59108865770663, Tokens per sec: 167498.1634668779, Loss: 3.519773006439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12031 , TFLOPS: 70.59038891928624, Tokens per sec: 174930.75991527125, Loss: 3.488431692123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12032 , TFLOPS: 67.67137214808629, Tokens per sec: 167697.11479999524, Loss: 3.507997512817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12033 , TFLOPS: 68.73647164125006, Tokens per sec: 170336.5486744501, Loss: 3.497243881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12034 , TFLOPS: 68.35595585945914, Tokens per sec: 169393.5886498985, Loss: 3.484816074371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12035 , TFLOPS: 68.00481410532127, Tokens per sec: 168523.42070168737, Loss: 3.472939968109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12036 , TFLOPS: 69.59927218738106, Tokens per sec: 172474.66347308908, Loss: 3.497046947479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12037 , TFLOPS: 69.44560130980486, Tokens per sec: 172093.8501102106, Loss: 3.5139575004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12038 , TFLOPS: 68.71431664078946, Tokens per sec: 170281.64614273398, Loss: 3.5086023807525635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12039 , TFLOPS: 68.85898531858268, Tokens per sec: 170640.15106287602, Loss: 3.493537187576294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12040 , TFLOPS: 68.66897360505335, Tokens per sec: 170169.28110523216, Loss: 3.5258257389068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12041 , TFLOPS: 69.66355095719398, Tokens per sec: 172633.9533455765, Loss: 3.482290506362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12042 , TFLOPS: 68.11436287628868, Tokens per sec: 168794.89462393793, Loss: 3.499405860900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12043 , TFLOPS: 69.70592270699527, Tokens per sec: 172738.95520921767, Loss: 3.5123870372772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12044 , TFLOPS: 68.13510125355047, Tokens per sec: 168846.28660731352, Loss: 3.526195526123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12045 , TFLOPS: 68.86066999560121, Tokens per sec: 170644.32587811045, Loss: 3.4776012897491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12046 , TFLOPS: 68.75530736203002, Tokens per sec: 170383.22566546878, Loss: 3.5093774795532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12047 , TFLOPS: 68.72788898763896, Tokens per sec: 170315.27991334445, Loss: 3.4729230403900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12048 , TFLOPS: 68.50059569797578, Tokens per sec: 169752.02210313626, Loss: 3.487973690032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12049 , TFLOPS: 69.06298932542974, Tokens per sec: 171145.69546473958, Loss: 3.495217800140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12050 , TFLOPS: 68.7846561569355, Tokens per sec: 170455.95521228117, Loss: 3.490938425064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12051 , TFLOPS: 70.05922721441634, Tokens per sec: 173614.4826983115, Loss: 3.489522933959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12052 , TFLOPS: 67.70054030688856, Tokens per sec: 167769.39671064517, Loss: 3.4855542182922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12053 , TFLOPS: 69.67021486744396, Tokens per sec: 172650.46724926628, Loss: 3.474184513092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12054 , TFLOPS: 68.63055705905025, Tokens per sec: 170074.08067230458, Loss: 3.4884755611419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12055 , TFLOPS: 67.79822703533475, Tokens per sec: 168011.47518481634, Loss: 3.4750804901123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12056 , TFLOPS: 69.5914254577085, Tokens per sec: 172455.21841257182, Loss: 3.504897356033325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12057 , TFLOPS: 69.56762681909677, Tokens per sec: 172396.24276445553, Loss: 3.47393798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12058 , TFLOPS: 69.67014088988978, Tokens per sec: 172650.28392473774, Loss: 3.473994493484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12059 , TFLOPS: 67.33041217097679, Tokens per sec: 166852.17841687574, Loss: 3.5000038146972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12060 , TFLOPS: 69.55149673846043, Tokens per sec: 172356.27064776362, Loss: 3.5040812492370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12061 , TFLOPS: 69.61720799474611, Tokens per sec: 172519.11037953195, Loss: 3.474884271621704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12062 , TFLOPS: 67.9731189600918, Tokens per sec: 168444.87664030027, Loss: 3.515700578689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12063 , TFLOPS: 68.56984828703031, Tokens per sec: 169923.63764761586, Loss: 3.500915765762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12064 , TFLOPS: 69.57830814532376, Tokens per sec: 172422.71226749124, Loss: 3.4574828147888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12065 , TFLOPS: 68.62679540186487, Tokens per sec: 170064.75887141845, Loss: 3.47902250289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12066 , TFLOPS: 68.9418752339902, Tokens per sec: 170845.5614043338, Loss: 3.5232248306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12067 , TFLOPS: 66.1658284418762, Tokens per sec: 163966.21164667612, Loss: 3.5140814781188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12068 , TFLOPS: 69.6377423142921, Tokens per sec: 172569.99668540896, Loss: 3.4740846157073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12069 , TFLOPS: 68.78806593268214, Tokens per sec: 170464.40501219922, Loss: 3.483025550842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12070 , TFLOPS: 67.10035271143522, Tokens per sec: 166282.06573299016, Loss: 3.4819462299346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12071 , TFLOPS: 69.09804256227198, Tokens per sec: 171232.56124706747, Loss: 3.487354040145874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12072 , TFLOPS: 67.88117866766522, Tokens per sec: 168217.03846760793, Loss: 3.503793954849243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12073 , TFLOPS: 69.54287493225115, Tokens per sec: 172334.90486220666, Loss: 3.513683557510376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12074 , TFLOPS: 67.86520074562245, Tokens per sec: 168177.4434166725, Loss: 3.4929797649383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12075 , TFLOPS: 68.75554423832696, Tokens per sec: 170383.81267102668, Loss: 3.4651072025299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12076 , TFLOPS: 69.11390514529623, Tokens per sec: 171271.87047520932, Loss: 3.4811975955963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12077 , TFLOPS: 68.49189016506088, Tokens per sec: 169730.4488335795, Loss: 3.473843574523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12078 , TFLOPS: 69.70825701820439, Tokens per sec: 172744.7398895395, Loss: 3.4761908054351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12079 , TFLOPS: 68.78376873316218, Tokens per sec: 170453.7560784131, Loss: 3.4795613288879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12080 , TFLOPS: 69.05654703617692, Tokens per sec: 171129.73076229435, Loss: 3.503708839416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12081 , TFLOPS: 67.9902756562025, Tokens per sec: 168487.39282322867, Loss: 3.496569871902466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12082 , TFLOPS: 67.92782469825008, Tokens per sec: 168332.63246988165, Loss: 3.4840281009674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12083 , TFLOPS: 69.7054355942349, Tokens per sec: 172737.74809013758, Loss: 3.498812675476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12084 , TFLOPS: 69.5233217783476, Tokens per sec: 172286.4499928759, Loss: 3.497490882873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12085 , TFLOPS: 67.92837432417149, Tokens per sec: 168333.99450346117, Loss: 3.4909844398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12086 , TFLOPS: 68.93332727544143, Tokens per sec: 170824.3786214146, Loss: 3.507803440093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12087 , TFLOPS: 69.53815603494533, Tokens per sec: 172323.21091485338, Loss: 3.511497974395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12088 , TFLOPS: 69.97878399760002, Tokens per sec: 173415.1355454888, Loss: 3.483847141265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12089 , TFLOPS: 67.90295994491245, Tokens per sec: 168271.01487203233, Loss: 3.502032995223999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12090 , TFLOPS: 69.56677518449443, Tokens per sec: 172394.1323200956, Loss: 3.4862709045410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12091 , TFLOPS: 68.26917937797936, Tokens per sec: 169178.54696957298, Loss: 3.516589641571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12092 , TFLOPS: 67.84325793357812, Tokens per sec: 168123.06671122633, Loss: 3.5176098346710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12093 , TFLOPS: 69.11110403071187, Tokens per sec: 171264.92900470103, Loss: 3.4774014949798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12094 , TFLOPS: 68.73923759644566, Tokens per sec: 170343.40301611993, Loss: 3.493744373321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12095 , TFLOPS: 68.66310557590738, Tokens per sec: 170154.73948259169, Loss: 3.4883313179016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12096 , TFLOPS: 68.61177651195466, Tokens per sec: 170027.54040775367, Loss: 3.5050368309020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12097 , TFLOPS: 68.83662094368263, Tokens per sec: 170584.72967823443, Loss: 3.4996566772460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12098 , TFLOPS: 70.53408072645112, Tokens per sec: 174791.2220105663, Loss: 3.5024449825286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12099 , TFLOPS: 68.2856266426525, Tokens per sec: 169219.3051032488, Loss: 3.509864568710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12100 , TFLOPS: 68.60925414907913, Tokens per sec: 170021.2897146869, Loss: 3.477586269378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12101 , TFLOPS: 69.72353141335378, Tokens per sec: 172782.59152333322, Loss: 3.492649555206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12102 , TFLOPS: 67.86675635224996, Tokens per sec: 168181.2983812597, Loss: 3.4920949935913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12103 , TFLOPS: 69.62210326808845, Tokens per sec: 172531.24140613317, Loss: 3.4753730297088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12104 , TFLOPS: 68.53847440561762, Tokens per sec: 169845.88971335685, Loss: 3.5063352584838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12105 , TFLOPS: 68.7888439542474, Tokens per sec: 170466.33303534403, Loss: 3.513969659805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12106 , TFLOPS: 68.86624989039392, Tokens per sec: 170658.15347208292, Loss: 3.527524948120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12107 , TFLOPS: 68.72824278370203, Tokens per sec: 170316.1566589626, Loss: 3.4953184127807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12108 , TFLOPS: 69.61450067476781, Tokens per sec: 172512.40134238984, Loss: 3.520244598388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12109 , TFLOPS: 68.0787696846753, Tokens per sec: 168706.69077420625, Loss: 3.506615400314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12110 , TFLOPS: 69.62672880423328, Tokens per sec: 172542.70399424495, Loss: 3.505572557449341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12111 , TFLOPS: 67.80495472085802, Tokens per sec: 168028.14713065873, Loss: 3.4937829971313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12112 , TFLOPS: 68.70523284958605, Tokens per sec: 170259.13550746973, Loss: 3.5011672973632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12113 , TFLOPS: 68.79906599366774, Tokens per sec: 170491.66437507825, Loss: 3.521721124649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12114 , TFLOPS: 67.666786061448, Tokens per sec: 167685.74997210637, Loss: 3.5045950412750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12115 , TFLOPS: 69.66676415163982, Tokens per sec: 172641.91599537595, Loss: 3.500089168548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12116 , TFLOPS: 69.1424130735573, Tokens per sec: 171342.5162618478, Loss: 3.508662223815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12117 , TFLOPS: 68.71952270812932, Tokens per sec: 170294.54735109778, Loss: 3.49963641166687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12118 , TFLOPS: 69.6384845423259, Tokens per sec: 172571.8360082399, Loss: 3.4596331119537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12119 , TFLOPS: 67.88801973167365, Tokens per sec: 168233.99137782623, Loss: 3.480069875717163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12120 , TFLOPS: 69.65792345457949, Tokens per sec: 172620.00777417724, Loss: 3.507336378097534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12121 , TFLOPS: 69.00849411706018, Tokens per sec: 171010.65033525703, Loss: 3.484531879425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12122 , TFLOPS: 67.13296262217183, Tokens per sec: 166362.87668407368, Loss: 3.491515874862671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12123 , TFLOPS: 69.67185009785317, Tokens per sec: 172654.51953035584, Loss: 3.510776996612549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12124 , TFLOPS: 69.52201775177366, Tokens per sec: 172283.21847137532, Loss: 3.5164072513580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12125 , TFLOPS: 69.70542388400413, Tokens per sec: 172737.71907089578, Loss: 3.5067481994628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12126 , TFLOPS: 67.11011060503809, Tokens per sec: 166306.24686825875, Loss: 3.491039276123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12127 , TFLOPS: 69.62910468137274, Tokens per sec: 172548.591679521, Loss: 3.49892258644104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12128 , TFLOPS: 69.71049243795271, Tokens per sec: 172750.27950592764, Loss: 3.4778385162353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12129 , TFLOPS: 67.75360606432882, Tokens per sec: 167900.89950325794, Loss: 3.517406463623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12130 , TFLOPS: 69.73640024535604, Tokens per sec: 172814.48190665367, Loss: 3.5102639198303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12131 , TFLOPS: 68.75921842738626, Tokens per sec: 170392.91771626094, Loss: 3.490619659423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12132 , TFLOPS: 68.74833099816128, Tokens per sec: 170365.9374672921, Loss: 3.508483409881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12133 , TFLOPS: 68.04948865178207, Tokens per sec: 168634.12914912478, Loss: 3.4820504188537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12134 , TFLOPS: 68.74933471505794, Tokens per sec: 170368.4247883306, Loss: 3.4919276237487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12135 , TFLOPS: 69.72147142600944, Tokens per sec: 172777.4866477676, Loss: 3.505931854248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12136 , TFLOPS: 68.7239163815344, Tokens per sec: 170305.43535779862, Loss: 3.5112662315368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12137 , TFLOPS: 68.62801347736449, Tokens per sec: 170067.77739668862, Loss: 3.499340534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12138 , TFLOPS: 69.8515426928351, Tokens per sec: 173099.8175754945, Loss: 3.5192620754241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12139 , TFLOPS: 68.01666248831621, Tokens per sec: 168552.78229995596, Loss: 3.5049610137939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12140 , TFLOPS: 69.72336813641415, Tokens per sec: 172782.1869050904, Loss: 3.5101890563964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12141 , TFLOPS: 68.73491112178125, Tokens per sec: 170332.68153530097, Loss: 3.47579288482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12142 , TFLOPS: 68.76078221191152, Tokens per sec: 170396.79294657963, Loss: 3.4986696243286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12143 , TFLOPS: 69.04465917973972, Tokens per sec: 171100.27134448747, Loss: 3.489396333694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12144 , TFLOPS: 68.64467740711531, Tokens per sec: 170109.07245029454, Loss: 3.4965715408325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12145 , TFLOPS: 69.72580230329864, Tokens per sec: 172788.21903877574, Loss: 3.5052826404571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12146 , TFLOPS: 68.80297843558158, Tokens per sec: 170501.3598371318, Loss: 3.47883677482605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12147 , TFLOPS: 69.0289541217902, Tokens per sec: 171061.35248083362, Loss: 3.5157790184020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12148 , TFLOPS: 68.75308858652701, Tokens per sec: 170377.72729535404, Loss: 3.4989888668060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12149 , TFLOPS: 67.91387905163641, Tokens per sec: 168298.0736213332, Loss: 3.514793634414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12150 , TFLOPS: 69.64629670210063, Tokens per sec: 172591.19540074212, Loss: 3.4972848892211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12151 , TFLOPS: 67.96321277142214, Tokens per sec: 168420.32801352005, Loss: 3.528970956802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12152 , TFLOPS: 68.73572299434093, Tokens per sec: 170334.693444939, Loss: 3.4799962043762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12153 , TFLOPS: 68.80433106814877, Tokens per sec: 170504.7118096381, Loss: 3.4930200576782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12154 , TFLOPS: 69.5750960557231, Tokens per sec: 172414.7523556199, Loss: 3.520752429962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12155 , TFLOPS: 68.9351143067478, Tokens per sec: 170828.80708184955, Loss: 3.4829647541046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12156 , TFLOPS: 68.75219545959987, Tokens per sec: 170375.51402844253, Loss: 3.486020088195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12157 , TFLOPS: 69.65235689960431, Tokens per sec: 172606.2132377421, Loss: 3.495619297027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12158 , TFLOPS: 69.60287381742266, Tokens per sec: 172483.5887090845, Loss: 3.5144193172454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12159 , TFLOPS: 68.70274447819398, Tokens per sec: 170252.96904904343, Loss: 3.514834403991699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12160 , TFLOPS: 69.15845470003214, Tokens per sec: 171382.2691793835, Loss: 3.4895951747894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12161 , TFLOPS: 68.81407967632204, Tokens per sec: 170528.86993458928, Loss: 3.5056583881378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12162 , TFLOPS: 68.78204496025744, Tokens per sec: 170449.48437926592, Loss: 3.494558095932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12163 , TFLOPS: 68.62162144718235, Tokens per sec: 170051.937241757, Loss: 3.5030813217163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12164 , TFLOPS: 69.06674107402942, Tokens per sec: 171154.99271105882, Loss: 3.4968583583831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12165 , TFLOPS: 70.56900003339733, Tokens per sec: 174877.75589985822, Loss: 3.494776487350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12166 , TFLOPS: 67.99719999125949, Tokens per sec: 168504.5520882784, Loss: 3.5028319358825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12167 , TFLOPS: 68.77134246971134, Tokens per sec: 170422.962428716, Loss: 3.5051515102386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12168 , TFLOPS: 69.52141288523627, Tokens per sec: 172281.71954546415, Loss: 3.5034000873565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12169 , TFLOPS: 67.87179763367088, Tokens per sec: 168193.79123196373, Loss: 3.5245168209075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12170 , TFLOPS: 69.66152979170103, Tokens per sec: 172628.94467483528, Loss: 3.5051770210266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12171 , TFLOPS: 68.6726123681956, Tokens per sec: 170178.2983611415, Loss: 3.5117506980895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12172 , TFLOPS: 68.67725493872793, Tokens per sec: 170189.80316234197, Loss: 3.482961416244507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12173 , TFLOPS: 68.87875174348179, Tokens per sec: 170689.13444122748, Loss: 3.473396062850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12174 , TFLOPS: 68.54390051583304, Tokens per sec: 169859.33621220605, Loss: 3.4962363243103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12175 , TFLOPS: 69.76287888358051, Tokens per sec: 172880.09888903672, Loss: 3.512787342071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12176 , TFLOPS: 68.12720750367605, Tokens per sec: 168826.72502555637, Loss: 3.4975826740264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12177 , TFLOPS: 69.75102812699332, Tokens per sec: 172850.73140874508, Loss: 3.508542776107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12178 , TFLOPS: 68.79367835907124, Tokens per sec: 170478.3132230512, Loss: 3.4993276596069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12179 , TFLOPS: 68.82549603858283, Tokens per sec: 170557.16093788765, Loss: 3.4853856563568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12180 , TFLOPS: 68.86721615116264, Tokens per sec: 170660.54797271022, Loss: 3.486405611038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12181 , TFLOPS: 67.29343669181729, Tokens per sec: 166760.54910633326, Loss: 3.4853129386901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12182 , TFLOPS: 69.73725108697097, Tokens per sec: 172816.59038590343, Loss: 3.506591320037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12183 , TFLOPS: 69.0707056557036, Tokens per sec: 171164.81738118216, Loss: 3.4949631690979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12184 , TFLOPS: 68.77664778640793, Tokens per sec: 170436.10958791905, Loss: 3.5154616832733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12185 , TFLOPS: 69.63858095835381, Tokens per sec: 172572.07493777815, Loss: 3.4971694946289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12186 , TFLOPS: 68.05888294117783, Tokens per sec: 168657.4092331139, Loss: 3.510833740234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12187 , TFLOPS: 69.64740534861785, Tokens per sec: 172593.94274893927, Loss: 3.489968776702881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12188 , TFLOPS: 68.11891483662498, Tokens per sec: 168806.17488309112, Loss: 3.484127998352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12189 , TFLOPS: 68.79308869187255, Tokens per sec: 170476.85196277563, Loss: 3.5085301399230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12190 , TFLOPS: 69.09092736631717, Tokens per sec: 171214.9290076879, Loss: 3.482713222503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12191 , TFLOPS: 68.7102175162756, Tokens per sec: 170271.4880606328, Loss: 3.4836947917938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12192 , TFLOPS: 68.615495821131, Tokens per sec: 170036.75726560835, Loss: 3.5062756538391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12193 , TFLOPS: 69.10298744164724, Tokens per sec: 171244.8152028828, Loss: 3.488567590713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12194 , TFLOPS: 69.00987384016726, Tokens per sec: 171014.0694411054, Loss: 3.50500750541687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12195 , TFLOPS: 69.77062960795398, Tokens per sec: 172899.30603784698, Loss: 3.4830472469329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12196 , TFLOPS: 67.00615064700739, Tokens per sec: 166048.6226401239, Loss: 3.5077693462371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12197 , TFLOPS: 69.67648097211541, Tokens per sec: 172665.9953469106, Loss: 3.4878671169281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12198 , TFLOPS: 69.56957375709844, Tokens per sec: 172401.06749129656, Loss: 3.5019357204437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12199 , TFLOPS: 68.7697172483273, Tokens per sec: 170418.93495109346, Loss: 3.503248691558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12200 , TFLOPS: 67.90453862981575, Tokens per sec: 168274.92702712916, Loss: 3.508631944656372 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/optimizer.pt +[rank0]:[2024-08-25 22:35:26,203] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006433542992454022, 'preprocessing_with_comm': 0.0012442339939298108, 'state_converting': 1.0060899579984834, : 1.0151470879936824}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012200, took 4.33s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010200 + eval ppl=23.92925453186035, eval loss=3.1751017570495605 +------------------------------------------------------------------ +iteration: 12201 , TFLOPS: 64.3891355131836, Tokens per sec: 159563.37085049276, Loss: 3.513631582260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12202 , TFLOPS: 69.65909853393265, Tokens per sec: 172622.91975026004, Loss: 3.5092997550964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12203 , TFLOPS: 66.96388049995699, Tokens per sec: 165943.87255929192, Loss: 3.491610288619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12204 , TFLOPS: 69.59760303374001, Tokens per sec: 172470.5271264941, Loss: 3.4745097160339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12205 , TFLOPS: 69.7042809578376, Tokens per sec: 172734.88677395138, Loss: 3.5029146671295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12206 , TFLOPS: 67.78305253606057, Tokens per sec: 167973.8711040058, Loss: 3.4893875122070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12207 , TFLOPS: 68.85898497817344, Tokens per sec: 170640.15021930443, Loss: 3.517235040664673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12208 , TFLOPS: 69.09637002152205, Tokens per sec: 171228.41650684355, Loss: 3.4981415271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12209 , TFLOPS: 68.82799960026435, Tokens per sec: 170563.36503952453, Loss: 3.500664710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12210 , TFLOPS: 68.77148154051365, Tokens per sec: 170423.30706148405, Loss: 3.4923744201660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12211 , TFLOPS: 67.10702168123086, Tokens per sec: 166298.59217479138, Loss: 3.500438690185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12212 , TFLOPS: 69.42390550138504, Tokens per sec: 172040.08550695571, Loss: 3.495959997177124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12213 , TFLOPS: 68.81722192524171, Tokens per sec: 170536.65677355955, Loss: 3.4987409114837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12214 , TFLOPS: 69.55026671172492, Tokens per sec: 172353.22250601437, Loss: 3.508962392807007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12215 , TFLOPS: 68.06631803292775, Tokens per sec: 168675.83420951315, Loss: 3.4957807064056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12216 , TFLOPS: 68.73671044921903, Tokens per sec: 170337.14046690406, Loss: 3.4957635402679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12217 , TFLOPS: 69.40142908510596, Tokens per sec: 171984.3864714345, Loss: 3.498560905456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12218 , TFLOPS: 68.74510395314574, Tokens per sec: 170357.94049425528, Loss: 3.4788198471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12219 , TFLOPS: 67.92287348931413, Tokens per sec: 168320.36282871765, Loss: 3.508497953414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12220 , TFLOPS: 69.74690014487516, Tokens per sec: 172840.50181432147, Loss: 3.5119104385375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12221 , TFLOPS: 68.76404481043038, Tokens per sec: 170404.87802511425, Loss: 3.514927864074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12222 , TFLOPS: 68.67922685715264, Tokens per sec: 170194.68979342317, Loss: 3.5149433612823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12223 , TFLOPS: 69.05792070961954, Tokens per sec: 171133.13487640777, Loss: 3.503753662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12224 , TFLOPS: 69.59633820244913, Tokens per sec: 172467.39273522265, Loss: 3.51419997215271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12225 , TFLOPS: 69.77567632818575, Tokens per sec: 172911.81236652343, Loss: 3.517902135848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12226 , TFLOPS: 67.63562105893314, Tokens per sec: 167608.51966276614, Loss: 3.4894628524780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12227 , TFLOPS: 69.65388127087773, Tokens per sec: 172609.99079768098, Loss: 3.495373487472534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12228 , TFLOPS: 68.5952509740239, Tokens per sec: 169986.58830432294, Loss: 3.4882853031158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12229 , TFLOPS: 68.79406704226253, Tokens per sec: 170479.27642281615, Loss: 3.4931395053863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12230 , TFLOPS: 67.76689853928454, Tokens per sec: 167933.83972048518, Loss: 3.4846692085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12231 , TFLOPS: 68.86174597902786, Tokens per sec: 170646.99228357105, Loss: 3.500351905822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12232 , TFLOPS: 69.51184505842954, Tokens per sec: 172258.0094166539, Loss: 3.4955549240112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12233 , TFLOPS: 67.7767268060275, Tokens per sec: 167958.19524814718, Loss: 3.4872782230377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12234 , TFLOPS: 69.51585123100337, Tokens per sec: 172267.9371536082, Loss: 3.5064806938171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12235 , TFLOPS: 69.66006160927698, Tokens per sec: 172625.3063570548, Loss: 3.501549005508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12236 , TFLOPS: 67.62938621592765, Tokens per sec: 167593.0690349739, Loss: 3.515852451324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12237 , TFLOPS: 68.54934131332378, Tokens per sec: 169872.81910774144, Loss: 3.482299327850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12238 , TFLOPS: 69.73313838365567, Tokens per sec: 172806.39865403643, Loss: 3.5008544921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12239 , TFLOPS: 68.66762191619287, Tokens per sec: 170165.93147133506, Loss: 3.51987361907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12240 , TFLOPS: 68.2297161603258, Tokens per sec: 169080.75276899515, Loss: 3.475076198577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12241 , TFLOPS: 68.72060243789682, Tokens per sec: 170297.2230404037, Loss: 3.490635395050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12242 , TFLOPS: 69.74280752596907, Tokens per sec: 172830.359853833, Loss: 3.4874730110168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12243 , TFLOPS: 68.78835319206813, Tokens per sec: 170465.1168726019, Loss: 3.4906558990478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12244 , TFLOPS: 68.56145206312573, Tokens per sec: 169902.83087985986, Loss: 3.5125789642333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12245 , TFLOPS: 69.73545275809019, Tokens per sec: 172812.13392883458, Loss: 3.4788265228271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12246 , TFLOPS: 67.79326543729316, Tokens per sec: 167999.17979830413, Loss: 3.514857769012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12247 , TFLOPS: 69.68240685407238, Tokens per sec: 172680.68033518974, Loss: 3.500556468963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12248 , TFLOPS: 67.67688745320876, Tokens per sec: 167710.78233364975, Loss: 3.4920315742492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12249 , TFLOPS: 68.78703705258118, Tokens per sec: 170461.85533396853, Loss: 3.5092580318450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12250 , TFLOPS: 68.93453423074529, Tokens per sec: 170827.36958961468, Loss: 3.479923725128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12251 , TFLOPS: 69.57792572856255, Tokens per sec: 172421.76459663024, Loss: 3.4913058280944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12252 , TFLOPS: 69.17598208100766, Tokens per sec: 171425.70396024126, Loss: 3.473342180252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12253 , TFLOPS: 67.96664250687424, Tokens per sec: 168428.82727578693, Loss: 3.4817538261413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12254 , TFLOPS: 69.57645688014594, Tokens per sec: 172418.12462844685, Loss: 3.5060577392578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12255 , TFLOPS: 68.59357097606463, Tokens per sec: 169982.4250843128, Loss: 3.4968204498291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12256 , TFLOPS: 68.804274065177, Tokens per sec: 170504.57054999462, Loss: 3.479557991027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12257 , TFLOPS: 69.99642507311513, Tokens per sec: 173458.85207394013, Loss: 3.4594016075134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12258 , TFLOPS: 69.0175869966781, Tokens per sec: 171033.18349261364, Loss: 3.4811599254608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12259 , TFLOPS: 67.89932791092329, Tokens per sec: 168262.01429170556, Loss: 3.4989500045776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12260 , TFLOPS: 67.96504610092025, Tokens per sec: 168424.8712059743, Loss: 3.4934988021850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12261 , TFLOPS: 68.7937242691063, Tokens per sec: 170478.4269931755, Loss: 3.485332727432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12262 , TFLOPS: 69.69053627438907, Tokens per sec: 172700.82593426408, Loss: 3.4768123626708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12263 , TFLOPS: 67.87463933579862, Tokens per sec: 168200.8332828756, Loss: 3.5062403678894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12264 , TFLOPS: 69.56736996262906, Tokens per sec: 172395.60624583345, Loss: 3.5139544010162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12265 , TFLOPS: 69.80926937656007, Tokens per sec: 172995.05964097474, Loss: 3.4851863384246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12266 , TFLOPS: 67.89355185522734, Tokens per sec: 168247.70058940595, Loss: 3.501887559890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12267 , TFLOPS: 68.68376869035183, Tokens per sec: 170205.94495641495, Loss: 3.4942355155944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12268 , TFLOPS: 68.17605502271887, Tokens per sec: 168947.7745587429, Loss: 3.5013132095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12269 , TFLOPS: 68.79157568225911, Tokens per sec: 170473.10255828174, Loss: 3.526355743408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12270 , TFLOPS: 68.77399873780332, Tokens per sec: 170429.54495366005, Loss: 3.5012154579162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12271 , TFLOPS: 69.57428187036844, Tokens per sec: 172412.73471462113, Loss: 3.496001720428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12272 , TFLOPS: 69.82588283131673, Tokens per sec: 173036.2296119849, Loss: 3.4699807167053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12273 , TFLOPS: 68.7128898609758, Tokens per sec: 170278.11042518297, Loss: 3.509938955307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12274 , TFLOPS: 68.84128918106556, Tokens per sec: 170596.29808471812, Loss: 3.5058341026306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12275 , TFLOPS: 68.94718008470512, Tokens per sec: 170858.70740878294, Loss: 3.509359359741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12276 , TFLOPS: 68.65982923740346, Tokens per sec: 170146.62035486044, Loss: 3.494252920150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12277 , TFLOPS: 68.81278807779363, Tokens per sec: 170525.66921115457, Loss: 3.5008957386016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12278 , TFLOPS: 68.71142823743388, Tokens per sec: 170274.48836103492, Loss: 3.5116305351257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12279 , TFLOPS: 68.85712532308692, Tokens per sec: 170635.54178914212, Loss: 3.504483699798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12280 , TFLOPS: 68.78280836029069, Tokens per sec: 170451.37616864376, Loss: 3.4758999347686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12281 , TFLOPS: 67.76146357174254, Tokens per sec: 167920.37127220503, Loss: 3.517134666442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12282 , TFLOPS: 70.56748800811089, Tokens per sec: 174874.0089346351, Loss: 3.4994547367095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12283 , TFLOPS: 68.04503778264838, Tokens per sec: 168623.0994051078, Loss: 3.5070457458496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12284 , TFLOPS: 69.47937121778996, Tokens per sec: 172177.53566226014, Loss: 3.488497734069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12285 , TFLOPS: 68.85897318336033, Tokens per sec: 170640.12099045824, Loss: 3.5250437259674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12286 , TFLOPS: 67.9286727682191, Tokens per sec: 168334.73408068749, Loss: 3.4943344593048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12287 , TFLOPS: 69.74437041361567, Tokens per sec: 172834.23286158757, Loss: 3.4776217937469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12288 , TFLOPS: 68.8016526077331, Tokens per sec: 170498.0742896713, Loss: 3.5115389823913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12289 , TFLOPS: 68.68300168655513, Tokens per sec: 170204.0442365143, Loss: 3.4895081520080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12290 , TFLOPS: 68.1783325140715, Tokens per sec: 168953.41843320112, Loss: 3.4886646270751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12291 , TFLOPS: 69.54863728388158, Tokens per sec: 172349.18460432228, Loss: 3.480874538421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12292 , TFLOPS: 68.84158502192048, Tokens per sec: 170597.03121094615, Loss: 3.487720489501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12293 , TFLOPS: 68.66277866630121, Tokens per sec: 170153.92936457548, Loss: 3.5164577960968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12294 , TFLOPS: 69.70867665496769, Tokens per sec: 172745.77979566576, Loss: 3.526156425476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12295 , TFLOPS: 68.78776598472923, Tokens per sec: 170463.66170812992, Loss: 3.5143325328826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12296 , TFLOPS: 67.67306253239168, Tokens per sec: 167701.3037585152, Loss: 3.5020859241485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12297 , TFLOPS: 69.6829996856897, Tokens per sec: 172682.1494372432, Loss: 3.506927251815796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12298 , TFLOPS: 68.73301881127644, Tokens per sec: 170327.99218141485, Loss: 3.493619680404663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12299 , TFLOPS: 68.71271119102899, Tokens per sec: 170277.6676613726, Loss: 3.5029172897338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12300 , TFLOPS: 67.86459196116581, Tokens per sec: 168175.9347817258, Loss: 3.5352888107299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12301 , TFLOPS: 69.67140689234834, Tokens per sec: 172653.42121829174, Loss: 3.5177018642425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12302 , TFLOPS: 69.97031049553041, Tokens per sec: 173394.13727392702, Loss: 3.490352153778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12303 , TFLOPS: 67.88651729317692, Tokens per sec: 168230.26816972403, Loss: 3.469031572341919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12304 , TFLOPS: 69.47798026165401, Tokens per sec: 172174.0887197288, Loss: 3.4752697944641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12305 , TFLOPS: 69.03284777588385, Tokens per sec: 171071.00138459908, Loss: 3.4892778396606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12306 , TFLOPS: 68.34387917104561, Tokens per sec: 169363.6612856527, Loss: 3.526709794998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12307 , TFLOPS: 68.90718590555792, Tokens per sec: 170759.59742713367, Loss: 3.476696729660034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12308 , TFLOPS: 68.74654392562641, Tokens per sec: 170361.5089046884, Loss: 3.4751837253570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12309 , TFLOPS: 69.69984283480134, Tokens per sec: 172723.88861616794, Loss: 3.4969186782836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12310 , TFLOPS: 68.00019469717239, Tokens per sec: 168511.97329942463, Loss: 3.504287004470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12311 , TFLOPS: 68.73010239840085, Tokens per sec: 170320.76498903972, Loss: 3.5054450035095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12312 , TFLOPS: 69.74243563714127, Tokens per sec: 172829.4382723508, Loss: 3.4806857109069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12313 , TFLOPS: 68.68284978663468, Tokens per sec: 170203.66781177826, Loss: 3.4999682903289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12314 , TFLOPS: 68.57774250549636, Tokens per sec: 169943.20039059466, Loss: 3.498554229736328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12315 , TFLOPS: 68.71315649872022, Tokens per sec: 170278.77118288618, Loss: 3.5097553730010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12316 , TFLOPS: 68.74587034496751, Tokens per sec: 170359.8396976146, Loss: 3.4988908767700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12317 , TFLOPS: 69.58544692101397, Tokens per sec: 172440.40294005262, Loss: 3.509880542755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12318 , TFLOPS: 68.67629915907642, Tokens per sec: 170187.43463507772, Loss: 3.488616466522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12319 , TFLOPS: 69.18616187711467, Tokens per sec: 171450.93061639153, Loss: 3.4702608585357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12320 , TFLOPS: 68.23300630143719, Tokens per sec: 169088.90610110818, Loss: 3.485750198364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12321 , TFLOPS: 69.50472835591133, Tokens per sec: 172240.3734438443, Loss: 3.5006256103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12322 , TFLOPS: 68.64077838196566, Tokens per sec: 170099.41023646062, Loss: 3.496307373046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12323 , TFLOPS: 68.6337557773374, Tokens per sec: 170082.007448588, Loss: 3.5128517150878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12324 , TFLOPS: 69.42620831679112, Tokens per sec: 172045.79213720802, Loss: 3.513810873031616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12325 , TFLOPS: 68.84669332713311, Tokens per sec: 170609.69015398936, Loss: 3.4772119522094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12326 , TFLOPS: 67.85065535057356, Tokens per sec: 168141.3983254338, Loss: 3.4897751808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12327 , TFLOPS: 68.81147344464237, Tokens per sec: 170522.41140538696, Loss: 3.4822440147399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12328 , TFLOPS: 68.652883558659, Tokens per sec: 170129.408197802, Loss: 3.5041534900665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12329 , TFLOPS: 68.69856606079489, Tokens per sec: 170242.61447043778, Loss: 3.493987560272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12330 , TFLOPS: 67.88089345198561, Tokens per sec: 168216.33167173475, Loss: 3.5020971298217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12331 , TFLOPS: 69.67720926254964, Tokens per sec: 172667.8001308344, Loss: 3.487351417541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12332 , TFLOPS: 69.74954547356208, Tokens per sec: 172847.0572302147, Loss: 3.505125045776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12333 , TFLOPS: 67.70127369008362, Tokens per sec: 167771.2141149914, Loss: 3.506052017211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12334 , TFLOPS: 69.68799555899542, Tokens per sec: 172694.52976163573, Loss: 3.498453140258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12335 , TFLOPS: 68.61911516038441, Tokens per sec: 170045.72638698143, Loss: 3.5014708042144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12336 , TFLOPS: 68.72429337553295, Tokens per sec: 170306.3695904564, Loss: 3.4857258796691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12337 , TFLOPS: 68.64831889191598, Tokens per sec: 170118.09645077327, Loss: 3.502969741821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12338 , TFLOPS: 69.64041761349756, Tokens per sec: 172576.62637154863, Loss: 3.5139541625976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12339 , TFLOPS: 69.7329293864202, Tokens per sec: 172805.88073586402, Loss: 3.488499164581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12340 , TFLOPS: 68.34461120737919, Tokens per sec: 169365.47535232778, Loss: 3.4740633964538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12341 , TFLOPS: 69.47596366840253, Tokens per sec: 172169.09137950558, Loss: 3.4819552898406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12342 , TFLOPS: 69.03657925420703, Tokens per sec: 171080.24839894017, Loss: 3.4931535720825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12343 , TFLOPS: 68.7778891551833, Tokens per sec: 170439.18583649248, Loss: 3.4868035316467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12344 , TFLOPS: 68.77076091269174, Tokens per sec: 170421.52126635768, Loss: 3.4964776039123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12345 , TFLOPS: 68.76409846004783, Tokens per sec: 170405.01097477632, Loss: 3.497767686843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12346 , TFLOPS: 68.64370869452928, Tokens per sec: 170106.67187379388, Loss: 3.4945108890533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12347 , TFLOPS: 68.76478271370993, Tokens per sec: 170406.70663072777, Loss: 3.490529775619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12348 , TFLOPS: 67.6125903063645, Tokens per sec: 167551.4469208542, Loss: 3.4963748455047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12349 , TFLOPS: 69.58303140554602, Tokens per sec: 172434.41702663206, Loss: 3.503509044647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12350 , TFLOPS: 67.8305519421623, Tokens per sec: 168091.57986481747, Loss: 3.494377374649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12351 , TFLOPS: 68.3514057238906, Tokens per sec: 169382.3129127208, Loss: 3.4933969974517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12352 , TFLOPS: 69.6397743467522, Tokens per sec: 172575.03228569197, Loss: 3.518958568572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12353 , TFLOPS: 69.49349081334765, Tokens per sec: 172212.52557545397, Loss: 3.5021324157714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12354 , TFLOPS: 69.09304091714102, Tokens per sec: 171220.16661945623, Loss: 3.4912986755371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12355 , TFLOPS: 67.75040195150774, Tokens per sec: 167892.95935872511, Loss: 3.4800426959991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12356 , TFLOPS: 69.6439684076129, Tokens per sec: 172585.4256305187, Loss: 3.5125341415405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12357 , TFLOPS: 68.974502329141, Tokens per sec: 170926.41494028879, Loss: 3.5147547721862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12358 , TFLOPS: 69.6392035106989, Tokens per sec: 172573.6176910691, Loss: 3.4911413192749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12359 , TFLOPS: 68.72885949754007, Tokens per sec: 170317.68494378988, Loss: 3.5068306922912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12360 , TFLOPS: 67.9441877480725, Tokens per sec: 168373.1819098803, Loss: 3.4963037967681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12361 , TFLOPS: 69.53919279321669, Tokens per sec: 172325.78011605816, Loss: 3.4970619678497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12362 , TFLOPS: 68.70120352460368, Tokens per sec: 170249.1503962966, Loss: 3.5057640075683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12363 , TFLOPS: 67.76467868810829, Tokens per sec: 167928.33868473364, Loss: 3.520188570022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12364 , TFLOPS: 69.67360032063162, Tokens per sec: 172658.85677520378, Loss: 3.5065970420837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12365 , TFLOPS: 68.78713343103975, Tokens per sec: 170462.09417040582, Loss: 3.4769504070281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12366 , TFLOPS: 67.77415706032008, Tokens per sec: 167951.82713520335, Loss: 3.5029594898223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12367 , TFLOPS: 68.89869894495614, Tokens per sec: 170738.56580384652, Loss: 3.5127837657928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12368 , TFLOPS: 69.5184504542235, Tokens per sec: 172274.37831507577, Loss: 3.492008924484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12369 , TFLOPS: 70.03441563924417, Tokens per sec: 173552.99688181427, Loss: 3.497328042984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12370 , TFLOPS: 67.39915896508535, Tokens per sec: 167022.5405457613, Loss: 3.505490779876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12371 , TFLOPS: 69.04063623977999, Tokens per sec: 171090.3020561038, Loss: 3.494961738586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12372 , TFLOPS: 69.7362393514368, Tokens per sec: 172814.08319379788, Loss: 3.501685619354248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12373 , TFLOPS: 69.45674978866575, Tokens per sec: 172121.47726893425, Loss: 3.509903907775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12374 , TFLOPS: 68.68299306158401, Tokens per sec: 170204.02286288576, Loss: 3.486314296722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12375 , TFLOPS: 68.78475394074329, Tokens per sec: 170456.19753132865, Loss: 3.494004249572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12376 , TFLOPS: 69.57431011275375, Tokens per sec: 172412.80470236306, Loss: 3.512894630432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12377 , TFLOPS: 68.80136966298406, Tokens per sec: 170497.37312141422, Loss: 3.4926438331604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12378 , TFLOPS: 68.68132979895339, Tokens per sec: 170199.90111486358, Loss: 3.502948760986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12379 , TFLOPS: 69.61161054718347, Tokens per sec: 172505.23928786197, Loss: 3.513747453689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12380 , TFLOPS: 67.86796252566178, Tokens per sec: 168184.28741184517, Loss: 3.4850668907165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12381 , TFLOPS: 68.60535473752732, Tokens per sec: 170011.62654330584, Loss: 3.499589443206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12382 , TFLOPS: 68.76797253689527, Tokens per sec: 170414.61136396904, Loss: 3.4969160556793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12383 , TFLOPS: 68.74615340484225, Tokens per sec: 170360.54115116582, Loss: 3.491091012954712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12384 , TFLOPS: 69.70909937166853, Tokens per sec: 172746.82733421653, Loss: 3.4974398612976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12385 , TFLOPS: 69.49623013573807, Tokens per sec: 172219.31391809872, Loss: 3.504746437072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12386 , TFLOPS: 68.96855182630246, Tokens per sec: 170911.66893875287, Loss: 3.490711212158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12387 , TFLOPS: 68.1949185684643, Tokens per sec: 168994.52050309122, Loss: 3.490736961364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12388 , TFLOPS: 69.51507669361, Tokens per sec: 172266.0177646251, Loss: 3.496199131011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12389 , TFLOPS: 69.78607121765377, Tokens per sec: 172937.57204771676, Loss: 3.491973638534546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12390 , TFLOPS: 68.76945839841356, Tokens per sec: 170418.29349249258, Loss: 3.486621856689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12391 , TFLOPS: 69.08673947218927, Tokens per sec: 171204.55094470433, Loss: 3.4842605590820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12392 , TFLOPS: 67.97637847404413, Tokens per sec: 168452.95407493925, Loss: 3.520876884460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12393 , TFLOPS: 69.45564252003301, Tokens per sec: 172118.73333528658, Loss: 3.48201584815979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12394 , TFLOPS: 68.80734424566862, Tokens per sec: 170512.1787954612, Loss: 3.513108491897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12395 , TFLOPS: 68.78685007859555, Tokens per sec: 170461.39199183526, Loss: 3.5005946159362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12396 , TFLOPS: 68.49691074114081, Tokens per sec: 169742.89037416768, Loss: 3.5156455039978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12397 , TFLOPS: 68.71823397007843, Tokens per sec: 170291.3537162415, Loss: 3.4806723594665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12398 , TFLOPS: 69.60091774098524, Tokens per sec: 172478.74133619294, Loss: 3.5001840591430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12399 , TFLOPS: 69.64941941798637, Tokens per sec: 172598.93383470253, Loss: 3.4847631454467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12400 , TFLOPS: 67.1813136614506, Tokens per sec: 166482.69588571315, Loss: 3.5014243125915527 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/optimizer.pt +[rank0]:[2024-08-25 23:01:11,415] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006350875002681278, 'preprocessing_with_comm': 0.001336125991656445, 'state_converting': 0.993259296999895, : 1.0023211859952426}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012400, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010400 + eval ppl=24.57215118408203, eval loss=3.201613664627075 +------------------------------------------------------------------ +iteration: 12401 , TFLOPS: 64.46511347667087, Tokens per sec: 159751.6526136778, Loss: 3.496267557144165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12402 , TFLOPS: 68.83862382820587, Tokens per sec: 170589.69304671895, Loss: 3.4883475303649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12403 , TFLOPS: 67.38770875616551, Tokens per sec: 166994.1656667133, Loss: 3.4824507236480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12404 , TFLOPS: 68.82577176447333, Tokens per sec: 170557.84421701802, Loss: 3.512995958328247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12405 , TFLOPS: 69.52811441134071, Tokens per sec: 172298.32666538496, Loss: 3.4855096340179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12406 , TFLOPS: 70.0459879009826, Tokens per sec: 173581.67422119193, Loss: 3.4889261722564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12407 , TFLOPS: 67.2252709351853, Tokens per sec: 166591.62685232118, Loss: 3.513550043106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12408 , TFLOPS: 70.44866435377239, Tokens per sec: 174579.55082967816, Loss: 3.4963574409484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12409 , TFLOPS: 68.7315709018938, Tokens per sec: 170324.4041024643, Loss: 3.4810075759887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12410 , TFLOPS: 67.03429060897173, Tokens per sec: 166118.35656574427, Loss: 3.4897513389587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12411 , TFLOPS: 69.4417535316777, Tokens per sec: 172084.31489214377, Loss: 3.499807357788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12412 , TFLOPS: 68.6827659515223, Tokens per sec: 170203.4600591346, Loss: 3.4833691120147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12413 , TFLOPS: 68.15408511214994, Tokens per sec: 168893.3307001677, Loss: 3.49495267868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12414 , TFLOPS: 68.0676644073852, Tokens per sec: 168679.17067373407, Loss: 3.4890358448028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12415 , TFLOPS: 69.63156473568766, Tokens per sec: 172554.6879651107, Loss: 3.4906797409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12416 , TFLOPS: 70.59179536947083, Tokens per sec: 174934.2452537619, Loss: 3.5055384635925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12417 , TFLOPS: 68.11170453182355, Tokens per sec: 168788.30695351257, Loss: 3.5132861137390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12418 , TFLOPS: 68.57880938990496, Tokens per sec: 169945.8442476862, Loss: 3.52115797996521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12419 , TFLOPS: 69.75223490226671, Tokens per sec: 172853.72193080976, Loss: 3.496032476425171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12420 , TFLOPS: 68.74012828042079, Tokens per sec: 170345.61022912723, Loss: 3.5024993419647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12421 , TFLOPS: 69.68384473987045, Tokens per sec: 172684.24357459357, Loss: 3.4959523677825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12422 , TFLOPS: 68.6056100775646, Tokens per sec: 170012.25930404625, Loss: 3.5025787353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12423 , TFLOPS: 68.73493335052106, Tokens per sec: 170332.73662056692, Loss: 3.4904427528381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12424 , TFLOPS: 69.64036911877908, Tokens per sec: 172576.5061962941, Loss: 3.496877431869507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12425 , TFLOPS: 69.53961445242147, Tokens per sec: 172326.82503401718, Loss: 3.511915922164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12426 , TFLOPS: 68.77457436411763, Tokens per sec: 170430.97141907737, Loss: 3.4928224086761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12427 , TFLOPS: 67.43588321882262, Tokens per sec: 167113.5472920328, Loss: 3.50351881980896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12428 , TFLOPS: 70.55836546026003, Tokens per sec: 174851.40225612908, Loss: 3.471501588821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12429 , TFLOPS: 67.17009286125554, Tokens per sec: 166454.8895067566, Loss: 3.460797071456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12430 , TFLOPS: 67.99783119988874, Tokens per sec: 168506.11629279502, Loss: 3.499490737915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12431 , TFLOPS: 69.96724768750218, Tokens per sec: 173386.5472982369, Loss: 3.500267744064331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12432 , TFLOPS: 68.83319158657143, Tokens per sec: 170576.231353538, Loss: 3.489004373550415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12433 , TFLOPS: 68.60465589115385, Tokens per sec: 170009.8947250081, Loss: 3.5113365650177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12434 , TFLOPS: 68.0201353469077, Tokens per sec: 168561.38842611262, Loss: 3.456639051437378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12435 , TFLOPS: 69.56127943176783, Tokens per sec: 172380.51323943198, Loss: 3.5154049396514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12436 , TFLOPS: 70.59108152984791, Tokens per sec: 174932.4762805404, Loss: 3.5093905925750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12437 , TFLOPS: 67.11124928442715, Tokens per sec: 166309.06864122074, Loss: 3.4915945529937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12438 , TFLOPS: 70.56685335691104, Tokens per sec: 174872.43619904926, Loss: 3.4895882606506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12439 , TFLOPS: 69.05496650949584, Tokens per sec: 171125.81404306923, Loss: 3.5007503032684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12440 , TFLOPS: 68.67865168545234, Tokens per sec: 170193.2644545894, Loss: 3.512251853942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12441 , TFLOPS: 68.64995653959114, Tokens per sec: 170122.1547221141, Loss: 3.4878077507019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12442 , TFLOPS: 69.53193059102806, Tokens per sec: 172307.78357903712, Loss: 3.4956586360931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12443 , TFLOPS: 69.55226226530921, Tokens per sec: 172358.1677076246, Loss: 3.492656946182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12444 , TFLOPS: 68.17410064687203, Tokens per sec: 168942.9314001022, Loss: 3.4939310550689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12445 , TFLOPS: 69.59259444336297, Tokens per sec: 172458.11528779805, Loss: 3.4767792224884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12446 , TFLOPS: 69.82029114125808, Tokens per sec: 173022.372788044, Loss: 3.481724262237549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12447 , TFLOPS: 67.8842757065936, Tokens per sec: 168224.71327123942, Loss: 3.5139200687408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12448 , TFLOPS: 69.65786509235011, Tokens per sec: 172619.8631461437, Loss: 3.489572048187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12449 , TFLOPS: 68.66582439624376, Tokens per sec: 170161.47701888802, Loss: 3.491283416748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12450 , TFLOPS: 68.7376015317157, Tokens per sec: 170339.34866749166, Loss: 3.503105401992798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12451 , TFLOPS: 68.15237661258753, Tokens per sec: 168889.0968500453, Loss: 3.494764566421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12452 , TFLOPS: 68.86644175722506, Tokens per sec: 170658.62893922636, Loss: 3.4847164154052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12453 , TFLOPS: 69.29707473775547, Tokens per sec: 171725.78490311964, Loss: 3.504002571105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12454 , TFLOPS: 66.95860521609076, Tokens per sec: 165930.79982475092, Loss: 3.474895477294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12455 , TFLOPS: 69.54412835708153, Tokens per sec: 172338.01098701244, Loss: 3.5160984992980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12456 , TFLOPS: 69.76624232996949, Tokens per sec: 172888.43387970282, Loss: 3.4942376613616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12457 , TFLOPS: 67.89675495522206, Tokens per sec: 168255.63822404348, Loss: 3.4911766052246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12458 , TFLOPS: 70.58519493238762, Tokens per sec: 174917.88864357155, Loss: 3.490887403488159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12459 , TFLOPS: 67.98546125382431, Tokens per sec: 168475.4622037853, Loss: 3.4807910919189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12460 , TFLOPS: 68.63128997399716, Tokens per sec: 170075.89691628027, Loss: 3.498760938644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12461 , TFLOPS: 69.7338793034504, Tokens per sec: 172808.23473490681, Loss: 3.475888967514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12462 , TFLOPS: 69.54146442424563, Tokens per sec: 172331.40946798868, Loss: 3.5190021991729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12463 , TFLOPS: 68.65639140354982, Tokens per sec: 170138.1010238625, Loss: 3.4812254905700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12464 , TFLOPS: 68.93251354648541, Tokens per sec: 170822.36211142194, Loss: 3.48496413230896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12465 , TFLOPS: 69.61255709760441, Tokens per sec: 172507.5849440762, Loss: 3.492297410964966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12466 , TFLOPS: 68.87469074248116, Tokens per sec: 170679.0708333917, Loss: 3.4904844760894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12467 , TFLOPS: 68.54560870749921, Tokens per sec: 169863.56929932768, Loss: 3.5088870525360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12468 , TFLOPS: 69.89166155199163, Tokens per sec: 173199.23652794154, Loss: 3.5061562061309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12469 , TFLOPS: 68.67940680504478, Tokens per sec: 170195.13572412275, Loss: 3.5004541873931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12470 , TFLOPS: 67.83847440326694, Tokens per sec: 168111.2125961659, Loss: 3.4828085899353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12471 , TFLOPS: 69.0125993127135, Tokens per sec: 171020.82346231063, Loss: 3.4919748306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12472 , TFLOPS: 69.59854935876679, Tokens per sec: 172472.87222415677, Loss: 3.498103618621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12473 , TFLOPS: 69.14978222703147, Tokens per sec: 171360.77783593602, Loss: 3.497387647628784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12474 , TFLOPS: 66.85177889004069, Tokens per sec: 165666.07242090756, Loss: 3.4769604206085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12475 , TFLOPS: 70.60855764603744, Tokens per sec: 174975.78402161118, Loss: 3.4789445400238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12476 , TFLOPS: 69.66798730146635, Tokens per sec: 172644.94709538712, Loss: 3.4967212677001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12477 , TFLOPS: 67.04309660851862, Tokens per sec: 166140.17880268238, Loss: 3.474057912826538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12478 , TFLOPS: 70.55607338651657, Tokens per sec: 174845.72224489998, Loss: 3.497332811355591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12479 , TFLOPS: 68.77969479001803, Tokens per sec: 170443.66039852, Loss: 3.505659341812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12480 , TFLOPS: 68.60564049945106, Tokens per sec: 170012.3346928321, Loss: 3.46067476272583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12481 , TFLOPS: 69.47595715876658, Tokens per sec: 172169.0752479106, Loss: 3.481513023376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12482 , TFLOPS: 69.68785549962956, Tokens per sec: 172694.18267909827, Loss: 3.4842112064361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12483 , TFLOPS: 70.60414490845758, Tokens per sec: 174964.84877178512, Loss: 3.4735021591186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12484 , TFLOPS: 68.18526067099909, Tokens per sec: 168970.58716926718, Loss: 3.5009212493896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12485 , TFLOPS: 68.58852339883268, Tokens per sec: 169969.9166318955, Loss: 3.4977173805236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12486 , TFLOPS: 69.82071503759158, Tokens per sec: 173023.42324985447, Loss: 3.5068774223327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12487 , TFLOPS: 68.0250174603607, Tokens per sec: 168573.48684105836, Loss: 3.4661757946014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12488 , TFLOPS: 68.77359609460522, Tokens per sec: 170428.54715946034, Loss: 3.5165252685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12489 , TFLOPS: 68.75105912546832, Tokens per sec: 170372.69806728698, Loss: 3.5004210472106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12490 , TFLOPS: 68.86333704710894, Tokens per sec: 170650.93512554656, Loss: 3.4931700229644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12491 , TFLOPS: 68.85677094219874, Tokens per sec: 170634.6635942629, Loss: 3.4893856048583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12492 , TFLOPS: 69.53803109013377, Tokens per sec: 172322.9012878465, Loss: 3.4848146438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12493 , TFLOPS: 69.75930958443551, Tokens per sec: 172871.25377256464, Loss: 3.510075092315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12494 , TFLOPS: 67.24670353142501, Tokens per sec: 166644.73918679857, Loss: 3.5121707916259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12495 , TFLOPS: 70.54583355052942, Tokens per sec: 174820.34680331277, Loss: 3.4816195964813232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12496 , TFLOPS: 69.49047330887637, Tokens per sec: 172205.0478669686, Loss: 3.5099050998687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12497 , TFLOPS: 67.96638076762467, Tokens per sec: 168428.17865709355, Loss: 3.500619649887085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12498 , TFLOPS: 70.5761447549751, Tokens per sec: 174895.461306984, Loss: 3.528571128845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12499 , TFLOPS: 68.79122531853224, Tokens per sec: 170472.23431837084, Loss: 3.48810076713562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12500 , TFLOPS: 68.70771596185648, Tokens per sec: 170265.28893321322, Loss: 3.4924685955047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12501 , TFLOPS: 68.94237291133865, Tokens per sec: 170846.79470362698, Loss: 3.4961116313934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12502 , TFLOPS: 69.6553467094179, Tokens per sec: 172613.62231581486, Loss: 3.514986038208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12503 , TFLOPS: 69.55731387839695, Tokens per sec: 172370.68616133716, Loss: 3.4796319007873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12504 , TFLOPS: 67.90347873907268, Tokens per sec: 168272.30050111125, Loss: 3.4996933937072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12505 , TFLOPS: 70.62579083795504, Tokens per sec: 175018.48976957577, Loss: 3.4930365085601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12506 , TFLOPS: 69.03211329820795, Tokens per sec: 171069.1812680088, Loss: 3.4915030002593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12507 , TFLOPS: 68.62880632415116, Tokens per sec: 170069.74215836526, Loss: 3.4930520057678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12508 , TFLOPS: 69.74376762541864, Tokens per sec: 172832.73908603273, Loss: 3.531461477279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12509 , TFLOPS: 68.80868502167085, Tokens per sec: 170515.50138609338, Loss: 3.503943920135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12510 , TFLOPS: 69.64829730287966, Tokens per sec: 172596.15310985688, Loss: 3.5022339820861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12511 , TFLOPS: 67.8363456093857, Tokens per sec: 168105.93721042218, Loss: 3.485865592956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12512 , TFLOPS: 69.51114126616498, Tokens per sec: 172256.26534189322, Loss: 3.5064148902893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12513 , TFLOPS: 69.74984490850522, Tokens per sec: 172847.79926298925, Loss: 3.4825010299682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12514 , TFLOPS: 67.11774731118071, Tokens per sec: 166325.1714673328, Loss: 3.4753427505493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12515 , TFLOPS: 69.63177541541815, Tokens per sec: 172555.21005269114, Loss: 3.4876019954681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12516 , TFLOPS: 69.64640706585693, Tokens per sec: 172591.4688942878, Loss: 3.5227465629577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12517 , TFLOPS: 68.78333611964581, Tokens per sec: 170452.68401446324, Loss: 3.5008351802825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12518 , TFLOPS: 68.89763632215991, Tokens per sec: 170735.93250749994, Loss: 3.480523109436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12519 , TFLOPS: 68.7886645138647, Tokens per sec: 170465.88836230873, Loss: 3.50663161277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12520 , TFLOPS: 69.56963483266122, Tokens per sec: 172401.2188432692, Loss: 3.5188217163085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12521 , TFLOPS: 68.82462870231052, Tokens per sec: 170555.01158305988, Loss: 3.4911563396453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12522 , TFLOPS: 69.59716615164075, Tokens per sec: 172469.4444845256, Loss: 3.488912582397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12523 , TFLOPS: 69.97565372757667, Tokens per sec: 173407.37839154323, Loss: 3.5013439655303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12524 , TFLOPS: 67.98129844173772, Tokens per sec: 168465.14629686315, Loss: 3.4965319633483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12525 , TFLOPS: 69.64903095996294, Tokens per sec: 172597.9711929283, Loss: 3.473567485809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12526 , TFLOPS: 68.79525902428219, Tokens per sec: 170482.23028556857, Loss: 3.4733920097351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12527 , TFLOPS: 68.46532932464167, Tokens per sec: 169664.62814510262, Loss: 3.474527597427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12528 , TFLOPS: 69.73109800612036, Tokens per sec: 172801.3423737378, Loss: 3.4809372425079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12529 , TFLOPS: 68.46042941546746, Tokens per sec: 169652.48563039934, Loss: 3.4633824825286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12530 , TFLOPS: 70.60231056536773, Tokens per sec: 174960.30306753892, Loss: 3.5083656311035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12531 , TFLOPS: 67.37517784352025, Tokens per sec: 166963.11268478437, Loss: 3.5009734630584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12532 , TFLOPS: 69.54400054063348, Tokens per sec: 172337.69424377408, Loss: 3.5092248916625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12533 , TFLOPS: 68.80309697087608, Tokens per sec: 170501.65358064932, Loss: 3.476362705230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12534 , TFLOPS: 67.6538435495546, Tokens per sec: 167653.67700189914, Loss: 3.49971604347229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12535 , TFLOPS: 69.87527406487423, Tokens per sec: 173158.6265296339, Loss: 3.4994523525238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12536 , TFLOPS: 69.61216110577209, Tokens per sec: 172506.60363269362, Loss: 3.4927024841308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12537 , TFLOPS: 67.98833177713543, Tokens per sec: 168482.57567676454, Loss: 3.503138542175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12538 , TFLOPS: 69.6609112581371, Tokens per sec: 172627.41188052675, Loss: 3.484539031982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12539 , TFLOPS: 69.57482568540307, Tokens per sec: 172414.0823481831, Loss: 3.51487398147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12540 , TFLOPS: 68.07485094811719, Tokens per sec: 168696.97971333083, Loss: 3.504019260406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12541 , TFLOPS: 68.76269067128855, Tokens per sec: 170401.52231915007, Loss: 3.4891722202301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12542 , TFLOPS: 70.58892639476817, Tokens per sec: 174927.13561840507, Loss: 3.507563829421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12543 , TFLOPS: 69.59928451170792, Tokens per sec: 172474.69401412865, Loss: 3.493788242340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12544 , TFLOPS: 67.05326557722776, Tokens per sec: 166165.37862734802, Loss: 3.496954917907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12545 , TFLOPS: 70.58717069045021, Tokens per sec: 174922.78478969837, Loss: 3.496289014816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12546 , TFLOPS: 68.76647471726687, Tokens per sec: 170410.89960193145, Loss: 3.4905457496643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12547 , TFLOPS: 68.70624492893722, Tokens per sec: 170261.64355159164, Loss: 3.4972805976867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12548 , TFLOPS: 68.86532061551593, Tokens per sec: 170655.85062656717, Loss: 3.479839324951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12549 , TFLOPS: 67.91267519174902, Tokens per sec: 168295.0903239161, Loss: 3.5025393962860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12550 , TFLOPS: 70.62697778863523, Tokens per sec: 175021.43116411474, Loss: 3.515413761138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12551 , TFLOPS: 66.23180843466315, Tokens per sec: 164129.7173370984, Loss: 3.4862101078033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12552 , TFLOPS: 70.51653828894428, Tokens per sec: 174747.74991796532, Loss: 3.4921183586120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12553 , TFLOPS: 69.00779482293048, Tokens per sec: 171008.91740737003, Loss: 3.489661931991577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12554 , TFLOPS: 67.82347286674057, Tokens per sec: 168074.03713609788, Loss: 3.49576473236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12555 , TFLOPS: 68.8024083512067, Tokens per sec: 170499.94710525085, Loss: 3.502991199493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12556 , TFLOPS: 69.52924432662854, Tokens per sec: 172301.12671993865, Loss: 3.493405818939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12557 , TFLOPS: 68.86073917433133, Tokens per sec: 170644.49731062434, Loss: 3.5004584789276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12558 , TFLOPS: 69.75912289425112, Tokens per sec: 172870.79113372212, Loss: 3.484876871109009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12559 , TFLOPS: 68.5569655591856, Tokens per sec: 169891.71282886702, Loss: 3.494023561477661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12560 , TFLOPS: 69.69555355977008, Tokens per sec: 172713.2593201378, Loss: 3.5006906986236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12561 , TFLOPS: 68.02357890586548, Tokens per sec: 168569.92194454934, Loss: 3.5016121864318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12562 , TFLOPS: 69.6672364322466, Tokens per sec: 172643.08635874552, Loss: 3.488203763961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12563 , TFLOPS: 69.5532704330898, Tokens per sec: 172360.66605844378, Loss: 3.4971179962158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12564 , TFLOPS: 68.03352464487571, Tokens per sec: 168594.5685814303, Loss: 3.4764628410339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12565 , TFLOPS: 70.60060122145242, Tokens per sec: 174956.067125017, Loss: 3.4746005535125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12566 , TFLOPS: 67.85783237167131, Tokens per sec: 168159.18377432818, Loss: 3.4793293476104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12567 , TFLOPS: 69.64449889277756, Tokens per sec: 172586.74023119448, Loss: 3.494215726852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12568 , TFLOPS: 69.01991941397412, Tokens per sec: 171038.96347961016, Loss: 3.469792366027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12569 , TFLOPS: 69.33526430094209, Tokens per sec: 171820.42284762376, Loss: 3.4883720874786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12570 , TFLOPS: 68.91247531884805, Tokens per sec: 170772.70517594359, Loss: 3.508197784423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12571 , TFLOPS: 67.06328179257652, Tokens per sec: 166190.19991235944, Loss: 3.512073278427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12572 , TFLOPS: 70.55171541770872, Tokens per sec: 174834.92271812423, Loss: 3.495931386947632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12573 , TFLOPS: 68.73826550812007, Tokens per sec: 170340.99407416472, Loss: 3.5096490383148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12574 , TFLOPS: 65.2855869881226, Tokens per sec: 161784.87635767378, Loss: 3.521821975708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12575 , TFLOPS: 69.65461319425009, Tokens per sec: 172611.8045844257, Loss: 3.494314432144165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12576 , TFLOPS: 69.43692747194117, Tokens per sec: 172072.3553844823, Loss: 3.4798638820648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12577 , TFLOPS: 67.11485784123816, Tokens per sec: 166318.01104251377, Loss: 3.4967715740203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12578 , TFLOPS: 68.96665495754651, Tokens per sec: 170906.96828901826, Loss: 3.4564177989959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12579 , TFLOPS: 69.6537838776873, Tokens per sec: 172609.74944662637, Loss: 3.5158140659332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12580 , TFLOPS: 69.694329120725, Tokens per sec: 172710.225025301, Loss: 3.4971117973327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12581 , TFLOPS: 66.96996060216154, Tokens per sec: 165958.93972233334, Loss: 3.477778911590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12582 , TFLOPS: 70.5742823076973, Tokens per sec: 174890.84595746413, Loss: 3.5088400840759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12583 , TFLOPS: 69.553799080425, Tokens per sec: 172361.97610477568, Loss: 3.523740530014038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12584 , TFLOPS: 68.8507665441031, Tokens per sec: 170619.78403434285, Loss: 3.4997334480285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12585 , TFLOPS: 69.4687106643803, Tokens per sec: 172151.11763655412, Loss: 3.5100696086883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12586 , TFLOPS: 68.80775021728851, Tokens per sec: 170513.18483785674, Loss: 3.4823358058929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12587 , TFLOPS: 69.65401947093966, Tokens per sec: 172610.33327265846, Loss: 3.500997543334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12588 , TFLOPS: 68.76018771475218, Tokens per sec: 170395.31971712952, Loss: 3.489553451538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12589 , TFLOPS: 69.54084206555316, Tokens per sec: 172329.86719459028, Loss: 3.481646776199341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12590 , TFLOPS: 69.01259621806626, Tokens per sec: 171020.81579343384, Loss: 3.4674932956695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12591 , TFLOPS: 67.05967577288014, Tokens per sec: 166181.26379831595, Loss: 3.4735498428344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12592 , TFLOPS: 69.5189855191416, Tokens per sec: 172275.70426488508, Loss: 3.491640567779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12593 , TFLOPS: 68.72349103262341, Tokens per sec: 170304.3812963412, Loss: 3.48175048828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12594 , TFLOPS: 67.89312276652932, Tokens per sec: 168246.63726034423, Loss: 3.5263969898223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12595 , TFLOPS: 69.60118897269155, Tokens per sec: 172479.4134782398, Loss: 3.4906086921691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12596 , TFLOPS: 68.60801077850093, Tokens per sec: 170018.2085054255, Loss: 3.4978952407836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12597 , TFLOPS: 69.73186166883102, Tokens per sec: 172803.23481405922, Loss: 3.499502658843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12598 , TFLOPS: 68.65453916736199, Tokens per sec: 170133.51097854838, Loss: 3.4739410877227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12599 , TFLOPS: 69.48204093666422, Tokens per sec: 172184.1515197244, Loss: 3.500178813934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12600 , TFLOPS: 69.62864408037139, Tokens per sec: 172547.45025950024, Loss: 3.4855761528015137 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/model.pt +[rank0]:[2024-08-25 23:26:54,114] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006315698003163561, 'preprocessing_with_comm': 0.0014172359951771796, 'state_converting': 0.9981969500076957, : 1.0072909229929792}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012600, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010600 + eval ppl=24.32666015625, eval loss=3.191572904586792 +------------------------------------------------------------------ +iteration: 12601 , TFLOPS: 61.908897358501704, Tokens per sec: 153417.0674824033, Loss: 3.5057735443115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12602 , TFLOPS: 70.50456030845665, Tokens per sec: 174718.0671628335, Loss: 3.4788665771484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12603 , TFLOPS: 67.43317824703414, Tokens per sec: 167106.84407396367, Loss: 3.516603946685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12604 , TFLOPS: 68.86553019523605, Tokens per sec: 170656.36998820075, Loss: 3.5066416263580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12605 , TFLOPS: 69.89660259643843, Tokens per sec: 173211.48098038166, Loss: 3.497046947479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12606 , TFLOPS: 68.72460876546235, Tokens per sec: 170307.15116144452, Loss: 3.4956066608428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12607 , TFLOPS: 68.62506265793711, Tokens per sec: 170060.46494108796, Loss: 3.4952356815338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12608 , TFLOPS: 68.00537835150934, Tokens per sec: 168524.81896589824, Loss: 3.4842286109924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12609 , TFLOPS: 70.51939122862466, Tokens per sec: 174754.81981677056, Loss: 3.5024280548095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12610 , TFLOPS: 67.97678079990925, Tokens per sec: 168453.95108275284, Loss: 3.492882251739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12611 , TFLOPS: 66.97988506866932, Tokens per sec: 165983.53364360987, Loss: 3.5108425617218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12612 , TFLOPS: 70.58719220416685, Tokens per sec: 174922.83810305817, Loss: 3.4971468448638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12613 , TFLOPS: 68.74146538507162, Tokens per sec: 170348.92372174622, Loss: 3.5077261924743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12614 , TFLOPS: 67.77313562627361, Tokens per sec: 167949.29590911596, Loss: 3.5218236446380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12615 , TFLOPS: 69.66736211687511, Tokens per sec: 172643.39781910027, Loss: 3.485175848007202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12616 , TFLOPS: 68.92683958960592, Tokens per sec: 170808.30142123663, Loss: 3.5022687911987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12617 , TFLOPS: 69.65935214532715, Tokens per sec: 172623.5482272332, Loss: 3.4951720237731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12618 , TFLOPS: 66.79317976823306, Tokens per sec: 165520.8573418421, Loss: 3.4913649559020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12619 , TFLOPS: 70.60774890644969, Tokens per sec: 174973.7798758227, Loss: 3.487025499343872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12620 , TFLOPS: 68.71025025150512, Tokens per sec: 170271.56918213758, Loss: 3.4857380390167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12621 , TFLOPS: 66.97038713185427, Tokens per sec: 165959.99670989806, Loss: 3.4675545692443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12622 , TFLOPS: 70.50566914348475, Tokens per sec: 174720.8149781814, Loss: 3.49858021736145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12623 , TFLOPS: 68.86167901071693, Tokens per sec: 170646.82632871935, Loss: 3.4838919639587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12624 , TFLOPS: 67.96215035040889, Tokens per sec: 168417.6952172139, Loss: 3.46993350982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12625 , TFLOPS: 68.61257660415194, Tokens per sec: 170029.52312435594, Loss: 3.5164031982421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12626 , TFLOPS: 69.53544842128171, Tokens per sec: 172316.50114992665, Loss: 3.5111169815063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12627 , TFLOPS: 69.18960718235495, Tokens per sec: 171459.46846230864, Loss: 3.5170931816101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12628 , TFLOPS: 67.86452639636148, Tokens per sec: 168175.77230491812, Loss: 3.4858100414276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12629 , TFLOPS: 69.52337826279451, Tokens per sec: 172286.58996755764, Loss: 3.4754884243011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12630 , TFLOPS: 68.70537225930559, Tokens per sec: 170259.48098011198, Loss: 3.4931797981262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12631 , TFLOPS: 66.6793762804114, Tokens per sec: 165238.83976253093, Loss: 3.4892144203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12632 , TFLOPS: 69.48432027978215, Tokens per sec: 172189.799983061, Loss: 3.509446382522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12633 , TFLOPS: 68.74787365628158, Tokens per sec: 170364.8041237339, Loss: 3.4959311485290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12634 , TFLOPS: 68.74839780867579, Tokens per sec: 170366.10303110688, Loss: 3.5133135318756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12635 , TFLOPS: 69.64805360312617, Tokens per sec: 172595.549195022, Loss: 3.4980292320251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12636 , TFLOPS: 68.68257736090104, Tokens per sec: 170202.99271080014, Loss: 3.492487907409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12637 , TFLOPS: 69.980953328862, Tokens per sec: 173420.51138732754, Loss: 3.486751079559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12638 , TFLOPS: 67.34009423760615, Tokens per sec: 166876.17164455098, Loss: 3.5122244358062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12639 , TFLOPS: 70.5862678524142, Tokens per sec: 174920.5474575893, Loss: 3.500507354736328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12640 , TFLOPS: 68.72740071843965, Tokens per sec: 170314.0699284813, Loss: 3.4919633865356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12641 , TFLOPS: 67.64375887051807, Tokens per sec: 167628.68605632754, Loss: 3.4672963619232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12642 , TFLOPS: 70.57326423928619, Tokens per sec: 174888.3230717919, Loss: 3.4803669452667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12643 , TFLOPS: 68.76317040379166, Tokens per sec: 170402.71114913968, Loss: 3.5219154357910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12644 , TFLOPS: 67.7635414487636, Tokens per sec: 167925.52048036025, Loss: 3.4845142364501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12645 , TFLOPS: 68.9679163746477, Tokens per sec: 170910.09421955157, Loss: 3.4985625743865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12646 , TFLOPS: 69.52133806388197, Tokens per sec: 172281.53412990592, Loss: 3.490891933441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12647 , TFLOPS: 68.69063976930775, Tokens per sec: 170222.97224697957, Loss: 3.491591691970825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12648 , TFLOPS: 67.46383674032148, Tokens per sec: 167182.81919764157, Loss: 3.5164952278137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12649 , TFLOPS: 70.58735681350478, Tokens per sec: 174923.24602313075, Loss: 3.495497703552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12650 , TFLOPS: 68.78973786386827, Tokens per sec: 170468.54824185686, Loss: 3.504013776779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12651 , TFLOPS: 66.17548990059267, Tokens per sec: 163990.1538056731, Loss: 3.5056815147399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12652 , TFLOPS: 70.54446356892265, Tokens per sec: 174816.95183797527, Loss: 3.4847002029418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12653 , TFLOPS: 68.7559787077136, Tokens per sec: 170384.8893340275, Loss: 3.513735771179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12654 , TFLOPS: 69.74119614691665, Tokens per sec: 172826.36667903303, Loss: 3.5055110454559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12655 , TFLOPS: 68.73519371638726, Tokens per sec: 170333.38183586503, Loss: 3.5037784576416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12656 , TFLOPS: 69.62434026144004, Tokens per sec: 172536.78492208358, Loss: 3.4831020832061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12657 , TFLOPS: 69.6421984021402, Tokens per sec: 172581.03936197513, Loss: 3.4889230728149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12658 , TFLOPS: 67.14196601186933, Tokens per sec: 166385.18807554932, Loss: 3.4887635707855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12659 , TFLOPS: 69.48521879542902, Tokens per sec: 172192.0266038136, Loss: 3.5028598308563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12660 , TFLOPS: 68.81994709606381, Tokens per sec: 170543.41004705906, Loss: 3.495523691177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12661 , TFLOPS: 68.71651403302886, Tokens per sec: 170287.09152276593, Loss: 3.500962734222412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12662 , TFLOPS: 68.92267734171186, Tokens per sec: 170797.9869124457, Loss: 3.4967093467712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12663 , TFLOPS: 68.6631009988456, Tokens per sec: 170154.72814012834, Loss: 3.493986129760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12664 , TFLOPS: 69.67819934068365, Tokens per sec: 172670.2536535166, Loss: 3.4923224449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12665 , TFLOPS: 68.11104713698103, Tokens per sec: 168786.67785667474, Loss: 3.492475986480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12666 , TFLOPS: 68.62358993183268, Tokens per sec: 170056.81536356703, Loss: 3.494755744934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12667 , TFLOPS: 69.73788000685008, Tokens per sec: 172818.1489186451, Loss: 3.497411012649536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12668 , TFLOPS: 67.30008421435629, Tokens per sec: 166777.02239946995, Loss: 3.500309705734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12669 , TFLOPS: 69.49187406898781, Tokens per sec: 172208.5191048312, Loss: 3.49452805519104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12670 , TFLOPS: 68.76149779910156, Tokens per sec: 170398.566250465, Loss: 3.4844162464141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12671 , TFLOPS: 67.90620827403556, Tokens per sec: 168279.06458943235, Loss: 3.48179292678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12672 , TFLOPS: 69.88725826481884, Tokens per sec: 173188.32469726523, Loss: 3.4856035709381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12673 , TFLOPS: 67.44772429258806, Tokens per sec: 167142.8907772255, Loss: 3.4915854930877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12674 , TFLOPS: 69.76856686423186, Tokens per sec: 172894.19433167359, Loss: 3.502776622772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12675 , TFLOPS: 68.60261952040308, Tokens per sec: 170004.84837396294, Loss: 3.4846436977386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12676 , TFLOPS: 70.60669419391455, Tokens per sec: 174971.16618197446, Loss: 3.498422622680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12677 , TFLOPS: 68.91621674244486, Tokens per sec: 170781.97683576832, Loss: 3.4952168464660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12678 , TFLOPS: 67.14759730997017, Tokens per sec: 166399.14305258193, Loss: 3.480070114135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12679 , TFLOPS: 70.60804913522645, Tokens per sec: 174974.52387580447, Loss: 3.511495351791382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12680 , TFLOPS: 68.78570084660547, Tokens per sec: 170458.54406836545, Loss: 3.5104010105133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12681 , TFLOPS: 67.81554355455985, Tokens per sec: 168054.38742702824, Loss: 3.4846270084381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12682 , TFLOPS: 69.68599531857906, Tokens per sec: 172689.5729455393, Loss: 3.4896600246429443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12683 , TFLOPS: 68.73205036101905, Tokens per sec: 170325.59225499345, Loss: 3.5071465969085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12684 , TFLOPS: 69.48214337597031, Tokens per sec: 172184.4053756095, Loss: 3.5015387535095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12685 , TFLOPS: 67.58655975209429, Tokens per sec: 167486.94033987034, Loss: 3.4770984649658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12686 , TFLOPS: 70.59435857941276, Tokens per sec: 174940.5971703589, Loss: 3.5057761669158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12687 , TFLOPS: 68.96278733435076, Tokens per sec: 170897.38389268532, Loss: 3.514901638031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12688 , TFLOPS: 67.05578637260902, Tokens per sec: 166171.6254359913, Loss: 3.4940621852874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12689 , TFLOPS: 70.55295272802465, Tokens per sec: 174837.9889093881, Loss: 3.5002472400665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12690 , TFLOPS: 68.51038023941143, Tokens per sec: 169776.26927467002, Loss: 3.50059175491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12691 , TFLOPS: 68.8414632032728, Tokens per sec: 170596.72933091776, Loss: 3.482267141342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12692 , TFLOPS: 68.54896001456318, Tokens per sec: 169871.8742074091, Loss: 3.5041329860687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12693 , TFLOPS: 69.65640004287974, Tokens per sec: 172616.2325921676, Loss: 3.511549949645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12694 , TFLOPS: 69.16017333289622, Tokens per sec: 171386.52814094373, Loss: 3.494751214981079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12695 , TFLOPS: 67.75109731348442, Tokens per sec: 167894.68254230378, Loss: 3.4839909076690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12696 , TFLOPS: 70.59753567558263, Tokens per sec: 174948.47036465388, Loss: 3.4946699142456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12697 , TFLOPS: 68.7102941776463, Tokens per sec: 170271.67803595433, Loss: 3.475315570831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12698 , TFLOPS: 66.4671635936583, Tokens per sec: 164712.95334759945, Loss: 3.4901225566864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12699 , TFLOPS: 69.6126474192376, Tokens per sec: 172507.80877103304, Loss: 3.515820026397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12700 , TFLOPS: 68.85484908144304, Tokens per sec: 170629.90101159984, Loss: 3.47506046295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12701 , TFLOPS: 68.70312932624942, Tokens per sec: 170253.9227449193, Loss: 3.520969867706299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12702 , TFLOPS: 69.69867820762296, Tokens per sec: 172721.0025417256, Loss: 3.5172650814056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12703 , TFLOPS: 68.70615536596779, Tokens per sec: 170261.4216046872, Loss: 3.5022623538970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12704 , TFLOPS: 69.83672132772548, Tokens per sec: 173063.0886000447, Loss: 3.513660192489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12705 , TFLOPS: 67.23935607031862, Tokens per sec: 166626.53136878728, Loss: 3.4959115982055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12706 , TFLOPS: 69.59715472472348, Tokens per sec: 172469.41616736582, Loss: 3.498871326446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12707 , TFLOPS: 68.86612092560932, Tokens per sec: 170657.83388314006, Loss: 3.5058798789978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12708 , TFLOPS: 67.14632964089043, Tokens per sec: 166396.00162895705, Loss: 3.489959478378296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12709 , TFLOPS: 70.47489204276332, Tokens per sec: 174644.5459322162, Loss: 3.472991704940796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12710 , TFLOPS: 68.64310463259098, Tokens per sec: 170105.17494176782, Loss: 3.4809014797210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12711 , TFLOPS: 68.7099264595825, Tokens per sec: 170270.76679008474, Loss: 3.5141985416412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12712 , TFLOPS: 68.86489049585259, Tokens per sec: 170654.78474265983, Loss: 3.4889328479766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12713 , TFLOPS: 69.5925335927822, Tokens per sec: 172457.96449335566, Loss: 3.5119361877441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12714 , TFLOPS: 68.85330290890626, Tokens per sec: 170626.06942572875, Loss: 3.5083093643188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12715 , TFLOPS: 68.14683890505911, Tokens per sec: 168875.373800761, Loss: 3.469356060028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12716 , TFLOPS: 70.44565151546462, Tokens per sec: 174572.08468446, Loss: 3.499530792236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12717 , TFLOPS: 68.60741342592956, Tokens per sec: 170016.72819994984, Loss: 3.497943639755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12718 , TFLOPS: 65.97872187585784, Tokens per sec: 163502.5409645929, Loss: 3.467223644256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12719 , TFLOPS: 70.57105080679582, Tokens per sec: 174882.83794225103, Loss: 3.4985287189483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12720 , TFLOPS: 68.88563998666584, Tokens per sec: 170706.20426664018, Loss: 3.5106496810913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12721 , TFLOPS: 68.78700829281871, Tokens per sec: 170461.78406410897, Loss: 3.4989914894104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12722 , TFLOPS: 68.9758389700062, Tokens per sec: 170929.72728359583, Loss: 3.5022902488708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12723 , TFLOPS: 70.445017254348, Tokens per sec: 174570.51291554337, Loss: 3.493908166885376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12724 , TFLOPS: 69.02843338666256, Tokens per sec: 171060.0620418236, Loss: 3.493272542953491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12725 , TFLOPS: 66.20976169850792, Tokens per sec: 164075.0830962573, Loss: 3.5088319778442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12726 , TFLOPS: 70.60173179603417, Tokens per sec: 174958.8688133736, Loss: 3.518765449523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12727 , TFLOPS: 68.82714461121377, Tokens per sec: 170561.24628247236, Loss: 3.5043253898620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12728 , TFLOPS: 67.43795635451052, Tokens per sec: 167118.68475064213, Loss: 3.4892327785491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12729 , TFLOPS: 69.49881961729565, Tokens per sec: 172225.73093865335, Loss: 3.4649171829223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12730 , TFLOPS: 68.78669354523716, Tokens per sec: 170461.00408492956, Loss: 3.4805994033813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12731 , TFLOPS: 69.61337592265033, Tokens per sec: 172509.61408274106, Loss: 3.4886975288391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12732 , TFLOPS: 68.14566568151254, Tokens per sec: 168872.46642357137, Loss: 3.500922203063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12733 , TFLOPS: 68.64302706195197, Tokens per sec: 170104.98271317926, Loss: 3.466184616088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12734 , TFLOPS: 69.64203670592647, Tokens per sec: 172580.63866094523, Loss: 3.499541997909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12735 , TFLOPS: 67.89584634059474, Tokens per sec: 168253.38657690483, Loss: 3.4745373725891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12736 , TFLOPS: 69.65445244826802, Tokens per sec: 172611.4062381745, Loss: 3.516669273376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12737 , TFLOPS: 68.38661345486973, Tokens per sec: 169469.56154853883, Loss: 3.505850076675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12738 , TFLOPS: 68.02112968502162, Tokens per sec: 168563.85250549432, Loss: 3.4992072582244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12739 , TFLOPS: 69.94736987581604, Tokens per sec: 173337.28789115688, Loss: 3.4974982738494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12740 , TFLOPS: 67.77556067806483, Tokens per sec: 167955.3054545961, Loss: 3.4796199798583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12741 , TFLOPS: 69.66646204711925, Tokens per sec: 172641.1673470944, Loss: 3.5022592544555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12742 , TFLOPS: 68.792953484705, Tokens per sec: 170476.51690452013, Loss: 3.4965031147003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12743 , TFLOPS: 69.46383530580307, Tokens per sec: 172139.0359609347, Loss: 3.5060250759124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12744 , TFLOPS: 68.72747520563296, Tokens per sec: 170314.25451595173, Loss: 3.505648374557495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12745 , TFLOPS: 66.93093541591084, Tokens per sec: 165862.23101182497, Loss: 3.482551336288452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12746 , TFLOPS: 70.55282888597067, Tokens per sec: 174837.682015136, Loss: 3.4979140758514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12747 , TFLOPS: 67.80359338934501, Tokens per sec: 168024.77360120646, Loss: 3.4962055683135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12748 , TFLOPS: 68.12123269632379, Tokens per sec: 168811.9187947581, Loss: 3.494549036026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12749 , TFLOPS: 69.73210537698827, Tokens per sec: 172803.83874971955, Loss: 3.517021894454956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12750 , TFLOPS: 68.76974138440451, Tokens per sec: 170418.9947629516, Loss: 3.4791784286499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12751 , TFLOPS: 68.61775311669057, Tokens per sec: 170042.3510926666, Loss: 3.486032485961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12752 , TFLOPS: 67.86532413578477, Tokens per sec: 168177.7491910873, Loss: 3.4892466068267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12753 , TFLOPS: 70.57079112492775, Tokens per sec: 174882.19442197564, Loss: 3.496758460998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12754 , TFLOPS: 69.01756005011843, Tokens per sec: 171033.11671607036, Loss: 3.507514238357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12755 , TFLOPS: 67.01497267484638, Tokens per sec: 166070.4845969355, Loss: 3.4761719703674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12756 , TFLOPS: 70.59885547064174, Tokens per sec: 174951.7409621978, Loss: 3.4907827377319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12757 , TFLOPS: 68.81993897338546, Tokens per sec: 170543.3899181673, Loss: 3.489565134048462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12758 , TFLOPS: 67.94888946009203, Tokens per sec: 168384.83327020108, Loss: 3.509486436843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12759 , TFLOPS: 69.65578698889841, Tokens per sec: 172614.71337686843, Loss: 3.500929117202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12760 , TFLOPS: 69.59769868702604, Tokens per sec: 172470.76416587396, Loss: 3.502617835998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12761 , TFLOPS: 69.94788053118388, Tokens per sec: 173338.55335141177, Loss: 3.474383592605591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12762 , TFLOPS: 67.18990354657048, Tokens per sec: 166503.98256729418, Loss: 3.4897518157958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12763 , TFLOPS: 70.59591806083377, Tokens per sec: 174944.46173711124, Loss: 3.4951581954956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12764 , TFLOPS: 68.84476287966571, Tokens per sec: 170604.9062925088, Loss: 3.513592004776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12765 , TFLOPS: 67.20628075500748, Tokens per sec: 166544.5670938995, Loss: 3.47829532623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12766 , TFLOPS: 69.47618444828531, Tokens per sec: 172169.638496376, Loss: 3.4897942543029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12767 , TFLOPS: 68.57844250027205, Tokens per sec: 169944.93505475958, Loss: 3.4930598735809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12768 , TFLOPS: 68.67676635537742, Tokens per sec: 170188.59239897752, Loss: 3.518533945083618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12769 , TFLOPS: 68.24627875525918, Tokens per sec: 169121.79670376057, Loss: 3.4994850158691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12770 , TFLOPS: 69.84789586647372, Tokens per sec: 173090.7803380399, Loss: 3.4929895401000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12771 , TFLOPS: 69.80046742152636, Tokens per sec: 172973.24742678585, Loss: 3.499659776687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12772 , TFLOPS: 67.38380870823127, Tokens per sec: 166984.5009183065, Loss: 3.4837162494659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12773 , TFLOPS: 69.58125831431819, Tokens per sec: 172430.02311124734, Loss: 3.5002822875976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12774 , TFLOPS: 68.59603911794865, Tokens per sec: 169988.5414117897, Loss: 3.502464532852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12775 , TFLOPS: 67.8354017933246, Tokens per sec: 168103.59833025286, Loss: 3.506183624267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12776 , TFLOPS: 70.52941793378211, Tokens per sec: 174779.6670966835, Loss: 3.5000557899475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12777 , TFLOPS: 67.84480063230981, Tokens per sec: 168126.88968862576, Loss: 3.4909591674804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12778 , TFLOPS: 68.9013572772089, Tokens per sec: 170745.15344400838, Loss: 3.499873399734497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12779 , TFLOPS: 69.06834378274311, Tokens per sec: 171158.96439980427, Loss: 3.5103342533111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12780 , TFLOPS: 68.61066537157437, Tokens per sec: 170024.78687948908, Loss: 3.4881746768951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12781 , TFLOPS: 69.60263843287629, Tokens per sec: 172483.00540024883, Loss: 3.527792453765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12782 , TFLOPS: 67.78331210169341, Tokens per sec: 167974.51433623754, Loss: 3.4868388175964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12783 , TFLOPS: 70.48244429435067, Tokens per sec: 174663.2612435988, Loss: 3.47989821434021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12784 , TFLOPS: 67.76159662792912, Tokens per sec: 167920.7010000927, Loss: 3.4944827556610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12785 , TFLOPS: 67.20063893636427, Tokens per sec: 166530.586045804, Loss: 3.506369113922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12786 , TFLOPS: 70.55764650423976, Tokens per sec: 174849.6206039113, Loss: 3.4937028884887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12787 , TFLOPS: 68.69164926185167, Tokens per sec: 170225.4738807082, Loss: 3.4859120845794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12788 , TFLOPS: 68.67215237209476, Tokens per sec: 170177.15844013085, Loss: 3.49371075630188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12789 , TFLOPS: 68.27123454348985, Tokens per sec: 169183.63989610315, Loss: 3.465280294418335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12790 , TFLOPS: 69.67782273023711, Tokens per sec: 172669.320371343, Loss: 3.473992347717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12791 , TFLOPS: 69.50004195856039, Tokens per sec: 172228.76003495936, Loss: 3.515420913696289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12792 , TFLOPS: 65.56539058688722, Tokens per sec: 162478.26049832342, Loss: 3.507491111755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12793 , TFLOPS: 70.60591093903881, Tokens per sec: 174969.22519010972, Loss: 3.4892406463623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12794 , TFLOPS: 68.69900863564077, Tokens per sec: 170243.71121965954, Loss: 3.4866228103637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12795 , TFLOPS: 67.95144120737837, Tokens per sec: 168391.15678107477, Loss: 3.481677532196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12796 , TFLOPS: 69.97388629002751, Tokens per sec: 173402.9984865971, Loss: 3.4862565994262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12797 , TFLOPS: 68.85742273292706, Tokens per sec: 170636.27880348839, Loss: 3.510392427444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12798 , TFLOPS: 69.74000063041785, Tokens per sec: 172823.40405745202, Loss: 3.491515636444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12799 , TFLOPS: 67.15788311837721, Tokens per sec: 166424.63241888993, Loss: 3.472752094268799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12800 , TFLOPS: 70.54595733040966, Tokens per sec: 174820.65354349153, Loss: 3.4974000453948975 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/model.pt +[rank0]:[2024-08-25 23:52:39,996] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006318428000668064, 'preprocessing_with_comm': 0.0012350200122455135, 'state_converting': 0.9992515460035065, : 1.008216016998631}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0012800, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0010800 + eval ppl=25.153871536254883, eval loss=3.2250118255615234 +------------------------------------------------------------------ +iteration: 12801 , TFLOPS: 63.19850058754975, Tokens per sec: 156612.84634550754, Loss: 3.485485076904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12802 , TFLOPS: 67.23312391982384, Tokens per sec: 166611.08741333315, Loss: 3.4929959774017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12803 , TFLOPS: 69.2094473215243, Tokens per sec: 171508.63451274202, Loss: 3.5118398666381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12804 , TFLOPS: 68.76012189039142, Tokens per sec: 170395.1565971124, Loss: 3.491152048110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12805 , TFLOPS: 68.55158643825573, Tokens per sec: 169878.3827746437, Loss: 3.5016849040985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12806 , TFLOPS: 69.49063735755936, Tokens per sec: 172205.4543976765, Loss: 3.4991419315338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12807 , TFLOPS: 69.56247316087867, Tokens per sec: 172383.47143166876, Loss: 3.518305778503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12808 , TFLOPS: 69.64569758697466, Tokens per sec: 172589.71072746196, Loss: 3.5011632442474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12809 , TFLOPS: 68.11374001797445, Tokens per sec: 168793.35111242178, Loss: 3.530437469482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12810 , TFLOPS: 69.52002724698144, Tokens per sec: 172278.285781228, Loss: 3.4912467002868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12811 , TFLOPS: 69.53712166398783, Tokens per sec: 172320.6476296753, Loss: 3.484401226043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12812 , TFLOPS: 67.87917159950904, Tokens per sec: 168212.06473162008, Loss: 3.4817745685577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12813 , TFLOPS: 70.56035328628204, Tokens per sec: 174856.3283079887, Loss: 3.492619514465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12814 , TFLOPS: 68.69586217989117, Tokens per sec: 170235.9139557013, Loss: 3.4715158939361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12815 , TFLOPS: 68.90510115231083, Tokens per sec: 170754.43117893283, Loss: 3.500884771347046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12816 , TFLOPS: 69.0841867367067, Tokens per sec: 171198.22498497152, Loss: 3.5026659965515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12817 , TFLOPS: 70.58719309816969, Tokens per sec: 174922.84031849567, Loss: 3.4973387718200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12818 , TFLOPS: 68.61516143251649, Tokens per sec: 170035.92861378565, Loss: 3.5317776203155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12819 , TFLOPS: 67.4990923522003, Tokens per sec: 167270.18648760353, Loss: 3.503730535507202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12820 , TFLOPS: 70.5157385027707, Tokens per sec: 174745.76795972348, Loss: 3.480849504470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12821 , TFLOPS: 67.70411716528407, Tokens per sec: 167778.26055977275, Loss: 3.5110933780670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12822 , TFLOPS: 67.20499194812933, Tokens per sec: 166541.37328848164, Loss: 3.5080206394195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12823 , TFLOPS: 70.53484544535911, Tokens per sec: 174793.11706826105, Loss: 3.506981134414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12824 , TFLOPS: 68.7303744517175, Tokens per sec: 170321.43916712754, Loss: 3.4890785217285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12825 , TFLOPS: 68.64893457274242, Tokens per sec: 170119.62217568402, Loss: 3.499447822570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12826 , TFLOPS: 69.76285057056508, Tokens per sec: 172880.02872626553, Loss: 3.4924564361572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12827 , TFLOPS: 69.57577375944989, Tokens per sec: 172416.4317801101, Loss: 3.5038774013519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12828 , TFLOPS: 69.76859627589634, Tokens per sec: 172894.26721701812, Loss: 3.492185115814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12829 , TFLOPS: 66.4973443131336, Tokens per sec: 164787.74449513995, Loss: 3.4835782051086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12830 , TFLOPS: 70.58641390851915, Tokens per sec: 174920.90940070682, Loss: 3.472857713699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12831 , TFLOPS: 69.53358745476328, Tokens per sec: 172311.8894698916, Loss: 3.498685836791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12832 , TFLOPS: 67.92950833484548, Tokens per sec: 168336.8047068329, Loss: 3.502561330795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12833 , TFLOPS: 69.545193265736, Tokens per sec: 172340.64994796764, Loss: 3.4892802238464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12834 , TFLOPS: 69.65024594600605, Tokens per sec: 172600.98206218486, Loss: 3.4789631366729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12835 , TFLOPS: 68.41084375651728, Tokens per sec: 169529.60690520957, Loss: 3.500603675842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12836 , TFLOPS: 67.86940087424755, Tokens per sec: 168187.85179808785, Loss: 3.504613161087036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12837 , TFLOPS: 70.58378490565703, Tokens per sec: 174914.39444200587, Loss: 3.5118257999420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12838 , TFLOPS: 69.64221536795066, Tokens per sec: 172581.0814051224, Loss: 3.500504970550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12839 , TFLOPS: 67.97429176535535, Tokens per sec: 168447.7829809385, Loss: 3.486001491546631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12840 , TFLOPS: 69.65052047615579, Tokens per sec: 172601.6623781379, Loss: 3.5136213302612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12841 , TFLOPS: 68.84101410950832, Tokens per sec: 170595.61642709764, Loss: 3.4996581077575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12842 , TFLOPS: 67.22917939008987, Tokens per sec: 166601.31243412656, Loss: 3.502171039581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12843 , TFLOPS: 70.60115155819972, Tokens per sec: 174957.43092010138, Loss: 3.512108325958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12844 , TFLOPS: 67.77286748034851, Tokens per sec: 167948.63141397483, Loss: 3.5165445804595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12845 , TFLOPS: 70.54279935890196, Tokens per sec: 174812.82774221615, Loss: 3.5097665786743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12846 , TFLOPS: 68.59128826557792, Tokens per sec: 169976.76827626541, Loss: 3.4775028228759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12847 , TFLOPS: 70.5747772460412, Tokens per sec: 174892.07246920225, Loss: 3.4678597450256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12848 , TFLOPS: 68.81595923441891, Tokens per sec: 170533.52768660377, Loss: 3.4869651794433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12849 , TFLOPS: 68.03319129731194, Tokens per sec: 168593.74250944602, Loss: 3.5167317390441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12850 , TFLOPS: 70.58291506084163, Tokens per sec: 174912.23887073202, Loss: 3.4724321365356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12851 , TFLOPS: 68.5997764271738, Tokens per sec: 169997.8028757482, Loss: 3.512045383453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12852 , TFLOPS: 67.93036749876653, Tokens per sec: 168338.933809674, Loss: 3.4847538471221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12853 , TFLOPS: 70.60157570628121, Tokens per sec: 174958.4820057711, Loss: 3.5027425289154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12854 , TFLOPS: 69.60318975555525, Tokens per sec: 172484.3716385805, Loss: 3.5015249252319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12855 , TFLOPS: 67.21984923422795, Tokens per sec: 166578.1912800992, Loss: 3.487832546234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12856 , TFLOPS: 68.78905984372308, Tokens per sec: 170466.86803324698, Loss: 3.5215768814086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12857 , TFLOPS: 70.57476278630868, Tokens per sec: 174892.0366363922, Loss: 3.4953365325927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12858 , TFLOPS: 68.35023081995308, Tokens per sec: 169379.4013713373, Loss: 3.492018461227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12859 , TFLOPS: 67.96297232833132, Tokens per sec: 168419.73216905404, Loss: 3.494046688079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12860 , TFLOPS: 70.5357960884358, Tokens per sec: 174795.47286652707, Loss: 3.487874746322632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12861 , TFLOPS: 68.81974232007039, Tokens per sec: 170542.90258958942, Loss: 3.506521224975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12862 , TFLOPS: 67.12562184889899, Tokens per sec: 166344.68543925064, Loss: 3.50246524810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12863 , TFLOPS: 70.00756569071585, Tokens per sec: 173486.45975159504, Loss: 3.5045814514160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12864 , TFLOPS: 69.49799717203797, Tokens per sec: 172223.69282870498, Loss: 3.4978840351104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12865 , TFLOPS: 69.87810046526761, Tokens per sec: 173165.6306611624, Loss: 3.4870171546936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12866 , TFLOPS: 68.60641904064111, Tokens per sec: 170014.2640036645, Loss: 3.4969491958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12867 , TFLOPS: 70.58608707115106, Tokens per sec: 174920.09946170455, Loss: 3.5095460414886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12868 , TFLOPS: 68.06834850564557, Tokens per sec: 168680.86594458297, Loss: 3.4861485958099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12869 , TFLOPS: 68.06021173284782, Tokens per sec: 168660.70212525062, Loss: 3.5127007961273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12870 , TFLOPS: 69.5538631071913, Tokens per sec: 172362.13477015577, Loss: 3.4951400756835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12871 , TFLOPS: 69.5814355123431, Tokens per sec: 172430.46222747306, Loss: 3.4796226024627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12872 , TFLOPS: 68.05415783379303, Tokens per sec: 168645.6998965008, Loss: 3.4890663623809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12873 , TFLOPS: 69.28592449285141, Tokens per sec: 171698.15336794648, Loss: 3.496579647064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12874 , TFLOPS: 69.56484547743956, Tokens per sec: 172389.35029343885, Loss: 3.503706693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12875 , TFLOPS: 68.9094048351246, Tokens per sec: 170765.09617903517, Loss: 3.488687753677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12876 , TFLOPS: 68.10065421424875, Tokens per sec: 168760.92304926921, Loss: 3.5009377002716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12877 , TFLOPS: 69.52087909756295, Tokens per sec: 172280.39676080807, Loss: 3.503326892852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12878 , TFLOPS: 69.54332452448169, Tokens per sec: 172336.01900128057, Loss: 3.4838390350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12879 , TFLOPS: 68.13203762802095, Tokens per sec: 168838.69460576517, Loss: 3.5164804458618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12880 , TFLOPS: 70.59738973345078, Tokens per sec: 174948.10870397434, Loss: 3.4809329509735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12881 , TFLOPS: 68.01334675635424, Tokens per sec: 168544.56555089616, Loss: 3.485485792160034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12882 , TFLOPS: 68.6666285635572, Tokens per sec: 170163.46983407706, Loss: 3.489865303039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12883 , TFLOPS: 69.24571758011808, Tokens per sec: 171598.51620903623, Loss: 3.492004871368408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12884 , TFLOPS: 69.59372893451102, Tokens per sec: 172460.9266818374, Loss: 3.489142894744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12885 , TFLOPS: 68.91549356250694, Tokens per sec: 170780.18471621728, Loss: 3.4880738258361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12886 , TFLOPS: 68.65214483230424, Tokens per sec: 170127.57755251753, Loss: 3.490694761276245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12887 , TFLOPS: 70.52920060342996, Tokens per sec: 174779.12852812995, Loss: 3.4919700622558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12888 , TFLOPS: 67.8364379864246, Tokens per sec: 168106.16613090076, Loss: 3.5324254035949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12889 , TFLOPS: 67.883602666821, Tokens per sec: 168223.0454045415, Loss: 3.50978684425354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12890 , TFLOPS: 70.56926260885483, Tokens per sec: 174878.406590767, Loss: 3.5031723976135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12891 , TFLOPS: 69.55839256078518, Tokens per sec: 172373.35925512188, Loss: 3.5106964111328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12892 , TFLOPS: 67.90432755985029, Tokens per sec: 168274.4039725035, Loss: 3.4967329502105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12893 , TFLOPS: 70.03123012697836, Tokens per sec: 173545.1028315064, Loss: 3.493701457977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12894 , TFLOPS: 69.68273221822706, Tokens per sec: 172681.48662340685, Loss: 3.4953315258026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12895 , TFLOPS: 68.69759199385408, Tokens per sec: 170240.2006252561, Loss: 3.482656955718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12896 , TFLOPS: 67.38395388754168, Tokens per sec: 166984.86068863052, Loss: 3.491239547729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12897 , TFLOPS: 70.6113424234077, Tokens per sec: 174982.68500670322, Loss: 3.4997005462646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12898 , TFLOPS: 69.54890278190499, Tokens per sec: 172349.8425376713, Loss: 3.5098557472229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12899 , TFLOPS: 67.93204426212924, Tokens per sec: 168343.08901399755, Loss: 3.5157759189605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12900 , TFLOPS: 70.53934237974929, Tokens per sec: 174804.2609670581, Loss: 3.490431547164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12901 , TFLOPS: 69.64446780815535, Tokens per sec: 172586.6632000764, Loss: 3.486475944519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12902 , TFLOPS: 68.26262485054905, Tokens per sec: 169162.3041285599, Loss: 3.5232467651367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12903 , TFLOPS: 68.77203105719526, Tokens per sec: 170424.66882435538, Loss: 3.4718480110168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12904 , TFLOPS: 70.58526849454434, Tokens per sec: 174918.07093869944, Loss: 3.4811558723449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12905 , TFLOPS: 68.69512546096904, Tokens per sec: 170234.088285056, Loss: 3.5201680660247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12906 , TFLOPS: 68.63433310674796, Tokens per sec: 170083.43813446548, Loss: 3.495119333267212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12907 , TFLOPS: 69.57596803672423, Tokens per sec: 172416.91322059734, Loss: 3.497077226638794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12908 , TFLOPS: 68.1944030500415, Tokens per sec: 168993.24299165045, Loss: 3.494194746017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12909 , TFLOPS: 67.97478631464526, Tokens per sec: 168449.008528558, Loss: 3.5222651958465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12910 , TFLOPS: 69.49485049081385, Tokens per sec: 172215.89500599605, Loss: 3.4837257862091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12911 , TFLOPS: 69.56348164305403, Tokens per sec: 172385.97056159278, Loss: 3.509633779525757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12912 , TFLOPS: 69.12145860772327, Tokens per sec: 171290.5887871861, Loss: 3.488722085952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12913 , TFLOPS: 68.99130799892828, Tokens per sec: 170968.06124131897, Loss: 3.482156991958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12914 , TFLOPS: 68.41390963723373, Tokens per sec: 169537.20449535956, Loss: 3.4909162521362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12915 , TFLOPS: 69.75855266436946, Tokens per sec: 172869.37804125936, Loss: 3.492887020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12916 , TFLOPS: 67.17985216891846, Tokens per sec: 166479.0741462218, Loss: 3.4999029636383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12917 , TFLOPS: 70.53191778524932, Tokens per sec: 174785.8620040006, Loss: 3.5123226642608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12918 , TFLOPS: 68.50898916735673, Tokens per sec: 169772.82204487926, Loss: 3.4969921112060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12919 , TFLOPS: 67.99912236854799, Tokens per sec: 168509.31595096661, Loss: 3.501915454864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12920 , TFLOPS: 70.59759402257626, Tokens per sec: 174948.61495493155, Loss: 3.4996676445007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12921 , TFLOPS: 69.67702750486673, Tokens per sec: 172667.3497152739, Loss: 3.519801139831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12922 , TFLOPS: 67.86991991259796, Tokens per sec: 168189.138032297, Loss: 3.471144437789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12923 , TFLOPS: 68.20218480072839, Tokens per sec: 169012.5270270832, Loss: 3.5129075050354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12924 , TFLOPS: 70.62460545343913, Tokens per sec: 175015.5522561645, Loss: 3.4960708618164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12925 , TFLOPS: 67.82721355049563, Tokens per sec: 168083.30696251325, Loss: 3.4856975078582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12926 , TFLOPS: 68.16176409968415, Tokens per sec: 168912.36007718774, Loss: 3.480107069015503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12927 , TFLOPS: 70.54727176647208, Tokens per sec: 174823.91086085123, Loss: 3.4780092239379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12928 , TFLOPS: 68.78895041066743, Tokens per sec: 170466.59684608004, Loss: 3.506176710128784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12929 , TFLOPS: 67.85824490678442, Tokens per sec: 168160.206081783, Loss: 3.4955196380615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12930 , TFLOPS: 70.03175936736318, Tokens per sec: 173546.41434748063, Loss: 3.493515968322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12931 , TFLOPS: 69.55451077280625, Tokens per sec: 172363.73975689584, Loss: 3.496704339981079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12932 , TFLOPS: 69.65309611283335, Tokens per sec: 172608.04508955474, Loss: 3.511049270629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12933 , TFLOPS: 68.18984492141642, Tokens per sec: 168981.9474467982, Loss: 3.4712159633636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12934 , TFLOPS: 70.56395178977584, Tokens per sec: 174865.24579605422, Loss: 3.505943536758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12935 , TFLOPS: 69.11944379728313, Tokens per sec: 171285.59586496573, Loss: 3.4935107231140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12936 , TFLOPS: 66.23070028915639, Tokens per sec: 164126.97123046036, Loss: 3.492594003677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12937 , TFLOPS: 70.59413661281563, Tokens per sec: 174940.0471126796, Loss: 3.483184814453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12938 , TFLOPS: 69.59831843166256, Tokens per sec: 172472.2999613538, Loss: 3.5062191486358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12939 , TFLOPS: 67.96345283229527, Tokens per sec: 168420.9229108086, Loss: 3.507533550262451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12940 , TFLOPS: 69.63051140226672, Tokens per sec: 172552.07768885934, Loss: 3.5062925815582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12941 , TFLOPS: 69.62729689938575, Tokens per sec: 172544.11179661358, Loss: 3.4893856048583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12942 , TFLOPS: 68.8891578539033, Tokens per sec: 170714.9219291797, Loss: 3.4925875663757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12943 , TFLOPS: 68.14700817951756, Tokens per sec: 168875.79328151615, Loss: 3.504213571548462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12944 , TFLOPS: 69.55965751187904, Tokens per sec: 172376.4939432783, Loss: 3.5001285076141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12945 , TFLOPS: 68.80595059349339, Tokens per sec: 170508.7251718475, Loss: 3.502828359603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12946 , TFLOPS: 68.53538311114336, Tokens per sec: 169838.22914511545, Loss: 3.5042107105255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12947 , TFLOPS: 69.72711050562451, Tokens per sec: 172791.46090824963, Loss: 3.4933743476867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12948 , TFLOPS: 68.71310291960854, Tokens per sec: 170278.63840794523, Loss: 3.470879554748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12949 , TFLOPS: 69.5950350307098, Tokens per sec: 172464.16333209636, Loss: 3.483072519302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12950 , TFLOPS: 69.22584827963371, Tokens per sec: 171549.27789365183, Loss: 3.5014045238494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12951 , TFLOPS: 69.53449184771053, Tokens per sec: 172314.13065524198, Loss: 3.483795166015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12952 , TFLOPS: 69.85340836760713, Tokens per sec: 173104.44092310063, Loss: 3.507007598876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12953 , TFLOPS: 67.90922097111486, Tokens per sec: 168286.53038467086, Loss: 3.512103319168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12954 , TFLOPS: 70.57762514358308, Tokens per sec: 174899.12987303283, Loss: 3.4963088035583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12955 , TFLOPS: 67.84013199440015, Tokens per sec: 168115.32028959272, Loss: 3.4885573387145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12956 , TFLOPS: 67.22633822760558, Tokens per sec: 166594.27172051062, Loss: 3.507244348526001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12957 , TFLOPS: 70.54721491283682, Tokens per sec: 174823.76997128016, Loss: 3.50107741355896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12958 , TFLOPS: 68.5704996746578, Tokens per sec: 169925.25185791304, Loss: 3.4613382816314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12959 , TFLOPS: 68.91445725192898, Tokens per sec: 170777.61662444606, Loss: 3.4980196952819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12960 , TFLOPS: 69.98917913416456, Tokens per sec: 173440.89583901456, Loss: 3.502774715423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12961 , TFLOPS: 69.65932222992272, Tokens per sec: 172623.4740935656, Loss: 3.50046443939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12962 , TFLOPS: 68.67758936280536, Tokens per sec: 170190.6319020457, Loss: 3.5132274627685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12963 , TFLOPS: 67.1445230040058, Tokens per sec: 166391.52458373943, Loss: 3.5042474269866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12964 , TFLOPS: 70.54980910613006, Tokens per sec: 174830.19866803478, Loss: 3.5125863552093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12965 , TFLOPS: 68.82285628756927, Tokens per sec: 170550.61934408345, Loss: 3.4915390014648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12966 , TFLOPS: 67.7345433714892, Tokens per sec: 167853.66005047236, Loss: 3.5011932849884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12967 , TFLOPS: 70.62401962034656, Tokens per sec: 175014.10049722626, Loss: 3.4925765991210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12968 , TFLOPS: 69.49086158740107, Tokens per sec: 172206.0100639252, Loss: 3.5073845386505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12969 , TFLOPS: 68.80540157182377, Tokens per sec: 170507.36463567044, Loss: 3.5045151710510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12970 , TFLOPS: 68.83847243834447, Tokens per sec: 170589.31788596543, Loss: 3.512665271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12971 , TFLOPS: 70.62135891447203, Tokens per sec: 175007.50697496822, Loss: 3.4819939136505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12972 , TFLOPS: 69.50395526162339, Tokens per sec: 172238.4576310354, Loss: 3.484581470489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12973 , TFLOPS: 66.97997854277502, Tokens per sec: 165983.76528274082, Loss: 3.4981913566589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12974 , TFLOPS: 70.52200236075687, Tokens per sec: 174761.29048983415, Loss: 3.510810136795044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12975 , TFLOPS: 68.77651887122879, Tokens per sec: 170435.79012190396, Loss: 3.4931588172912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12976 , TFLOPS: 67.23288911853416, Tokens per sec: 166610.5055498718, Loss: 3.511323928833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12977 , TFLOPS: 69.7056860649736, Tokens per sec: 172738.36878421975, Loss: 3.5044360160827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12978 , TFLOPS: 69.56228110679189, Tokens per sec: 172382.99550048518, Loss: 3.4935290813446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12979 , TFLOPS: 69.72950840086607, Tokens per sec: 172797.4031568082, Loss: 3.5142271518707275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12980 , TFLOPS: 68.54349065008005, Tokens per sec: 169858.32051971968, Loss: 3.4974920749664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12981 , TFLOPS: 68.46230362106189, Tokens per sec: 169657.13011832314, Loss: 3.5044338703155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12982 , TFLOPS: 69.67851196471501, Tokens per sec: 172671.0283703047, Loss: 3.479825258255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12983 , TFLOPS: 67.19565533620906, Tokens per sec: 166518.23613563052, Loss: 3.492838144302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12984 , TFLOPS: 69.62725577604365, Tokens per sec: 172544.00988844174, Loss: 3.493530511856079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12985 , TFLOPS: 68.80939265461616, Tokens per sec: 170517.25497848407, Loss: 3.5112781524658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12986 , TFLOPS: 68.6563227151198, Tokens per sec: 170137.9308063662, Loss: 3.4956603050231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12987 , TFLOPS: 70.54486901666559, Tokens per sec: 174817.95658214603, Loss: 3.5014243125915527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12988 , TFLOPS: 68.02302489791025, Tokens per sec: 168568.54905180726, Loss: 3.491920232772827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12989 , TFLOPS: 68.71634145376765, Tokens per sec: 170286.6638523455, Loss: 3.5141067504882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12990 , TFLOPS: 68.77579594000906, Tokens per sec: 170433.99861870403, Loss: 3.520904064178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12991 , TFLOPS: 70.60355687248959, Tokens per sec: 174963.39155387893, Loss: 3.4878146648406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12992 , TFLOPS: 68.67100924972917, Tokens per sec: 170174.32565698362, Loss: 3.502596855163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12993 , TFLOPS: 68.03629700346744, Tokens per sec: 168601.4387914211, Loss: 3.4888968467712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12994 , TFLOPS: 70.56376243272611, Tokens per sec: 174864.7765484256, Loss: 3.4783084392547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12995 , TFLOPS: 68.39889568017189, Tokens per sec: 169499.9982558364, Loss: 3.4719529151916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12996 , TFLOPS: 66.91710952933212, Tokens per sec: 165827.96894183574, Loss: 3.4811980724334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12997 , TFLOPS: 70.0217615939094, Tokens per sec: 173521.6387635165, Loss: 3.4765350818634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12998 , TFLOPS: 69.5682717153062, Tokens per sec: 172397.84088830368, Loss: 3.484623908996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12999 , TFLOPS: 69.62138302312998, Tokens per sec: 172529.45655978454, Loss: 3.4936022758483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13000 , TFLOPS: 68.3716917354849, Tokens per sec: 169432.5838841402, Loss: 3.500452756881714 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/optimizer.pt +[rank0]:[2024-08-26 00:18:21,029] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006365741995978169, 'preprocessing_with_comm': 0.0013947590050520375, 'state_converting': 0.9766391299926909, : 0.9857893750013318}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013000, took 4.25s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011000 + eval ppl=24.1252384185791, eval loss=3.183258533477783 +------------------------------------------------------------------ +iteration: 13001 , TFLOPS: 65.21785569851505, Tokens per sec: 161617.0307608088, Loss: 3.5029797554016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13002 , TFLOPS: 69.6635765864554, Tokens per sec: 172634.01685770965, Loss: 3.4789295196533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13003 , TFLOPS: 65.78008616660145, Tokens per sec: 163010.29979552582, Loss: 3.499561309814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13004 , TFLOPS: 70.5457886465784, Tokens per sec: 174820.2355263756, Loss: 3.510164976119995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13005 , TFLOPS: 68.8074604172271, Tokens per sec: 170512.46668138047, Loss: 3.491217613220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13006 , TFLOPS: 69.3528291634616, Tokens per sec: 171863.95051187646, Loss: 3.4978301525115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13007 , TFLOPS: 69.1561299742977, Tokens per sec: 171376.50825292387, Loss: 3.487687110900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13008 , TFLOPS: 69.51273280478911, Tokens per sec: 172260.20934992624, Loss: 3.503404140472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13009 , TFLOPS: 68.58256958186684, Tokens per sec: 169955.16241758707, Loss: 3.5170698165893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13010 , TFLOPS: 66.33149816226468, Tokens per sec: 164376.7594638251, Loss: 3.4824962615966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13011 , TFLOPS: 70.52992206154381, Tokens per sec: 174780.91638081122, Loss: 3.487251043319702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13012 , TFLOPS: 69.03370443561526, Tokens per sec: 171073.12428178237, Loss: 3.527900218963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13013 , TFLOPS: 67.24437984008327, Tokens per sec: 166638.98082367802, Loss: 3.4941763877868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13014 , TFLOPS: 69.51636141025485, Tokens per sec: 172269.20143399425, Loss: 3.4986066818237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13015 , TFLOPS: 69.62809642924286, Tokens per sec: 172546.09311967497, Loss: 3.5037407875061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13016 , TFLOPS: 68.18147723920855, Tokens per sec: 168961.2114085108, Loss: 3.4851391315460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13017 , TFLOPS: 68.87312453221361, Tokens per sec: 170675.1895918164, Loss: 3.504345417022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13018 , TFLOPS: 68.52730745209018, Tokens per sec: 169818.21677237435, Loss: 3.4931514263153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13019 , TFLOPS: 68.50012549823217, Tokens per sec: 169750.8568963748, Loss: 3.500697135925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13020 , TFLOPS: 68.06035433803758, Tokens per sec: 168661.0555166201, Loss: 3.508960723876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13021 , TFLOPS: 68.03357941637421, Tokens per sec: 168594.70431123715, Loss: 3.507112741470337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13022 , TFLOPS: 69.70258820850668, Tokens per sec: 172730.69195463764, Loss: 3.489718198776245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13023 , TFLOPS: 67.40286227410783, Tokens per sec: 167031.7177534722, Loss: 3.503662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13024 , TFLOPS: 70.56328047693866, Tokens per sec: 174863.58220889248, Loss: 3.4717350006103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13025 , TFLOPS: 68.46823540548095, Tokens per sec: 169671.8297335536, Loss: 3.49206280708313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13026 , TFLOPS: 68.5459919657877, Tokens per sec: 169864.51905558593, Loss: 3.4767842292785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13027 , TFLOPS: 69.10637531606348, Tokens per sec: 171253.2107288914, Loss: 3.51874041557312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13028 , TFLOPS: 68.60208190778958, Tokens per sec: 170003.5161106838, Loss: 3.4872865676879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13029 , TFLOPS: 68.93285549610289, Tokens per sec: 170823.20950024293, Loss: 3.487156867980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13030 , TFLOPS: 67.98889195199239, Tokens per sec: 168483.96385176844, Loss: 3.4922001361846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13031 , TFLOPS: 70.62115242485754, Tokens per sec: 175006.99527095683, Loss: 3.479302406311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13032 , TFLOPS: 67.05108965065136, Tokens per sec: 166159.98644159362, Loss: 3.4934005737304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13033 , TFLOPS: 67.76872525693844, Tokens per sec: 167938.3665280611, Loss: 3.4735167026519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13034 , TFLOPS: 70.5781678118029, Tokens per sec: 174900.4746646614, Loss: 3.472637176513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13035 , TFLOPS: 69.53855172407843, Tokens per sec: 172324.19147611398, Loss: 3.4807534217834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13036 , TFLOPS: 66.89418035812511, Tokens per sec: 165771.14792972806, Loss: 3.49784255027771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13037 , TFLOPS: 68.88558068877647, Tokens per sec: 170706.05731993812, Loss: 3.4956576824188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13038 , TFLOPS: 69.52632680654452, Tokens per sec: 172293.89678377908, Loss: 3.4810659885406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13039 , TFLOPS: 69.6857057599072, Tokens per sec: 172688.85538725284, Loss: 3.512521743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13040 , TFLOPS: 66.36162840112577, Tokens per sec: 164451.42551484267, Loss: 3.4906678199768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13041 , TFLOPS: 69.58659223683927, Tokens per sec: 172443.24115883466, Loss: 3.502100944519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13042 , TFLOPS: 69.74950227600678, Tokens per sec: 172846.95018191406, Loss: 3.491347074508667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13043 , TFLOPS: 67.15618811532258, Tokens per sec: 166420.4320146005, Loss: 3.505056142807007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13044 , TFLOPS: 70.0511351811433, Tokens per sec: 173594.42974844982, Loss: 3.5102970600128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13045 , TFLOPS: 68.73462368396625, Tokens per sec: 170331.96923273153, Loss: 3.513397455215454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13046 , TFLOPS: 69.59073511854953, Tokens per sec: 172453.5076760888, Loss: 3.4994287490844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13047 , TFLOPS: 67.07635219952017, Tokens per sec: 166222.58982059002, Loss: 3.5157065391540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13048 , TFLOPS: 69.54696504011038, Tokens per sec: 172345.0406000444, Loss: 3.503908157348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13049 , TFLOPS: 68.92814026504925, Tokens per sec: 170811.52463826642, Loss: 3.488409996032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13050 , TFLOPS: 68.05326410588336, Tokens per sec: 168643.4851402884, Loss: 3.470951557159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13051 , TFLOPS: 69.35854030870048, Tokens per sec: 171878.103358913, Loss: 3.5108602046966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13052 , TFLOPS: 68.4548444205598, Tokens per sec: 169638.64539778017, Loss: 3.494720935821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13053 , TFLOPS: 67.16399863922112, Tokens per sec: 166439.78735322133, Loss: 3.524477958679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13054 , TFLOPS: 70.27105885780972, Tokens per sec: 174139.4248458222, Loss: 3.475433826446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13055 , TFLOPS: 67.91485874213306, Tokens per sec: 168300.5014023057, Loss: 3.5082156658172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13056 , TFLOPS: 68.17954456399114, Tokens per sec: 168956.42202642033, Loss: 3.506404399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13057 , TFLOPS: 67.39363557498262, Tokens per sec: 167008.85297663513, Loss: 3.4846792221069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13058 , TFLOPS: 67.94081995729371, Tokens per sec: 168364.83615333773, Loss: 3.48252534866333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13059 , TFLOPS: 69.3169423250816, Tokens per sec: 171775.0189731092, Loss: 3.4874629974365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13060 , TFLOPS: 65.95212663262862, Tokens per sec: 163436.63502216298, Loss: 3.485079050064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13061 , TFLOPS: 70.22051712127185, Tokens per sec: 174014.17685504947, Loss: 3.491292953491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13062 , TFLOPS: 67.74346336574969, Tokens per sec: 167875.76477886835, Loss: 3.4914705753326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13063 , TFLOPS: 68.76331689883496, Tokens per sec: 170403.0741799947, Loss: 3.475944757461548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13064 , TFLOPS: 69.04913311717526, Tokens per sec: 171111.3582542976, Loss: 3.5171658992767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13065 , TFLOPS: 68.40930840888724, Tokens per sec: 169525.80214464982, Loss: 3.5071537494659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13066 , TFLOPS: 69.00789077767627, Tokens per sec: 171009.15519380037, Loss: 3.4991965293884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13067 , TFLOPS: 67.86548321089775, Tokens per sec: 168178.1433967408, Loss: 3.4825515747070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13068 , TFLOPS: 69.51462585996887, Tokens per sec: 172264.90054919897, Loss: 3.4812188148498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13069 , TFLOPS: 68.81815428570309, Tokens per sec: 170538.96726549082, Loss: 3.49804949760437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13070 , TFLOPS: 67.1487014582044, Tokens per sec: 166401.8792535383, Loss: 3.5237250328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13071 , TFLOPS: 70.60251226542742, Tokens per sec: 174960.80290250588, Loss: 3.4816908836364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13072 , TFLOPS: 68.52311684928785, Tokens per sec: 169807.83199699718, Loss: 3.4811153411865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13073 , TFLOPS: 66.96174405267047, Tokens per sec: 165938.57820756684, Loss: 3.483860731124878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13074 , TFLOPS: 69.30783513695101, Tokens per sec: 171752.45035768204, Loss: 3.5061938762664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13075 , TFLOPS: 69.06704592872839, Tokens per sec: 171155.74817458537, Loss: 3.5072453022003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13076 , TFLOPS: 67.99193783622347, Tokens per sec: 168491.5118884249, Loss: 3.491281270980835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13077 , TFLOPS: 65.56824199916747, Tokens per sec: 162485.32661206287, Loss: 3.488842487335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13078 , TFLOPS: 70.04505968197428, Tokens per sec: 173579.37399223755, Loss: 3.488152027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13079 , TFLOPS: 68.43760650477418, Tokens per sec: 169595.92794355025, Loss: 3.5077054500579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13080 , TFLOPS: 65.11179034756798, Tokens per sec: 161354.189137099, Loss: 3.49664044380188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13081 , TFLOPS: 70.41024406842102, Tokens per sec: 174484.34113023372, Loss: 3.487205743789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13082 , TFLOPS: 69.5457812983911, Tokens per sec: 172342.1071576641, Loss: 3.484187602996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13083 , TFLOPS: 68.01359389926272, Tokens per sec: 168545.17799824843, Loss: 3.497457504272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13084 , TFLOPS: 64.88909640887084, Tokens per sec: 160802.329025245, Loss: 3.479726552963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13085 , TFLOPS: 68.91734184768438, Tokens per sec: 170784.76497049243, Loss: 3.4873034954071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13086 , TFLOPS: 68.34513026984209, Tokens per sec: 169366.76164629028, Loss: 3.5010523796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13087 , TFLOPS: 66.93335421892957, Tokens per sec: 165868.22507215853, Loss: 3.476769208908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13088 , TFLOPS: 67.75771927081398, Tokens per sec: 167911.09248203525, Loss: 3.51601505279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13089 , TFLOPS: 69.22014920904647, Tokens per sec: 171535.15496893218, Loss: 3.4973604679107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13090 , TFLOPS: 66.0995231648439, Tokens per sec: 163801.89986606088, Loss: 3.494710683822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13091 , TFLOPS: 70.10538497856425, Tokens per sec: 173728.86672256753, Loss: 3.475933313369751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13092 , TFLOPS: 67.14268550394203, Tokens per sec: 166386.97105616407, Loss: 3.4843552112579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13093 , TFLOPS: 69.6180000336833, Tokens per sec: 172521.0731392686, Loss: 3.4846324920654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13094 , TFLOPS: 68.10529811039187, Tokens per sec: 168772.43113547927, Loss: 3.503117799758911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13095 , TFLOPS: 68.87466745554119, Tokens per sec: 170679.01312578918, Loss: 3.4983575344085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13096 , TFLOPS: 68.10398285342212, Tokens per sec: 168769.17178382073, Loss: 3.474261999130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13097 , TFLOPS: 66.9216647926192, Tokens per sec: 165839.2573860648, Loss: 3.5009658336639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13098 , TFLOPS: 70.17698685318877, Tokens per sec: 173906.30405548468, Loss: 3.4887006282806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13099 , TFLOPS: 66.85847681386866, Tokens per sec: 165682.67061399049, Loss: 3.48632550239563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13100 , TFLOPS: 65.56501768710028, Tokens per sec: 162477.33641157308, Loss: 3.496778964996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13101 , TFLOPS: 70.2134879644044, Tokens per sec: 173996.75783000633, Loss: 3.485118865966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13102 , TFLOPS: 68.07405222272942, Tokens per sec: 168695.00038383304, Loss: 3.4967169761657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13103 , TFLOPS: 67.7897551806116, Tokens per sec: 167990.48099556053, Loss: 3.4917144775390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13104 , TFLOPS: 68.10467984197729, Tokens per sec: 168770.89899824004, Loss: 3.5323023796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13105 , TFLOPS: 69.00323676481504, Tokens per sec: 170997.62203724842, Loss: 3.4956588745117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13106 , TFLOPS: 68.00887361907242, Tokens per sec: 168533.48062395505, Loss: 3.510676860809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13107 , TFLOPS: 66.55332546725874, Tokens per sec: 164926.47196189532, Loss: 3.491917848587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13108 , TFLOPS: 70.18858153155826, Tokens per sec: 173935.03694574352, Loss: 3.4935226440429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13109 , TFLOPS: 68.03267592195368, Tokens per sec: 168592.46535253496, Loss: 3.523892402648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13110 , TFLOPS: 65.91073058612962, Tokens per sec: 163334.05105878747, Loss: 3.5120861530303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13111 , TFLOPS: 69.375945604984, Tokens per sec: 171921.23560045523, Loss: 3.4856057167053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13112 , TFLOPS: 69.05270472832383, Tokens per sec: 171120.20910016925, Loss: 3.5070109367370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13113 , TFLOPS: 69.13620838453494, Tokens per sec: 171327.14035894745, Loss: 3.502164363861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13114 , TFLOPS: 68.25926491548906, Tokens per sec: 169153.97783935384, Loss: 3.4988975524902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13115 , TFLOPS: 69.05369364731844, Tokens per sec: 171122.65975037628, Loss: 3.494882583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13116 , TFLOPS: 68.16775970257143, Tokens per sec: 168927.2178416126, Loss: 3.493171215057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13117 , TFLOPS: 67.06982464757748, Tokens per sec: 166206.41382780677, Loss: 3.533263921737671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13118 , TFLOPS: 70.3256899917743, Tokens per sec: 174274.80681390263, Loss: 3.4909703731536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13119 , TFLOPS: 68.22593502938012, Tokens per sec: 169071.38270998586, Loss: 3.5049328804016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13120 , TFLOPS: 66.05704178283395, Tokens per sec: 163696.62632172965, Loss: 3.4853878021240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13121 , TFLOPS: 69.06325993945913, Tokens per sec: 171146.36607611502, Loss: 3.49796199798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13122 , TFLOPS: 68.94758583155387, Tokens per sec: 170859.71289417084, Loss: 3.487844467163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13123 , TFLOPS: 68.69326731956261, Tokens per sec: 170229.4836059597, Loss: 3.4900097846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13124 , TFLOPS: 68.0700346958561, Tokens per sec: 168685.04450967384, Loss: 3.527204990386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13125 , TFLOPS: 68.03243511827216, Tokens per sec: 168591.86861448543, Loss: 3.483611822128296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13126 , TFLOPS: 69.36488621010103, Tokens per sec: 171893.8292016423, Loss: 3.5174477100372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13127 , TFLOPS: 66.15065749204157, Tokens per sec: 163928.61636176784, Loss: 3.4945459365844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13128 , TFLOPS: 70.29676402110155, Tokens per sec: 174203.12507211685, Loss: 3.5030105113983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13129 , TFLOPS: 66.64130959334688, Tokens per sec: 165144.50631859325, Loss: 3.49379563331604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13130 , TFLOPS: 67.97229705965228, Tokens per sec: 168442.8398804713, Loss: 3.485743999481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13131 , TFLOPS: 67.859098902908, Tokens per sec: 168162.32237825263, Loss: 3.499814510345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13132 , TFLOPS: 67.94317147984854, Tokens per sec: 168370.66348527017, Loss: 3.509226083755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13133 , TFLOPS: 68.0025057158497, Tokens per sec: 168517.70025828673, Loss: 3.464085817337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13134 , TFLOPS: 68.37633923813982, Tokens per sec: 169444.10090768372, Loss: 3.4881765842437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13135 , TFLOPS: 69.59129134766032, Tokens per sec: 172454.88607309898, Loss: 3.4835550785064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13136 , TFLOPS: 67.8939490135069, Tokens per sec: 168248.68479137332, Loss: 3.5010082721710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13137 , TFLOPS: 67.79788221124201, Tokens per sec: 168010.62067272904, Loss: 3.5258402824401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13138 , TFLOPS: 70.49308435703765, Tokens per sec: 174689.6284910368, Loss: 3.494171619415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13139 , TFLOPS: 67.80589364326309, Tokens per sec: 168030.47388380935, Loss: 3.506865978240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13140 , TFLOPS: 67.93822244479341, Tokens per sec: 168358.39923122234, Loss: 3.4910433292388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13141 , TFLOPS: 69.88931865032126, Tokens per sec: 173193.43055951042, Loss: 3.49961256980896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13142 , TFLOPS: 69.05609442033808, Tokens per sec: 171128.60913038594, Loss: 3.5100224018096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13143 , TFLOPS: 67.03420387515857, Tokens per sec: 166118.14162980072, Loss: 3.497981548309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13144 , TFLOPS: 67.0774067461831, Tokens per sec: 166225.20310338866, Loss: 3.496389389038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13145 , TFLOPS: 70.17568605820321, Tokens per sec: 173903.0805422161, Loss: 3.4842891693115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13146 , TFLOPS: 68.08446046546337, Tokens per sec: 168720.79315589034, Loss: 3.503095865249634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13147 , TFLOPS: 64.90151345154368, Tokens per sec: 160833.09982483843, Loss: 3.48521089553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13148 , TFLOPS: 70.25915978420447, Tokens per sec: 174109.9376306372, Loss: 3.4865238666534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13149 , TFLOPS: 68.99678100740734, Tokens per sec: 170981.62395923174, Loss: 3.4928674697875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13150 , TFLOPS: 67.8909612030663, Tokens per sec: 168241.28066796702, Loss: 3.5240302085876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13151 , TFLOPS: 67.89938038534261, Tokens per sec: 168262.14432909718, Loss: 3.4762167930603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13152 , TFLOPS: 68.82912911058712, Tokens per sec: 170566.1640905302, Loss: 3.484227418899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13153 , TFLOPS: 67.86529646338617, Tokens per sec: 168177.68061583518, Loss: 3.486314296722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13154 , TFLOPS: 66.65789046136604, Tokens per sec: 165185.59553607815, Loss: 3.5096359252929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13155 , TFLOPS: 68.93881723745297, Tokens per sec: 170837.98335204847, Loss: 3.493912696838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13156 , TFLOPS: 69.42139523157965, Tokens per sec: 172033.8647818489, Loss: 3.490326166152954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13157 , TFLOPS: 66.38287978669976, Tokens per sec: 164504.0887893284, Loss: 3.480611562728882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13158 , TFLOPS: 68.92599712856804, Tokens per sec: 170806.21370998002, Loss: 3.5074918270111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13159 , TFLOPS: 67.19887086214167, Tokens per sec: 166526.20456311095, Loss: 3.4741249084472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13160 , TFLOPS: 68.44622453224865, Tokens per sec: 169617.2843649861, Loss: 3.515063524246216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13161 , TFLOPS: 68.91968763567802, Tokens per sec: 170790.5780915505, Loss: 3.5144612789154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13162 , TFLOPS: 68.11455484414664, Tokens per sec: 168795.37034143697, Loss: 3.4795656204223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13163 , TFLOPS: 68.34735997338503, Tokens per sec: 169372.28709729182, Loss: 3.4936866760253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13164 , TFLOPS: 66.98600549603427, Tokens per sec: 165998.70073684095, Loss: 3.4915878772735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13165 , TFLOPS: 70.20712825579517, Tokens per sec: 173980.99777149296, Loss: 3.5009257793426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13166 , TFLOPS: 67.26962028064693, Tokens per sec: 166701.52941585882, Loss: 3.5094075202941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13167 , TFLOPS: 66.1187815074821, Tokens per sec: 163849.6241605999, Loss: 3.505000352859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13168 , TFLOPS: 70.29465682521973, Tokens per sec: 174197.90320859491, Loss: 3.4787683486938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13169 , TFLOPS: 67.8394644454509, Tokens per sec: 168113.6660297599, Loss: 3.489626884460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13170 , TFLOPS: 68.14686137783566, Tokens per sec: 168875.42949077688, Loss: 3.5021042823791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13171 , TFLOPS: 67.79369627702634, Tokens per sec: 168000.2474666239, Loss: 3.514425754547119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13172 , TFLOPS: 69.41680015828925, Tokens per sec: 172022.4776840462, Loss: 3.5121610164642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13173 , TFLOPS: 68.7197848928172, Tokens per sec: 170295.19707363637, Loss: 3.4945218563079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13174 , TFLOPS: 67.45524397820918, Tokens per sec: 167161.5253865564, Loss: 3.5169405937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13175 , TFLOPS: 68.95586553983445, Tokens per sec: 170880.2309233753, Loss: 3.473998546600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13176 , TFLOPS: 68.06987293623602, Tokens per sec: 168684.643651516, Loss: 3.4703001976013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13177 , TFLOPS: 67.05353292521663, Tokens per sec: 166166.04114511525, Loss: 3.503434658050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13178 , TFLOPS: 69.38658655231676, Tokens per sec: 171947.60504014292, Loss: 3.4884822368621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13179 , TFLOPS: 67.92919863153065, Tokens per sec: 168336.03722790215, Loss: 3.494131088256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13180 , TFLOPS: 68.2732228947231, Tokens per sec: 169188.56724949414, Loss: 3.47291898727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13181 , TFLOPS: 67.52004270015769, Tokens per sec: 167322.10375771328, Loss: 3.4979052543640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13182 , TFLOPS: 69.07713449941104, Tokens per sec: 171180.74876408477, Loss: 3.4896271228790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13183 , TFLOPS: 68.86570180496385, Tokens per sec: 170656.79525601064, Loss: 3.4915146827697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13184 , TFLOPS: 67.67223196515434, Tokens per sec: 167699.2455214065, Loss: 3.476945400238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13185 , TFLOPS: 70.59055851292676, Tokens per sec: 174931.18018699472, Loss: 3.5132691860198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13186 , TFLOPS: 68.58221728386073, Tokens per sec: 169954.2893843191, Loss: 3.4979214668273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13187 , TFLOPS: 66.9965484585219, Tokens per sec: 166024.8273592885, Loss: 3.5043587684631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13188 , TFLOPS: 69.5929054946025, Tokens per sec: 172458.88610703472, Loss: 3.481682300567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13189 , TFLOPS: 69.14077730038322, Tokens per sec: 171338.46263572713, Loss: 3.4926917552948 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13190 , TFLOPS: 68.77742588930855, Tokens per sec: 170438.0378126219, Loss: 3.490717887878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13191 , TFLOPS: 65.30535522493766, Tokens per sec: 161833.86422615254, Loss: 3.4812543392181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13192 , TFLOPS: 68.88083245600284, Tokens per sec: 170694.29067606403, Loss: 3.493506908416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13193 , TFLOPS: 69.01962303714723, Tokens per sec: 171038.22902518447, Loss: 3.4963572025299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13194 , TFLOPS: 67.28220054626178, Tokens per sec: 166732.70469988306, Loss: 3.481595039367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13195 , TFLOPS: 69.50950881604585, Tokens per sec: 172252.21995066878, Loss: 3.480024814605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13196 , TFLOPS: 68.89028151997839, Tokens per sec: 170717.70649749576, Loss: 3.507049083709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13197 , TFLOPS: 68.29018576471937, Tokens per sec: 169230.60310995928, Loss: 3.487884998321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13198 , TFLOPS: 68.76297904049193, Tokens per sec: 170402.23692980254, Loss: 3.5205295085906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13199 , TFLOPS: 67.76928115080142, Tokens per sec: 167939.74409429033, Loss: 3.501002073287964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13200 , TFLOPS: 68.39942256623574, Tokens per sec: 169501.3039375439, Loss: 3.4635207653045654 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/optimizer.pt +[rank0]:[2024-08-26 00:44:17,192] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.009129573008976877, 'preprocessing_with_comm': 0.0017666309868218377, 'state_converting': 1.0999514409922995, : 1.1128944869997213}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013200, took 5.72s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011200 + eval ppl=23.777938842773438, eval loss=3.1687581539154053 +------------------------------------------------------------------ +iteration: 13201 , TFLOPS: 63.244514488234984, Tokens per sec: 156726.87386025515, Loss: 3.5150845050811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13202 , TFLOPS: 68.95085046694206, Tokens per sec: 170867.80302029205, Loss: 3.484079360961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13203 , TFLOPS: 68.5209045739328, Tokens per sec: 169802.34973496513, Loss: 3.502393960952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13204 , TFLOPS: 68.04572836147517, Tokens per sec: 168624.81073551413, Loss: 3.462010145187378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13205 , TFLOPS: 70.18204481616218, Tokens per sec: 173918.8382449134, Loss: 3.5005154609680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13206 , TFLOPS: 67.1328678607802, Tokens per sec: 166362.6418549064, Loss: 3.4845099449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13207 , TFLOPS: 66.08855777854477, Tokens per sec: 163774.7264308746, Loss: 3.491591215133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13208 , TFLOPS: 70.14442386397336, Tokens per sec: 173825.60938109952, Loss: 3.4940710067749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13209 , TFLOPS: 67.70463697750859, Tokens per sec: 167779.54871172714, Loss: 3.4871766567230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13210 , TFLOPS: 66.77637676587194, Tokens per sec: 165479.21765098773, Loss: 3.4934511184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13211 , TFLOPS: 69.01266358823308, Tokens per sec: 171020.98274412868, Loss: 3.4808290004730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13212 , TFLOPS: 69.09651681236184, Tokens per sec: 171228.7802707149, Loss: 3.4839766025543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13213 , TFLOPS: 67.91809617737707, Tokens per sec: 168308.52412347327, Loss: 3.4798858165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13214 , TFLOPS: 66.4170213365748, Tokens per sec: 164588.69531092097, Loss: 3.487555980682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13215 , TFLOPS: 70.26634943560857, Tokens per sec: 174127.75437882208, Loss: 3.5028042793273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13216 , TFLOPS: 68.15812036727975, Tokens per sec: 168903.3305068994, Loss: 3.4854037761688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13217 , TFLOPS: 66.68286302874618, Tokens per sec: 165247.48030900155, Loss: 3.4902169704437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13218 , TFLOPS: 69.55190767385255, Tokens per sec: 172357.28899093356, Loss: 3.5071845054626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13219 , TFLOPS: 69.5535548993302, Tokens per sec: 172361.370997124, Loss: 3.502924919128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13220 , TFLOPS: 68.99906917009643, Tokens per sec: 170987.29427843724, Loss: 3.4885873794555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13221 , TFLOPS: 67.97402313374197, Tokens per sec: 168447.11728220832, Loss: 3.4850289821624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13222 , TFLOPS: 69.46329004361618, Tokens per sec: 172137.68474116997, Loss: 3.4872756004333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13223 , TFLOPS: 69.2876687268836, Tokens per sec: 171702.4757720215, Loss: 3.492363929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13224 , TFLOPS: 67.3703077940376, Tokens per sec: 166951.04416568612, Loss: 3.492755174636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13225 , TFLOPS: 67.97546797118845, Tokens per sec: 168450.6977485627, Loss: 3.4929516315460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13226 , TFLOPS: 68.77200465055896, Tokens per sec: 170424.60338580166, Loss: 3.5088255405426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13227 , TFLOPS: 67.00590532702455, Tokens per sec: 166048.01471018224, Loss: 3.477715253829956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13228 , TFLOPS: 68.3712187035468, Tokens per sec: 169431.41165888874, Loss: 3.495346784591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13229 , TFLOPS: 68.7174162316356, Tokens per sec: 170289.32727029882, Loss: 3.488259792327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13230 , TFLOPS: 68.5289124633089, Tokens per sec: 169822.19416698095, Loss: 3.504624605178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13231 , TFLOPS: 68.1631677938596, Tokens per sec: 168915.8385859842, Loss: 3.497162342071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13232 , TFLOPS: 67.71430070592709, Tokens per sec: 167803.49649529575, Loss: 3.5024335384368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13233 , TFLOPS: 69.23195547934125, Tokens per sec: 171564.41220729073, Loss: 3.5002639293670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13234 , TFLOPS: 65.89675912748389, Tokens per sec: 163299.42824518017, Loss: 3.494783878326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13235 , TFLOPS: 70.22568520350053, Tokens per sec: 174026.98393212337, Loss: 3.4923255443573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13236 , TFLOPS: 66.85289110906174, Tokens per sec: 165668.82862216278, Loss: 3.5238797664642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13237 , TFLOPS: 66.98705754285088, Tokens per sec: 166001.30782474505, Loss: 3.4948983192443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13238 , TFLOPS: 69.04426428328654, Tokens per sec: 171099.29274757506, Loss: 3.487454652786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13239 , TFLOPS: 68.1937325248782, Tokens per sec: 168991.58135643133, Loss: 3.4848499298095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13240 , TFLOPS: 68.07413181267972, Tokens per sec: 168695.19761649746, Loss: 3.480910301208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13241 , TFLOPS: 67.06100032843608, Tokens per sec: 166184.54619289542, Loss: 3.4873716831207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13242 , TFLOPS: 70.00757295759813, Tokens per sec: 173486.47775972987, Loss: 3.5043346881866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13243 , TFLOPS: 67.20159770503373, Tokens per sec: 166532.96198018402, Loss: 3.5204989910125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13244 , TFLOPS: 66.90612026178344, Tokens per sec: 165800.73632628238, Loss: 3.485870122909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13245 , TFLOPS: 70.22009363707393, Tokens per sec: 174013.1274145564, Loss: 3.4977645874023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13246 , TFLOPS: 67.94466861416586, Tokens per sec: 168374.37354903147, Loss: 3.4616000652313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13247 , TFLOPS: 65.78177083036844, Tokens per sec: 163014.47457792147, Loss: 3.486415147781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13248 , TFLOPS: 69.16125473029722, Tokens per sec: 171389.20796282915, Loss: 3.489196538925171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13249 , TFLOPS: 68.62132300247424, Tokens per sec: 170051.1976628939, Loss: 3.49996018409729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13250 , TFLOPS: 69.43558332816848, Tokens per sec: 172069.0244481439, Loss: 3.505201578140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13251 , TFLOPS: 67.62751054153931, Tokens per sec: 167588.42090721696, Loss: 3.484947443008423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13252 , TFLOPS: 69.072784311274, Tokens per sec: 171169.96851866844, Loss: 3.4869883060455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13253 , TFLOPS: 69.07171021127176, Tokens per sec: 171167.30678054102, Loss: 3.503573417663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13254 , TFLOPS: 65.9584804098968, Tokens per sec: 163452.38038215984, Loss: 3.5043387413024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13255 , TFLOPS: 70.20614259313679, Tokens per sec: 173978.55519084577, Loss: 3.5130515098571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13256 , TFLOPS: 67.92490602706853, Tokens per sec: 168325.39968117562, Loss: 3.490729570388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13257 , TFLOPS: 66.81246660822873, Tokens per sec: 165568.6522559722, Loss: 3.497739315032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13258 , TFLOPS: 67.97223177510354, Tokens per sec: 168442.67809816787, Loss: 3.487884759902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13259 , TFLOPS: 69.02668674555235, Tokens per sec: 171055.73367274413, Loss: 3.5050175189971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13260 , TFLOPS: 68.35986074118145, Tokens per sec: 169403.2653769646, Loss: 3.4847354888916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13261 , TFLOPS: 67.826136738802, Tokens per sec: 168080.63850451584, Loss: 3.509634256362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13262 , TFLOPS: 69.0071531072398, Tokens per sec: 171007.3271651977, Loss: 3.5027618408203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13263 , TFLOPS: 67.90494008825331, Tokens per sec: 168275.92188536175, Loss: 3.494661808013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13264 , TFLOPS: 68.09597827331199, Tokens per sec: 168749.3355525305, Loss: 3.512009620666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13265 , TFLOPS: 68.35190928792275, Tokens per sec: 169383.56079986464, Loss: 3.5006210803985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13266 , TFLOPS: 68.03507038741164, Tokens per sec: 168598.39910171193, Loss: 3.5228543281555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13267 , TFLOPS: 67.7905349226801, Tokens per sec: 167992.41328230195, Loss: 3.5129010677337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13268 , TFLOPS: 68.39969419072204, Tokens per sec: 169501.9770529429, Loss: 3.5109190940856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13269 , TFLOPS: 67.62727561214635, Tokens per sec: 167587.8387263016, Loss: 3.5015463829040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13270 , TFLOPS: 69.50245467185519, Tokens per sec: 172234.73900428618, Loss: 3.4914073944091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13271 , TFLOPS: 67.6167271345692, Tokens per sec: 167561.69843685473, Loss: 3.513953924179077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13272 , TFLOPS: 70.31710401469616, Tokens per sec: 174253.5298168795, Loss: 3.485039472579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13273 , TFLOPS: 67.92505087182583, Tokens per sec: 168325.75862244016, Loss: 3.516115665435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13274 , TFLOPS: 66.07105265333912, Tokens per sec: 163731.34680226626, Loss: 3.490926742553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13275 , TFLOPS: 70.16689094136956, Tokens per sec: 173881.28527384016, Loss: 3.5101208686828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13276 , TFLOPS: 67.91562632761895, Tokens per sec: 168302.40356369625, Loss: 3.508913993835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13277 , TFLOPS: 65.65646718967878, Tokens per sec: 162703.95835295695, Loss: 3.5112552642822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13278 , TFLOPS: 69.11773461435449, Tokens per sec: 171281.36032138654, Loss: 3.452753782272339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13279 , TFLOPS: 69.10011007494882, Tokens per sec: 171237.68477123597, Loss: 3.511531352996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13280 , TFLOPS: 67.83677473696214, Tokens per sec: 168107.0006358289, Loss: 3.4919018745422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13281 , TFLOPS: 67.11364284508583, Tokens per sec: 166315.00014820546, Loss: 3.490147113800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13282 , TFLOPS: 70.58085457412398, Tokens per sec: 174907.1327576643, Loss: 3.4840612411499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13283 , TFLOPS: 68.56899019721779, Tokens per sec: 169921.511206534, Loss: 3.4788362979888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13284 , TFLOPS: 66.71030788295423, Tokens per sec: 165315.49168102938, Loss: 3.4788923263549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13285 , TFLOPS: 69.20184343833445, Tokens per sec: 171489.7912525607, Loss: 3.4748456478118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13286 , TFLOPS: 68.93254270623363, Tokens per sec: 170822.43437249027, Loss: 3.4975225925445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13287 , TFLOPS: 68.09086168852205, Tokens per sec: 168736.6560917827, Loss: 3.4902236461639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13288 , TFLOPS: 67.35468260600187, Tokens per sec: 166912.32322847703, Loss: 3.498685836791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13289 , TFLOPS: 69.04574991642393, Tokens per sec: 171102.97431014548, Loss: 3.5122690200805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13290 , TFLOPS: 69.17506013699516, Tokens per sec: 171423.41928141774, Loss: 3.506640911102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13291 , TFLOPS: 66.11387241330547, Tokens per sec: 163837.45888444886, Loss: 3.480041265487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13292 , TFLOPS: 70.17623170664847, Tokens per sec: 173904.43271917157, Loss: 3.50309419631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13293 , TFLOPS: 68.64190288607925, Tokens per sec: 170102.19688152845, Loss: 3.4984631538391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13294 , TFLOPS: 67.02359755610463, Tokens per sec: 166091.8580028026, Loss: 3.4888014793395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13295 , TFLOPS: 69.4863383614765, Tokens per sec: 172194.80101180964, Loss: 3.474968671798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13296 , TFLOPS: 68.62566240852765, Tokens per sec: 170061.9511891193, Loss: 3.4720752239227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13297 , TFLOPS: 68.68294379737756, Tokens per sec: 170203.90078075527, Loss: 3.4784152507781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13298 , TFLOPS: 68.93971636865047, Tokens per sec: 170840.21149820328, Loss: 3.475100517272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13299 , TFLOPS: 68.63098872256573, Tokens per sec: 170075.15038204758, Loss: 3.511859893798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13300 , TFLOPS: 69.67521752689905, Tokens per sec: 172662.86439048444, Loss: 3.4667470455169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13301 , TFLOPS: 67.07740367821252, Tokens per sec: 166225.1955006196, Loss: 3.5045719146728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13302 , TFLOPS: 69.53354191244948, Tokens per sec: 172311.7766110211, Loss: 3.519627571105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13303 , TFLOPS: 68.61088996021893, Tokens per sec: 170025.3434348908, Loss: 3.4889352321624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13304 , TFLOPS: 68.82418949984095, Tokens per sec: 170553.92319095816, Loss: 3.4902307987213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13305 , TFLOPS: 69.75305229291799, Tokens per sec: 172855.74751488667, Loss: 3.491791009902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13306 , TFLOPS: 68.52121661885955, Tokens per sec: 169803.12301666822, Loss: 3.4704952239990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13307 , TFLOPS: 68.68767939158735, Tokens per sec: 170215.6361048759, Loss: 3.494401454925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13308 , TFLOPS: 67.89552384006898, Tokens per sec: 168252.5873850756, Loss: 3.4665703773498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13309 , TFLOPS: 70.52838814678225, Tokens per sec: 174777.11517105743, Loss: 3.508096933364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13310 , TFLOPS: 68.59109294359905, Tokens per sec: 169976.2842468853, Loss: 3.4839234352111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13311 , TFLOPS: 67.88470485750601, Tokens per sec: 168225.77675447526, Loss: 3.502450466156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13312 , TFLOPS: 70.59167791390365, Tokens per sec: 174933.95418592764, Loss: 3.504596471786499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13313 , TFLOPS: 67.63465808272424, Tokens per sec: 167606.1333016399, Loss: 3.5006039142608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13314 , TFLOPS: 67.93546860356489, Tokens per sec: 168351.5749092974, Loss: 3.4706971645355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13315 , TFLOPS: 70.01310067484282, Tokens per sec: 173500.1760519905, Loss: 3.4968292713165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13316 , TFLOPS: 68.79857274323489, Tokens per sec: 170490.44204616963, Loss: 3.4652419090270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13317 , TFLOPS: 68.53106000222262, Tokens per sec: 169827.51600498008, Loss: 3.4933152198791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13318 , TFLOPS: 68.13777500831503, Tokens per sec: 168852.9124661585, Loss: 3.4982705116271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13319 , TFLOPS: 69.56068037093941, Tokens per sec: 172379.02870070713, Loss: 3.5093350410461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13320 , TFLOPS: 69.51751827288882, Tokens per sec: 172272.0682670402, Loss: 3.5263679027557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13321 , TFLOPS: 65.27951917802548, Tokens per sec: 161769.83965582866, Loss: 3.487501859664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13322 , TFLOPS: 70.19255139254419, Tokens per sec: 173944.87469858295, Loss: 3.4742977619171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13323 , TFLOPS: 67.8410258095855, Tokens per sec: 168117.5352620839, Loss: 3.4781548976898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13324 , TFLOPS: 67.7206550154996, Tokens per sec: 167819.2431744014, Loss: 3.5083940029144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13325 , TFLOPS: 69.0066457876121, Tokens per sec: 171006.0699712744, Loss: 3.470412254333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13326 , TFLOPS: 68.97439840677053, Tokens per sec: 170926.15740920673, Loss: 3.489048957824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13327 , TFLOPS: 69.10389903233036, Tokens per sec: 171247.07422501556, Loss: 3.5165345668792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13328 , TFLOPS: 65.15509593066945, Tokens per sec: 161461.50514252824, Loss: 3.479161024093628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13329 , TFLOPS: 70.4394254011926, Tokens per sec: 174556.65568743084, Loss: 3.5292890071868896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13330 , TFLOPS: 68.73851692122, Tokens per sec: 170341.6171035218, Loss: 3.5007057189941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13331 , TFLOPS: 68.59818755107092, Tokens per sec: 169993.86546573613, Loss: 3.494935989379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13332 , TFLOPS: 68.86424269294054, Tokens per sec: 170653.1794156824, Loss: 3.4759531021118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13333 , TFLOPS: 68.63581610833667, Tokens per sec: 170087.11317576843, Loss: 3.5127861499786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13334 , TFLOPS: 68.75557590062725, Tokens per sec: 170383.89113369468, Loss: 3.499875545501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13335 , TFLOPS: 69.05593088560501, Tokens per sec: 171128.20387330258, Loss: 3.479024648666382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13336 , TFLOPS: 68.535295856031, Tokens per sec: 169838.01291733506, Loss: 3.487358331680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13337 , TFLOPS: 69.93361685050272, Tokens per sec: 173303.20637940968, Loss: 3.5117006301879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13338 , TFLOPS: 67.3251875566697, Tokens per sec: 166839.23124708404, Loss: 3.482991933822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13339 , TFLOPS: 68.69278164559083, Tokens per sec: 170228.28005235616, Loss: 3.4673779010772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13340 , TFLOPS: 69.75079977993364, Tokens per sec: 172850.1655395767, Loss: 3.510709762573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13341 , TFLOPS: 67.06912917219027, Tokens per sec: 166204.69036318437, Loss: 3.5067386627197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13342 , TFLOPS: 70.58538287990805, Tokens per sec: 174918.3543982313, Loss: 3.4776854515075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13343 , TFLOPS: 68.70494883396158, Tokens per sec: 170258.43168546536, Loss: 3.4903247356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13344 , TFLOPS: 68.83663513859639, Tokens per sec: 170584.76485479443, Loss: 3.5050740242004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13345 , TFLOPS: 69.67757409196261, Tokens per sec: 172668.70421830902, Loss: 3.476374387741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13346 , TFLOPS: 69.56638111202047, Tokens per sec: 172393.15576509447, Loss: 3.5076489448547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13347 , TFLOPS: 68.70930848035277, Tokens per sec: 170269.23536947742, Loss: 3.5114076137542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13348 , TFLOPS: 67.08065758318273, Tokens per sec: 166233.259035583, Loss: 3.5126774311065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13349 , TFLOPS: 70.5377412736247, Tokens per sec: 174800.293249705, Loss: 3.4830257892608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13350 , TFLOPS: 68.75312313037463, Tokens per sec: 170377.81289881375, Loss: 3.496217727661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13351 , TFLOPS: 67.71046837724266, Tokens per sec: 167793.99956264987, Loss: 3.5068135261535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13352 , TFLOPS: 69.58352953928004, Tokens per sec: 172435.6514568989, Loss: 3.484353542327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13353 , TFLOPS: 69.52388324820134, Tokens per sec: 172287.8413770245, Loss: 3.4694080352783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13354 , TFLOPS: 67.99202216766874, Tokens per sec: 168491.72087103614, Loss: 3.4958670139312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13355 , TFLOPS: 68.25077604216794, Tokens per sec: 169132.94147613752, Loss: 3.479919910430908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13356 , TFLOPS: 69.58449246497206, Tokens per sec: 172438.03769283873, Loss: 3.5011942386627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13357 , TFLOPS: 69.82952011087461, Tokens per sec: 173045.2431914087, Loss: 3.485107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13358 , TFLOPS: 67.00541128330875, Tokens per sec: 166046.79041543105, Loss: 3.477647304534912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13359 , TFLOPS: 70.5798287323417, Tokens per sec: 174904.59060872204, Loss: 3.5093588829040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13360 , TFLOPS: 69.08361629154828, Tokens per sec: 171196.81135902923, Loss: 3.4870548248291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13361 , TFLOPS: 66.93786983578335, Tokens per sec: 165879.41526815842, Loss: 3.5001754760742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13362 , TFLOPS: 70.59618383001587, Tokens per sec: 174945.12034242137, Loss: 3.495090961456299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13363 , TFLOPS: 68.77539566117044, Tokens per sec: 170433.00668364757, Loss: 3.4748647212982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13364 , TFLOPS: 69.82305743044707, Tokens per sec: 173029.22795738623, Loss: 3.4892711639404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13365 , TFLOPS: 67.67407785555346, Tokens per sec: 167703.81984115715, Loss: 3.4769704341888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13366 , TFLOPS: 69.66039319900091, Tokens per sec: 172626.1280729183, Loss: 3.485217571258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13367 , TFLOPS: 69.64926302078744, Tokens per sec: 172598.546265215, Loss: 3.475947856903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13368 , TFLOPS: 67.15973963922622, Tokens per sec: 166429.23308206638, Loss: 3.47884464263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13369 , TFLOPS: 69.53397867176126, Tokens per sec: 172312.85894870872, Loss: 3.5053510665893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13370 , TFLOPS: 68.7933655705402, Tokens per sec: 170477.53809861478, Loss: 3.504457712173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13371 , TFLOPS: 68.64146255955654, Tokens per sec: 170101.10570389923, Loss: 3.4817299842834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13372 , TFLOPS: 69.57370691435749, Tokens per sec: 172411.30991028936, Loss: 3.5019969940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13373 , TFLOPS: 68.62524532891213, Tokens per sec: 170060.91761988692, Loss: 3.493560314178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13374 , TFLOPS: 68.75495944823825, Tokens per sec: 170382.36349676692, Loss: 3.4783918857574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13375 , TFLOPS: 68.07114770134147, Tokens per sec: 168687.80265987426, Loss: 3.5040078163146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13376 , TFLOPS: 69.66598805377936, Tokens per sec: 172639.99273938345, Loss: 3.475644588470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13377 , TFLOPS: 68.9869953025108, Tokens per sec: 170957.37390451378, Loss: 3.4852967262268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13378 , TFLOPS: 67.13527297544402, Tokens per sec: 166368.6019939886, Loss: 3.5104827880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13379 , TFLOPS: 70.57126480185303, Tokens per sec: 174883.3682455763, Loss: 3.5242745876312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13380 , TFLOPS: 67.7481563754267, Tokens per sec: 167887.39457382564, Loss: 3.514451503753662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13381 , TFLOPS: 67.91468034742817, Tokens per sec: 168300.05932057518, Loss: 3.499389410018921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13382 , TFLOPS: 70.09099163295815, Tokens per sec: 173693.19842659766, Loss: 3.4919564723968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13383 , TFLOPS: 69.4096807544455, Tokens per sec: 172004.83501705428, Loss: 3.488159656524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13384 , TFLOPS: 68.8312183609982, Tokens per sec: 170571.34148319878, Loss: 3.474053382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13385 , TFLOPS: 68.65381966176561, Tokens per sec: 170131.72796442048, Loss: 3.534560203552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13386 , TFLOPS: 69.67964673691236, Tokens per sec: 172673.84046081518, Loss: 3.497762680053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13387 , TFLOPS: 68.67303636592632, Tokens per sec: 170179.34907422547, Loss: 3.5072593688964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13388 , TFLOPS: 67.0407310825837, Tokens per sec: 166134.31676883146, Loss: 3.502579689025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13389 , TFLOPS: 70.54454642709244, Tokens per sec: 174817.1571696476, Loss: 3.5119082927703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13390 , TFLOPS: 68.78450685434893, Tokens per sec: 170455.58522402463, Loss: 3.4996132850646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13391 , TFLOPS: 67.9523487989541, Tokens per sec: 168393.40589297892, Loss: 3.4875848293304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13392 , TFLOPS: 69.56381139088336, Tokens per sec: 172386.78771303856, Loss: 3.487246513366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13393 , TFLOPS: 69.64095692994121, Tokens per sec: 172577.9628571066, Loss: 3.4972732067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13394 , TFLOPS: 68.74783136915111, Tokens per sec: 170364.69933156625, Loss: 3.4934258460998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13395 , TFLOPS: 67.87865025256826, Tokens per sec: 168210.77277646973, Loss: 3.488614797592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13396 , TFLOPS: 70.54572576051537, Tokens per sec: 174820.0796877839, Loss: 3.492457628250122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13397 , TFLOPS: 68.32974954015243, Tokens per sec: 169328.64650379206, Loss: 3.484060764312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13398 , TFLOPS: 67.25112454036822, Tokens per sec: 166655.69493397418, Loss: 3.511547565460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13399 , TFLOPS: 69.96451733680084, Tokens per sec: 173379.781188426, Loss: 3.5051846504211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13400 , TFLOPS: 68.73930417512742, Tokens per sec: 170343.56800542757, Loss: 3.4917285442352295 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/optimizer.pt +[rank0]:[2024-08-26 01:10:11,117] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006400337006198242, 'preprocessing_with_comm': 0.0015080009907251224, 'state_converting': 0.9784970640030224, : 0.9877772300096694}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0013400, took 4.30s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B/iter_0011400 + eval ppl=25.01797866821289, eval loss=3.219594717025757 +------------------------------------------------------------------ +iteration: 13401 , TFLOPS: 63.200343151770205, Tokens per sec: 156617.41242262104, Loss: 3.4935240745544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13402 , TFLOPS: 67.61256153607242, Tokens per sec: 167551.37562490112, Loss: 3.493352174758911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13403 , TFLOPS: 69.54566510522561, Tokens per sec: 172341.81921820028, Loss: 3.501213788986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13404 , TFLOPS: 69.16899697576125, Tokens per sec: 171408.39409996776, Loss: 3.486853837966919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13405 , TFLOPS: 67.95168028680634, Tokens per sec: 168391.74924623407, Loss: 3.5085349082946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13406 , TFLOPS: 68.55352919479073, Tokens per sec: 169883.1971393497, Loss: 3.4766335487365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13407 , TFLOPS: 69.72218608556372, Tokens per sec: 172779.25765286936, Loss: 3.50203275680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13408 , TFLOPS: 66.58255734921022, Tokens per sec: 164998.91178552867, Loss: 3.503960132598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13409 , TFLOPS: 69.51865195142915, Tokens per sec: 172274.8776473481, Loss: 3.47391676902771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13410 , TFLOPS: 68.74549216804556, Tokens per sec: 170358.90253354242, Loss: 3.5143606662750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13411 , TFLOPS: 68.85890404414812, Tokens per sec: 170639.94965587405, Loss: 3.5106067657470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13412 , TFLOPS: 68.91580962448343, Tokens per sec: 170780.96795261212, Loss: 3.5156307220458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13413 , TFLOPS: 68.55433603350248, Tokens per sec: 169885.1965745581, Loss: 3.4793715476989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13414 , TFLOPS: 68.7224531558879, Tokens per sec: 170301.80932345905, Loss: 3.4917116165161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13415 , TFLOPS: 68.23796939917135, Tokens per sec: 169101.20520402378, Loss: 3.492483139038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13416 , TFLOPS: 69.56201610977169, Tokens per sec: 172382.3388086773, Loss: 3.4807443618774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13417 , TFLOPS: 68.73558302406205, Tokens per sec: 170334.34658316895, Loss: 3.5104336738586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13418 , TFLOPS: 67.14162845261751, Tokens per sec: 166384.3515665381, Loss: 3.4921016693115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13419 , TFLOPS: 70.61028301738011, Tokens per sec: 174980.05968186367, Loss: 3.509854793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13420 , TFLOPS: 68.89992372394062, Tokens per sec: 170741.6009410906, Loss: 3.4866790771484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13421 , TFLOPS: 67.91001093766059, Tokens per sec: 168288.48800879318, Loss: 3.480588912963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13422 , TFLOPS: 69.72173337885216, Tokens per sec: 172778.13579576835, Loss: 3.497533082962036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13423 , TFLOPS: 69.58205558006003, Tokens per sec: 172431.9988235817, Loss: 3.49346661567688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13424 , TFLOPS: 68.25633171148175, Tokens per sec: 169146.70903670465, Loss: 3.4741616249084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13425 , TFLOPS: 67.21611098305617, Tokens per sec: 166568.9274818932, Loss: 3.5123674869537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13426 , TFLOPS: 69.52846804803936, Tokens per sec: 172299.20301608054, Loss: 3.4871633052825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13427 , TFLOPS: 69.67389710484387, Tokens per sec: 172659.59223917522, Loss: 3.493236541748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13428 , TFLOPS: 66.98884381537628, Tokens per sec: 166005.73440483716, Loss: 3.488896131515503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13429 , TFLOPS: 70.4709453368775, Tokens per sec: 174634.7655602656, Loss: 3.5101823806762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13430 , TFLOPS: 68.80156885783713, Tokens per sec: 170497.86674820323, Loss: 3.5037126541137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13431 , TFLOPS: 68.7490630293101, Tokens per sec: 170367.75152111877, Loss: 3.4900126457214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13432 , TFLOPS: 68.58734668221648, Tokens per sec: 169967.00059849452, Loss: 3.5061206817626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13433 , TFLOPS: 69.51510574583726, Tokens per sec: 172266.0897592446, Loss: 3.5210883617401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13434 , TFLOPS: 69.70286404160298, Tokens per sec: 172731.3754994358, Loss: 3.502819061279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13435 , TFLOPS: 67.45813145094154, Tokens per sec: 167168.68086206855, Loss: 3.4763107299804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13436 , TFLOPS: 69.57650832141702, Tokens per sec: 172418.25210558335, Loss: 3.4978604316711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13437 , TFLOPS: 69.057441476484, Tokens per sec: 171131.94728390602, Loss: 3.4973652362823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13438 , TFLOPS: 67.01427647884394, Tokens per sec: 166068.75934654917, Loss: 3.4975779056549072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13439 , TFLOPS: 69.58947143320034, Tokens per sec: 172450.37612458706, Loss: 3.5121564865112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13440 , TFLOPS: 69.62700896345794, Tokens per sec: 172543.39825966573, Loss: 3.493718147277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13441 , TFLOPS: 69.09232604063078, Tokens per sec: 171218.39507671373, Loss: 3.5009584426879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13442 , TFLOPS: 68.83788774042577, Tokens per sec: 170587.8689401131, Loss: 3.4950733184814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13443 , TFLOPS: 68.51105314962538, Tokens per sec: 169777.93682030737, Loss: 3.5152690410614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13444 , TFLOPS: 68.92362943947482, Tokens per sec: 170800.34631558452, Loss: 3.5067620277404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13445 , TFLOPS: 67.8047635649662, Tokens per sec: 168027.67342530127, Loss: 3.5032296180725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13446 , TFLOPS: 68.6428330916354, Tokens per sec: 170104.5020333672, Loss: 3.4938201904296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13447 , TFLOPS: 69.67759274397507, Tokens per sec: 172668.7504400506, Loss: 3.5076210498809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13448 , TFLOPS: 67.76123684318217, Tokens per sec: 167919.8094138562, Loss: 3.4751644134521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13449 , TFLOPS: 69.64739225594793, Tokens per sec: 172593.91030386093, Loss: 3.481208562850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13450 , TFLOPS: 67.74055283667315, Tokens per sec: 167868.55216719542, Loss: 3.4985289573669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13451 , TFLOPS: 69.2043248652215, Tokens per sec: 171495.94050173854, Loss: 3.503817081451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13452 , TFLOPS: 67.08860939295724, Tokens per sec: 166252.9644961684, Loss: 3.493436098098755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13453 , TFLOPS: 68.99621220275976, Tokens per sec: 170980.21439865604, Loss: 3.501762628555298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13454 , TFLOPS: 68.17728320727564, Tokens per sec: 168950.81813537027, Loss: 3.5047245025634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13455 , TFLOPS: 66.06634930321772, Tokens per sec: 163719.69138254868, Loss: 3.497478723526001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13456 , TFLOPS: 70.14375001778734, Tokens per sec: 173823.93951602027, Loss: 3.5064456462860107 +------------------------------------------------------------------ +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train + loss.backward() + File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward + torch.autograd.backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward + _engine_run_backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20240824_202743-hay4ewdq/files/requirements.txt b/wandb/run-20240824_202743-hay4ewdq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240824_202743-hay4ewdq/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240824_202743-hay4ewdq/files/wandb-metadata.json b/wandb/run-20240824_202743-hay4ewdq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..08ab89f511c915a8b526d18ac2aa04b3a17e2137 --- /dev/null +++ b/wandb/run-20240824_202743-hay4ewdq/files/wandb-metadata.json @@ -0,0 +1,880 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-24T11:27:44.511339", + "startedAt": "2024-08-24T11:27:43.841439", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "16", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "1280", + "--train-iters", + "23178", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--test-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--lr", + "3.5e-6", + "--min-lr", + "3.5e-7", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "23178", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B_train_2024-08-24-20:27:28" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 144, + "cpu_count_logical": 144, + "cpu_freq": { + "current": 2400.0340000000015, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 453.4449462890625 + } +} diff --git a/wandb/run-20240824_202743-hay4ewdq/files/wandb-summary.json b/wandb/run-20240824_202743-hay4ewdq/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a6f698e5abba6a39e44b5447e2b5d910b8d9dcf1 --- /dev/null +++ b/wandb/run-20240824_202743-hay4ewdq/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 3.5064456462860107, "training/perplexity": 33.32959184219802, "utils/batch_size": 16, "utils/global_batch_size": 1280, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 10, "utils/iteration": 13456, "optimizer/lr": 1.5751394286751747e-06, "optimizer/variance_l2": 0.019451412876513852, "optimizer/variance_sqrt_l2": 1.0008521025901655, "optimizer/momentum_l2": 0.8679927575390453, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 1.0009918212890625, "optimizer/variance_sqrt_l1": 6315.0, "optimizer/momentum_l1": 3813.0, "optimizer/weight_l1": 6898688.0, "optimizer/variance_abs_max": 0.01031494140625, "optimizer/variance_sqrt_abs_max": 0.1015625, "optimizer/momentum_abs_max": 0.10205078125, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 7.547867132991087, "stats/tokens_per_sec": 173823.93951602027, "stats/tokens_per_sec_per_gpu": 21727.992439502534, "stats/tflops": 70.14375001778734, "_timestamp": 1724602648.3759308, "_runtime": 103784.52010989189, "_step": 13456, "evaluation/val_loss": 3.219594717025757, "evaluation/val_ppl": 25.01797866821289, "_wandb": {"runtime": 103790}} \ No newline at end of file diff --git a/wandb/run-20240824_202743-hay4ewdq/logs/debug.log b/wandb/run-20240824_202743-hay4ewdq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..20ffd082c2e232fc32d524bbdc779e816fb68b2b --- /dev/null +++ b/wandb/run-20240824_202743-hay4ewdq/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-24 20:27:43,848 INFO MainThread:35070 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_setup.py:_flush():76] Configure stats pid to 35070 +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train baseline'} +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240824_202743-hay4ewdq/logs/debug.log +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240824_202743-hay4ewdq/logs/debug-internal.log +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_init.py:init():566] calling init triggers +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'test_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B_train_2024-08-24-20:27:28', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 3.5e-06, 'lr_decay_style': 'cosine', 'lr_decay_iters': 23178, 'lr_warmup_iters': 500, 'min_lr': 3.5e-07, 'train_iters': 23178, 'train_samples': None, 'global_batch_size': 1280, 'micro_batch_size': 16, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 8, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 10} +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_init.py:init():616] starting backend +2024-08-24 20:27:43,849 INFO MainThread:35070 [wandb_init.py:init():620] setting up manager +2024-08-24 20:27:43,854 INFO MainThread:35070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-24 20:27:43,855 INFO MainThread:35070 [wandb_init.py:init():628] backend started and connected +2024-08-24 20:27:43,862 INFO MainThread:35070 [wandb_init.py:init():720] updated telemetry +2024-08-24 20:27:43,875 INFO MainThread:35070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-24 20:27:44,375 INFO MainThread:35070 [wandb_run.py:_on_init():2262] communicating current version +2024-08-24 20:27:44,398 INFO MainThread:35070 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-24 20:27:44,398 INFO MainThread:35070 [wandb_init.py:init():804] starting run threads in backend +2024-08-24 20:27:44,545 INFO MainThread:35070 [wandb_run.py:_console_start():2241] atexit reg +2024-08-24 20:27:44,545 INFO MainThread:35070 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-24 20:27:44,546 INFO MainThread:35070 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-24 20:27:44,546 INFO MainThread:35070 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-24 20:27:44,547 INFO MainThread:35070 [wandb_init.py:init():847] run started, returning control to user process +2024-08-24 20:27:50,414 INFO MainThread:35070 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-24 20:27:50,415 INFO MainThread:35070 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 8} +2024-08-26 01:17:46,015 WARNING MsgRouterThr:35070 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240826_015431-e8t13a6a/files/config.yaml b/wandb/run-20240826_015431-e8t13a6a/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb9def50e2de193eea072c883d7d04be227e03f5 --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +test_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B-3e-5_train_2024-08-26-01:53:55 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5 +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 3.5e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 23178 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 3.5e-06 +train_iters: + desc: null + value: 23178 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1280 +micro_batch_size: + desc: null + value: 16 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B-3e-5 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 8 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724604871.337745 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240826_015431-e8t13a6a/files/output.log b/wandb/run-20240826_015431-e8t13a6a/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3e2d7662626683963ade54a0d9156554380d8dac --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/files/output.log @@ -0,0 +1,300 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B-3e-5. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... +Let split = None +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 20.153525093609847, Tokens per sec: 49942.65527602798, Loss: 4.021102428436279 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 70.68164617689358, Tokens per sec: 175156.90545243904, Loss: 3.9813687801361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 69.98240576364007, Tokens per sec: 173424.1106807065, Loss: 3.9846248626708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 71.02544188401596, Tokens per sec: 176008.86908691184, Loss: 3.975637435913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 68.9460643932007, Tokens per sec: 170855.94260233143, Loss: 3.974337100982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 70.94214202250056, Tokens per sec: 175802.44285383946, Loss: 3.9794557094573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 69.92176253176417, Tokens per sec: 173273.83007171456, Loss: 3.9648544788360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 69.83485059346295, Tokens per sec: 173058.45271446375, Loss: 3.9264297485351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 71.01256239064217, Tokens per sec: 175976.95228353832, Loss: 3.922988176345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 69.9968723680435, Tokens per sec: 173459.9605200462, Loss: 3.937690496444702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 69.89290613826594, Tokens per sec: 173202.32074983176, Loss: 3.919388771057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 70.94903956147712, Tokens per sec: 175819.53571524998, Loss: 3.8986051082611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 70.97963436277455, Tokens per sec: 175895.35300316062, Loss: 3.886922836303711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 69.90473181703398, Tokens per sec: 173231.62608452505, Loss: 3.892519950866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 69.87233773396991, Tokens per sec: 173151.34997818942, Loss: 3.867856025695801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 70.9066354908006, Tokens per sec: 175714.45375691957, Loss: 3.8647890090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 71.04575544620172, Tokens per sec: 176059.2083317318, Loss: 3.8641560077667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 68.80144436424263, Tokens per sec: 170497.55823936197, Loss: 3.8610637187957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 70.99357849372309, Tokens per sec: 175929.90809572392, Loss: 3.831857681274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 71.03027978013282, Tokens per sec: 176020.85792642768, Loss: 3.8541829586029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 71.01813733862383, Tokens per sec: 175990.76761876806, Loss: 3.8348309993743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 68.90635801396095, Tokens per sec: 170757.54582055658, Loss: 3.8218417167663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 71.02594941165434, Tokens per sec: 176010.12679630856, Loss: 3.787583351135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 70.96431869282776, Tokens per sec: 175857.39908587214, Loss: 3.8005518913269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 68.81296461222809, Tokens per sec: 170526.10668293003, Loss: 3.799785614013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 71.06738245880707, Tokens per sec: 176112.80245137995, Loss: 3.8177859783172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 71.10954700261692, Tokens per sec: 176217.29083574872, Loss: 3.8010947704315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 69.83614922973895, Tokens per sec: 173061.67087821226, Loss: 3.7915537357330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 69.98882001809054, Tokens per sec: 173440.0059098232, Loss: 3.805101156234741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 70.98765983425363, Tokens per sec: 175915.24100556446, Loss: 3.775930404663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 71.03395267071123, Tokens per sec: 176029.95975388327, Loss: 3.7867870330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 70.12617116384435, Tokens per sec: 173780.37717947894, Loss: 3.7502622604370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 70.08300410533441, Tokens per sec: 173673.40445324726, Loss: 3.7561774253845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 70.90225457240996, Tokens per sec: 175703.59735854957, Loss: 3.72761607170105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 69.87001526079857, Tokens per sec: 173145.5946338293, Loss: 3.7720561027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 70.94725609607076, Tokens per sec: 175815.1160915081, Loss: 3.7569081783294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 69.99463423846356, Tokens per sec: 173454.41418839683, Loss: 3.73331880569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 69.8430588085075, Tokens per sec: 173078.79357555366, Loss: 3.735466241836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 71.0770979858011, Tokens per sec: 176136.8785975248, Loss: 3.733441114425659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 70.0709273945501, Tokens per sec: 173643.47703356418, Loss: 3.7298524379730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 70.98405241073439, Tokens per sec: 175906.30141269325, Loss: 3.7257115840911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 69.85065325706042, Tokens per sec: 173097.61345566539, Loss: 3.7346835136413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 70.94158009242848, Tokens per sec: 175801.05032921908, Loss: 3.68245267868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 69.97394440600713, Tokens per sec: 173403.1425043966, Loss: 3.7320098876953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 69.88806128534492, Tokens per sec: 173190.31467058888, Loss: 3.7051639556884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 70.9872018143685, Tokens per sec: 175914.10598183403, Loss: 3.7359979152679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 70.07172206568119, Tokens per sec: 173645.44631616728, Loss: 3.712636947631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 69.8703011251113, Tokens per sec: 173146.3030370868, Loss: 3.6813783645629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 70.93783147075688, Tokens per sec: 175791.76083177212, Loss: 3.7108938694000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 70.83709842696908, Tokens per sec: 175542.13325260513, Loss: 3.6969285011291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 69.92248657633661, Tokens per sec: 173275.62433392514, Loss: 3.646868944168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 69.92198452888964, Tokens per sec: 173274.38020504647, Loss: 3.684920310974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 70.9778821684373, Tokens per sec: 175891.01087257278, Loss: 3.6564295291900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 70.93974205181634, Tokens per sec: 175796.49546210535, Loss: 3.660841464996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 68.78629739357255, Tokens per sec: 170460.02237746498, Loss: 3.6749563217163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 70.92119497383085, Tokens per sec: 175750.533759728, Loss: 3.6741089820861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 70.99713846329914, Tokens per sec: 175938.7300924964, Loss: 3.6460211277008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 69.84674892694953, Tokens per sec: 173087.93809555314, Loss: 3.663341760635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 69.16108791457162, Tokens per sec: 171388.79457508484, Loss: 3.6386585235595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 71.00604673939436, Tokens per sec: 175960.805781988, Loss: 3.6252686977386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 71.0247399137389, Tokens per sec: 176007.1295272369, Loss: 3.650737762451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 67.97204400337, Tokens per sec: 168442.2127791274, Loss: 3.6425609588623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 70.87117698713219, Tokens per sec: 175626.58367875265, Loss: 3.629193067550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 70.97734876054649, Tokens per sec: 175889.6890290596, Loss: 3.5961902141571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 69.71091369874202, Tokens per sec: 172751.32343656928, Loss: 3.586948871612549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 69.9388093814564, Tokens per sec: 173316.0740431174, Loss: 3.6139042377471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 70.96065448324389, Tokens per sec: 175848.31877087732, Loss: 3.612912654876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 70.98020020748864, Tokens per sec: 175896.75522869508, Loss: 3.6172361373901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 70.07209992287763, Tokens per sec: 173646.3826879245, Loss: 3.6056020259857178 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240826_015431-e8t13a6a/files/requirements.txt b/wandb/run-20240826_015431-e8t13a6a/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240826_015431-e8t13a6a/files/wandb-metadata.json b/wandb/run-20240826_015431-e8t13a6a/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d73b53fdbfeae174a9920b4b2ea0a9b34806c405 --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/files/wandb-metadata.json @@ -0,0 +1,880 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-25T16:54:31.975678", + "startedAt": "2024-08-25T16:54:31.321752", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "16", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "1280", + "--train-iters", + "23178", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--test-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--lr", + "3.5e-5", + "--min-lr", + "3.5e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "23178", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B-3e-5", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B-3e-5_train_2024-08-26-01:53:55" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 144, + "cpu_count_logical": 144, + "cpu_freq": { + "current": 2400.0410000000047, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 453.44499588012695 + } +} diff --git a/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json b/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..27caa6771c30987f2757ca3f67001095bd36d4a9 --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 3.6056020259857178, "training/perplexity": 36.80383405355257, "utils/batch_size": 16, "utils/global_batch_size": 1280, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 10, "utils/iteration": 69, "optimizer/lr": 7.847e-06, "optimizer/variance_l2": 0.029041982137657456, "optimizer/variance_sqrt_l2": 0.9854715083361724, "optimizer/momentum_l2": 0.8998315030890887, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.971954345703125, "optimizer/variance_sqrt_l1": 5438.0, "optimizer/momentum_l1": 3915.0, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.0159912109375, "optimizer/variance_sqrt_abs_max": 0.1259765625, "optimizer/momentum_abs_max": 0.09814453125, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 7.555584975000102, "stats/tokens_per_sec": 173646.3826879245, "stats/tokens_per_sec_per_gpu": 21705.79783599056, "stats/tflops": 70.07209992287763, "_timestamp": 1724605444.1350832, "_runtime": 572.7973382472992, "_step": 69, "_wandb": {"runtime": 576}} \ No newline at end of file diff --git a/wandb/run-20240826_015431-e8t13a6a/logs/debug-internal.log b/wandb/run-20240826_015431-e8t13a6a/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ba5c875aa8ffd21507386c4e2f81a8c2c2aaa0d0 --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/logs/debug-internal.log @@ -0,0 +1,783 @@ +2024-08-26 01:54:31,339 INFO StreamThr :12084 [internal.py:wandb_internal():86] W&B internal server running at pid: 12084, started at: 2024-08-26 01:54:31.338073 +2024-08-26 01:54:31,340 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status +2024-08-26 01:54:31,344 INFO WriterThread:12084 [datastore.py:open_for_write():87] open: /project/wandb/run-20240826_015431-e8t13a6a/run-e8t13a6a.wandb +2024-08-26 01:54:31,346 DEBUG SenderThread:12084 [sender.py:send():382] send: header +2024-08-26 01:54:31,418 DEBUG SenderThread:12084 [sender.py:send():382] send: run +2024-08-26 01:54:31,825 INFO SenderThread:12084 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240826_015431-e8t13a6a/files +2024-08-26 01:54:31,826 INFO SenderThread:12084 [sender.py:_start_run_threads():1136] run started: e8t13a6a with start time 1724604871.337745 +2024-08-26 01:54:31,831 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: check_version +2024-08-26 01:54:31,831 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: check_version +2024-08-26 01:54:31,902 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: run_start +2024-08-26 01:54:31,909 DEBUG HandlerThread:12084 [system_info.py:__init__():27] System info init +2024-08-26 01:54:31,909 DEBUG HandlerThread:12084 [system_info.py:__init__():42] System info init done +2024-08-26 01:54:31,909 INFO HandlerThread:12084 [system_monitor.py:start():194] Starting system monitor +2024-08-26 01:54:31,909 INFO SystemMonitor:12084 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-26 01:54:31,909 INFO HandlerThread:12084 [system_monitor.py:probe():214] Collecting system info +2024-08-26 01:54:31,910 INFO SystemMonitor:12084 [interfaces.py:start():190] Started cpu monitoring +2024-08-26 01:54:31,910 INFO SystemMonitor:12084 [interfaces.py:start():190] Started disk monitoring +2024-08-26 01:54:31,911 INFO SystemMonitor:12084 [interfaces.py:start():190] Started gpu monitoring +2024-08-26 01:54:31,911 INFO SystemMonitor:12084 [interfaces.py:start():190] Started memory monitoring +2024-08-26 01:54:31,913 INFO SystemMonitor:12084 [interfaces.py:start():190] Started network monitoring +2024-08-26 01:54:31,975 DEBUG HandlerThread:12084 [system_info.py:probe():151] Probing system +2024-08-26 01:54:31,977 DEBUG HandlerThread:12084 [system_info.py:_probe_git():136] Probing git +2024-08-26 01:54:31,991 DEBUG HandlerThread:12084 [system_info.py:_probe_git():144] Probing git done +2024-08-26 01:54:31,991 DEBUG HandlerThread:12084 [system_info.py:probe():199] Probing system done +2024-08-26 01:54:31,991 DEBUG HandlerThread:12084 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-25T16:54:31.975678', 'startedAt': '2024-08-25T16:54:31.321752', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '16', '--valid_micro_batch_size', '1', '--global-batch-size', '1280', '--train-iters', '23178', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--test-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--lr', '3.5e-5', '--min-lr', '3.5e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '23178', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'NO_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-0.5B-3e-5', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-0.5B-3e-5_train_2024-08-26-01:53:55'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 144, 'cpu_count_logical': 144, 'cpu_freq': {'current': 2400.0410000000047, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 453.44499588012695}} +2024-08-26 01:54:31,991 INFO HandlerThread:12084 [system_monitor.py:probe():224] Finished collecting system info +2024-08-26 01:54:31,991 INFO HandlerThread:12084 [system_monitor.py:probe():227] Publishing system info +2024-08-26 01:54:31,993 INFO HandlerThread:12084 [system_monitor.py:probe():229] Finished publishing system info +2024-08-26 01:54:32,022 DEBUG SenderThread:12084 [sender.py:send():382] send: files +2024-08-26 01:54:32,022 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-26 01:54:32,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: python_packages +2024-08-26 01:54:32,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:54:32,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:54:32,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: python_packages +2024-08-26 01:54:32,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:54:32,247 DEBUG SenderThread:12084 [sender.py:send():382] send: telemetry +2024-08-26 01:54:32,740 INFO wandb-upload_0:12084 [upload_job.py:push():131] Uploaded file /tmp/tmppg5gcwe3wandb/j2fhhxn4-wandb-metadata.json +2024-08-26 01:54:32,827 INFO Thread-12 :12084 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240826_015431-e8t13a6a/files/requirements.txt +2024-08-26 01:54:32,828 INFO Thread-12 :12084 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-metadata.json +2024-08-26 01:54:32,828 INFO Thread-12 :12084 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:34,828 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:37,328 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:54:38,830 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:39,831 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:40,832 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:42,416 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:54:47,033 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:54:47,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:54:47,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:54:48,252 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:54:53,253 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:54:54,840 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:55,840 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:56,216 DEBUG SenderThread:12084 [sender.py:send():382] send: config +2024-08-26 01:54:56,216 DEBUG SenderThread:12084 [sender.py:send():382] send: config +2024-08-26 01:54:56,841 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:57,842 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:54:59,217 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:02,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:55:02,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:55:02,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:55:04,301 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:04,847 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/config.yaml +2024-08-26 01:55:09,515 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:14,515 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:17,033 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:55:17,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:55:17,077 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:55:20,301 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:25,301 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:30,302 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:31,913 DEBUG SystemMonitor:12084 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-26 01:55:31,919 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:55:32,033 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:55:32,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:55:32,077 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:55:32,177 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:55:34,864 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:55:36,251 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:39,669 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:55:39,672 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:55:39,672 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:55:39,674 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:55:39,869 INFO Thread-12 :12084 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:55:40,869 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:55:41,715 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:46,716 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:47,033 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:55:47,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:55:47,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:55:47,236 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:55:47,307 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:55:47,307 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:55:47,309 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:55:47,875 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:55:48,876 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:55:52,310 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:55:54,692 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:55:54,694 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:55:54,695 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:55:54,696 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:55:54,880 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:55:56,882 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:55:57,735 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:01,922 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:56:02,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:56:02,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:56:02,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:56:02,373 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:02,377 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:02,377 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:02,378 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:02,886 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:03,418 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:04,888 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:08,419 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:09,839 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:09,841 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:09,842 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:09,843 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:09,892 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:10,893 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:13,844 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:17,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:56:17,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:56:17,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:56:17,412 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:17,414 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:17,414 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:17,416 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:17,898 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:18,899 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:19,455 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:24,455 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:24,996 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:24,998 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:24,999 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:25,000 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:25,904 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:26,905 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:30,001 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:31,925 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:56:32,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:56:32,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:56:32,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:56:32,453 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:32,455 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:32,456 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:32,457 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:32,910 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:34,911 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:35,495 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:40,019 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:40,021 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:40,022 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:40,023 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:40,916 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:40,916 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:41,024 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:46,025 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:47,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:56:47,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:56:47,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:56:47,596 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:47,598 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:47,598 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:47,600 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:47,922 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:48,922 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:51,639 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:56:55,060 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:56:55,063 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:56:55,063 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:56:55,064 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:56:55,927 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:56:56,928 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:56:57,065 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:01,928 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:57:02,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:57:02,034 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:57:02,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:57:02,304 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:02,521 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:02,522 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:02,523 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:02,524 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:02,932 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:04,934 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:07,567 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:10,096 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:10,100 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:10,100 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:10,101 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:10,939 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:10,939 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:13,102 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:17,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:57:17,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:57:17,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:57:17,675 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:17,677 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:17,678 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:17,680 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:17,944 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:18,718 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:18,945 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:23,719 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:25,144 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:25,146 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:25,147 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:25,148 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:25,950 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:26,951 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:29,149 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:31,931 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:57:32,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:57:32,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:57:32,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:57:32,598 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:32,600 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:32,600 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:32,601 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:32,955 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:34,642 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:34,957 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:39,643 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:40,297 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:40,299 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:40,300 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:40,301 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:40,961 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:42,962 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:45,302 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:47,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:57:47,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:57:47,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:57:47,757 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:47,760 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:47,760 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:47,761 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:47,967 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:48,967 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:57:50,799 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:55,213 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:57:55,216 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:57:55,216 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:57:55,218 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:57:55,973 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:57:56,219 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:57:56,973 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:01,219 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:01,934 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:58:02,034 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:58:02,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:58:02,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:58:02,669 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:02,671 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:02,671 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:02,673 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:02,977 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:04,979 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:06,715 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:10,355 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:10,358 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:10,358 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:10,360 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:10,983 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:12,361 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:12,984 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:17,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:58:17,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:58:17,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:58:17,811 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:17,813 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:17,813 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:17,813 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:17,815 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:17,988 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:18,989 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:22,855 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:25,274 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:25,308 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:25,308 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:25,310 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:25,995 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:26,995 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:28,311 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:31,939 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:58:32,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:58:32,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:58:32,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:58:32,969 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:32,971 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:32,971 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:32,972 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:33,000 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:34,014 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:35,001 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:39,015 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:40,421 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:40,424 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:40,424 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:40,426 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:41,006 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:43,007 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:44,427 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:47,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:58:47,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:58:47,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:58:47,868 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:47,870 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:47,870 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:47,872 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:48,011 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:49,011 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:58:49,911 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:54,912 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:58:55,452 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:58:55,454 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:58:55,455 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:58:55,457 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:58:56,017 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:58:57,018 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:00,458 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:01,941 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:59:02,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:59:02,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:59:02,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:59:03,018 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:03,020 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:03,020 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:03,021 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:03,022 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:05,024 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:06,067 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:10,478 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:10,481 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:10,481 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:10,483 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:11,028 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:11,484 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:13,029 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:16,484 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:17,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:59:17,035 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:59:17,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:59:17,933 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:17,935 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:17,935 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:17,937 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:18,033 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:19,034 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:21,975 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:25,485 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:25,488 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:25,488 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:25,490 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:26,039 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:27,040 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:27,491 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:31,944 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 01:59:32,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:59:32,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:59:32,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:59:33,041 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:33,043 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:33,043 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:33,043 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:33,044 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:33,045 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:35,046 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:38,083 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:40,510 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:40,513 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:40,513 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:40,515 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:41,050 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:43,052 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:43,515 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:47,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 01:59:47,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 01:59:47,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 01:59:48,090 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:48,091 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:48,092 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:48,093 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:49,056 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:49,056 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:49,135 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:54,136 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 01:59:55,554 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 01:59:55,558 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 01:59:55,558 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 01:59:55,560 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 01:59:56,061 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 01:59:57,062 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 01:59:59,561 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:01,947 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:00:02,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:00:02,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:00:02,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:00:03,120 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:03,122 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:03,122 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:03,123 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:04,067 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:05,068 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:05,167 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:10,167 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:10,702 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:10,705 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:10,705 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:10,707 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:11,073 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:13,074 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:15,708 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:17,035 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:00:17,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:00:17,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:00:18,153 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:18,155 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:18,155 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:18,156 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:19,079 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:19,079 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:21,195 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:25,710 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:25,713 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:25,713 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:25,715 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:26,084 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:26,716 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:27,085 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:31,717 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:31,948 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:00:32,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:00:32,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:00:32,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:00:33,171 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:33,173 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:33,173 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:33,174 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:34,090 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:35,091 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:37,216 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:40,753 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:40,755 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:40,755 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:40,757 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:41,095 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:42,758 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:43,097 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:47,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:00:47,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:00:47,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:00:48,218 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:48,219 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:48,220 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:48,220 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:48,221 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:49,102 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:49,102 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:53,267 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:00:55,786 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:00:55,788 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:00:55,789 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:00:55,791 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:00:56,107 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:00:57,108 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:00:58,791 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:01,954 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:01:02,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:01:02,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:01:02,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:01:03,363 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:03,365 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:03,365 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:03,366 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:04,113 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:04,406 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:05,114 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:09,407 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:10,823 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:10,826 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:10,826 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:10,827 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:11,118 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:13,120 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:14,828 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:17,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:01:17,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:01:17,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:01:18,380 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:18,382 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:18,382 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:18,384 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:19,124 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:19,124 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:20,422 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:25,423 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:25,960 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:25,963 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:25,963 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:25,965 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:26,129 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:27,129 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:30,966 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:31,955 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:01:32,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:01:32,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:01:32,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:01:33,426 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:33,428 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:33,428 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:33,429 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:34,135 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:35,136 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:36,471 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:40,902 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:40,905 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:40,905 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:40,907 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:41,140 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:41,908 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:43,141 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:46,908 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:47,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:01:47,036 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:01:47,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:01:48,476 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:48,478 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:48,478 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:48,479 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:49,146 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:51,148 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:52,523 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:01:56,051 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:01:56,053 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:01:56,053 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:01:56,055 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:01:56,151 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:01:57,152 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:01:58,056 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:01,959 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:02:02,036 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:02:02,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:02:02,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:02:03,220 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:03,512 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:03,514 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:03,514 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:03,515 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:04,157 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:05,158 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:08,555 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:10,978 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:10,980 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:10,981 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:10,982 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:11,163 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:13,164 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:13,983 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:17,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:02:17,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:02:17,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:02:18,677 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:18,679 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:18,679 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:18,680 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:19,169 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:19,720 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:21,170 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:24,720 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:26,144 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:26,147 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:26,147 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:26,150 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:26,173 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:27,174 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:30,150 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:31,962 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:02:32,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:02:32,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:02:32,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:02:33,604 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:33,606 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:33,606 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:33,607 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:34,179 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:35,180 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:35,646 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:40,647 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:41,186 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:41,189 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:41,189 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:41,190 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:42,185 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:43,186 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:46,192 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:47,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:02:47,038 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:02:47,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:02:48,843 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:48,845 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:48,845 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:48,847 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:49,190 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:51,191 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:51,891 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:02:56,302 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:02:56,305 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:02:56,306 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:02:56,308 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:02:57,197 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:02:57,197 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:02:57,309 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:01,965 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:03:02,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:03:02,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:03:02,040 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:03:03,307 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:03,758 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:03,760 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:03,760 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:03,762 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:04,202 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:05,202 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:08,803 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:11,549 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:11,552 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:11,552 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:11,554 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:12,207 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:13,208 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:14,555 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:17,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:03:17,038 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:03:17,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:03:19,024 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:19,025 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:19,026 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:19,027 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:19,213 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:20,070 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:21,214 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:25,071 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:26,486 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:26,488 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:26,489 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:26,491 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:27,218 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:27,219 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:30,492 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:31,969 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:03:32,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:03:32,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:03:32,038 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:03:34,082 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:34,084 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:34,084 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:34,086 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:34,223 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:35,224 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:36,126 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:41,127 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:41,655 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:41,658 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:41,658 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:41,659 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:42,228 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:43,229 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:46,660 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:47,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:03:47,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:03:47,039 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:03:49,117 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:49,119 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:49,119 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:49,120 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:49,233 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:51,235 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:52,162 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:03:56,578 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:03:56,581 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:03:56,581 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:03:56,584 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:03:57,239 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:03:57,240 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:03:57,585 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:04:01,971 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:04:02,037 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: stop_status +2024-08-26 02:04:02,037 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: stop_status +2024-08-26 02:04:02,039 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-26 02:04:03,307 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:04:04,136 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: partial_history +2024-08-26 02:04:04,138 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:04:04,138 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:04:04,139 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:04:04,245 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:04:05,245 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:04:08,188 DEBUG SenderThread:12084 [sender.py:send():382] send: exit +2024-08-26 02:04:08,188 INFO SenderThread:12084 [sender.py:send_exit():589] handling exit code: 255 +2024-08-26 02:04:08,188 INFO SenderThread:12084 [sender.py:send_exit():591] handling runtime: 576 +2024-08-26 02:04:08,189 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:04:08,190 INFO SenderThread:12084 [sender.py:send_exit():597] send defer +2024-08-26 02:04:08,190 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:08,190 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-26 02:04:08,190 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:08,190 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-26 02:04:08,190 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 1 +2024-08-26 02:04:08,190 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:08,190 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-26 02:04:08,190 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:08,190 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-26 02:04:08,191 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 2 +2024-08-26 02:04:08,191 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:08,191 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-26 02:04:08,191 INFO HandlerThread:12084 [system_monitor.py:finish():203] Stopping system monitor +2024-08-26 02:04:08,191 DEBUG SystemMonitor:12084 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-26 02:04:08,191 INFO HandlerThread:12084 [interfaces.py:finish():202] Joined cpu monitor +2024-08-26 02:04:08,191 DEBUG SystemMonitor:12084 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-26 02:04:08,192 INFO HandlerThread:12084 [interfaces.py:finish():202] Joined disk monitor +2024-08-26 02:04:08,248 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:04:09,343 INFO HandlerThread:12084 [interfaces.py:finish():202] Joined gpu monitor +2024-08-26 02:04:09,343 INFO HandlerThread:12084 [interfaces.py:finish():202] Joined memory monitor +2024-08-26 02:04:09,343 INFO HandlerThread:12084 [interfaces.py:finish():202] Joined network monitor +2024-08-26 02:04:09,343 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-26 02:04:09,345 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:04:09,345 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:09,345 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-26 02:04:09,345 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 3 +2024-08-26 02:04:09,345 DEBUG SenderThread:12084 [sender.py:send():382] send: stats +2024-08-26 02:04:09,345 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:09,346 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: poll_exit +2024-08-26 02:04:09,346 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-26 02:04:09,348 DEBUG SenderThread:12084 [sender.py:send():382] send: history +2024-08-26 02:04:09,348 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: summary_record +2024-08-26 02:04:09,349 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:04:09,349 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:09,349 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-26 02:04:09,350 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 4 +2024-08-26 02:04:09,350 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:09,350 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-26 02:04:09,350 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:09,350 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-26 02:04:09,350 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 5 +2024-08-26 02:04:09,350 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:09,350 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-26 02:04:09,351 DEBUG SenderThread:12084 [sender.py:send():382] send: summary +2024-08-26 02:04:09,352 INFO SenderThread:12084 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-26 02:04:09,352 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:09,352 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-26 02:04:09,352 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 6 +2024-08-26 02:04:09,352 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:09,352 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-26 02:04:09,352 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:09,352 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-26 02:04:09,352 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 7 +2024-08-26 02:04:09,352 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: status_report +2024-08-26 02:04:09,353 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:09,353 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-26 02:04:09,353 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:09,353 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-26 02:04:10,189 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-26 02:04:10,251 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/wandb-summary.json +2024-08-26 02:04:12,277 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 8 +2024-08-26 02:04:12,278 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: poll_exit +2024-08-26 02:04:12,278 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:12,278 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-26 02:04:12,278 DEBUG SenderThread:12084 [sender.py:send_request():409] send_request: defer +2024-08-26 02:04:12,278 INFO SenderThread:12084 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-26 02:04:12,278 INFO SenderThread:12084 [job_builder.py:build():296] Attempting to build job artifact +2024-08-26 02:04:12,280 INFO SenderThread:12084 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-26 02:04:12,296 INFO SenderThread:12084 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-26 02:04:12,307 INFO SenderThread:12084 [sender.py:transition_state():617] send defer: 9 +2024-08-26 02:04:12,307 DEBUG SenderThread:12084 [sender.py:send():382] send: artifact +2024-08-26 02:04:12,307 DEBUG HandlerThread:12084 [handler.py:handle_request():146] handle_request: defer +2024-08-26 02:04:12,309 INFO HandlerThread:12084 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-26 02:04:13,253 INFO Thread-12 :12084 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240826_015431-e8t13a6a/files/output.log +2024-08-26 02:04:40,968 WARNING StreamThr :12084 [internal.py:is_dead():414] Internal process exiting, parent pid 11439 disappeared +2024-08-26 02:04:40,968 ERROR StreamThr :12084 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-26 02:04:41,311 INFO HandlerThread:12084 [handler.py:finish():869] shutting down handler +2024-08-26 02:04:41,312 INFO WriterThread:12084 [datastore.py:close():296] close: /project/wandb/run-20240826_015431-e8t13a6a/run-e8t13a6a.wandb diff --git a/wandb/run-20240826_015431-e8t13a6a/logs/debug.log b/wandb/run-20240826_015431-e8t13a6a/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dc1e7c37d758a4120569265a217dd71e5b298b1e --- /dev/null +++ b/wandb/run-20240826_015431-e8t13a6a/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Configure stats pid to 11439 +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train baseline'} +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-26 01:54:31,329 INFO MainThread:11439 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240826_015431-e8t13a6a/logs/debug.log +2024-08-26 01:54:31,330 INFO MainThread:11439 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240826_015431-e8t13a6a/logs/debug-internal.log +2024-08-26 01:54:31,330 INFO MainThread:11439 [wandb_init.py:init():566] calling init triggers +2024-08-26 01:54:31,330 INFO MainThread:11439 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'test_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B-3e-5_train_2024-08-26-01:53:55', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3e-5', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 3.5e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 23178, 'lr_warmup_iters': 500, 'min_lr': 3.5e-06, 'train_iters': 23178, 'train_samples': None, 'global_batch_size': 1280, 'micro_batch_size': 16, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B-3e-5', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 8, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 10} +2024-08-26 01:54:31,330 INFO MainThread:11439 [wandb_init.py:init():616] starting backend +2024-08-26 01:54:31,330 INFO MainThread:11439 [wandb_init.py:init():620] setting up manager +2024-08-26 01:54:31,335 INFO MainThread:11439 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-26 01:54:31,337 INFO MainThread:11439 [wandb_init.py:init():628] backend started and connected +2024-08-26 01:54:31,344 INFO MainThread:11439 [wandb_init.py:init():720] updated telemetry +2024-08-26 01:54:31,414 INFO MainThread:11439 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-26 01:54:31,830 INFO MainThread:11439 [wandb_run.py:_on_init():2262] communicating current version +2024-08-26 01:54:31,853 INFO MainThread:11439 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-26 01:54:31,853 INFO MainThread:11439 [wandb_init.py:init():804] starting run threads in backend +2024-08-26 01:54:32,033 INFO MainThread:11439 [wandb_run.py:_console_start():2241] atexit reg +2024-08-26 01:54:32,033 INFO MainThread:11439 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-26 01:54:32,034 INFO MainThread:11439 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-26 01:54:32,034 INFO MainThread:11439 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-26 01:54:32,034 INFO MainThread:11439 [wandb_init.py:init():847] run started, returning control to user process +2024-08-26 01:54:56,215 INFO MainThread:11439 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-26 01:54:56,215 INFO MainThread:11439 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 8} diff --git a/wandb/run-20240826_015431-e8t13a6a/run-e8t13a6a.wandb b/wandb/run-20240826_015431-e8t13a6a/run-e8t13a6a.wandb new file mode 100644 index 0000000000000000000000000000000000000000..099c31d1376e7094e97ca3aa9e14959ca11ed267 Binary files /dev/null and b/wandb/run-20240826_015431-e8t13a6a/run-e8t13a6a.wandb differ diff --git a/wandb/run-20240826_020628-460rkh6q/files/output.log b/wandb/run-20240826_020628-460rkh6q/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0bccc1bf9e2b21703680fc98391ef79ac2625919 --- /dev/null +++ b/wandb/run-20240826_020628-460rkh6q/files/output.log @@ -0,0 +1,71128 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B-3.5e-5. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 47.23609380567841, Tokens per sec: 117056.24393576314, Loss: 4.021102428436279 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 71.45048528068875, Tokens per sec: 177062.17344626246, Loss: 3.9813292026519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 70.32574414422021, Tokens per sec: 174274.941009629, Loss: 3.984628200531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 71.44720646176258, Tokens per sec: 177054.0481717717, Loss: 3.9756455421447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 70.25868901035398, Tokens per sec: 174108.7710011757, Loss: 3.9743731021881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 71.45408557085071, Tokens per sec: 177071.0953618885, Loss: 3.979422092437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 70.36500373658667, Tokens per sec: 174372.23060431422, Loss: 3.9648513793945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 70.22677202828135, Tokens per sec: 174029.67720365388, Loss: 3.9263715744018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 71.47198081318153, Tokens per sec: 177115.44174370795, Loss: 3.922959089279175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 70.36478715642751, Tokens per sec: 174371.6938948215, Loss: 3.93768572807312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 71.46386724640476, Tokens per sec: 177095.33543145197, Loss: 3.9194202423095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 70.5767949840116, Tokens per sec: 174897.07264616506, Loss: 3.8986778259277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 71.4333616476317, Tokens per sec: 177019.73919722319, Loss: 3.8868370056152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 70.33690910964148, Tokens per sec: 174302.60902386537, Loss: 3.89249324798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 70.2578960132675, Tokens per sec: 174106.80586703954, Loss: 3.867856979370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 71.49068622549443, Tokens per sec: 177161.7958159906, Loss: 3.864773988723755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 71.49586643836733, Tokens per sec: 177174.6329541372, Loss: 3.86422061920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 69.2490602907767, Tokens per sec: 171606.7998142783, Loss: 3.8609912395477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 71.4473787125997, Tokens per sec: 177054.47502832106, Loss: 3.8318569660186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 71.47450770149493, Tokens per sec: 177121.7036513083, Loss: 3.8541150093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 71.42259801682485, Tokens per sec: 176993.065734372, Loss: 3.8348324298858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 69.28674867269844, Tokens per sec: 171700.19577639832, Loss: 3.821932077407837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 71.48895731220613, Tokens per sec: 177157.51137840666, Loss: 3.7875680923461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 71.4685185023491, Tokens per sec: 177106.86175606019, Loss: 3.800515651702881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 69.39824914294793, Tokens per sec: 171976.5062244674, Loss: 3.7996599674224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 71.43549464923323, Tokens per sec: 177025.0250101613, Loss: 3.817782402038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 71.39648709466856, Tokens per sec: 176928.3599929132, Loss: 3.8010289669036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 71.46246010392676, Tokens per sec: 177091.84837738195, Loss: 3.7915406227111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 69.27241362352963, Tokens per sec: 171664.67194542201, Loss: 3.8051555156707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 71.40807478199571, Tokens per sec: 176957.07555859903, Loss: 3.7758750915527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 71.41509181715293, Tokens per sec: 176974.4645447088, Loss: 3.7867324352264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 70.33843751964407, Tokens per sec: 174306.39659222006, Loss: 3.7502923011779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 70.39968770028386, Tokens per sec: 174458.18128710985, Loss: 3.756230592727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 71.45625128901322, Tokens per sec: 177076.46225006875, Loss: 3.727635622024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 70.33126467918964, Tokens per sec: 174288.62150342038, Loss: 3.772068500518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 71.4387102860038, Tokens per sec: 177032.99371231042, Loss: 3.756934404373169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 70.54151651925196, Tokens per sec: 174809.64872424697, Loss: 3.7333366870880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 71.47353610305525, Tokens per sec: 177119.29592334435, Loss: 3.7354934215545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 70.31741458119224, Tokens per sec: 174254.2994348686, Loss: 3.733400344848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 71.43411225682763, Tokens per sec: 177021.59928949716, Loss: 3.729809045791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 70.41057935515818, Tokens per sec: 174485.17200770325, Loss: 3.725726842880249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 70.29828711184274, Tokens per sec: 174206.8994587558, Loss: 3.7346508502960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 71.41856482053339, Tokens per sec: 176983.07102966853, Loss: 3.6823890209198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 70.49576809029453, Tokens per sec: 174696.279077685, Loss: 3.7319891452789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 70.29260530387695, Tokens per sec: 174192.81931271378, Loss: 3.7052111625671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 71.42557856963806, Tokens per sec: 177000.45187257815, Loss: 3.735996723175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 71.39464854133756, Tokens per sec: 176923.80385522562, Loss: 3.7125039100646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 70.42514797082751, Tokens per sec: 174521.27464219733, Loss: 3.681244373321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 70.31020783778087, Tokens per sec: 174236.44033080686, Loss: 3.710787773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 71.42843077867354, Tokens per sec: 177007.51996076485, Loss: 3.6968562602996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 71.47410456222367, Tokens per sec: 177120.70462778467, Loss: 3.6467201709747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 69.36843587394945, Tokens per sec: 171902.62565968643, Loss: 3.684784412384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 71.48909781267132, Tokens per sec: 177157.8595540368, Loss: 3.6562142372131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 71.46146382187527, Tokens per sec: 177089.37948070883, Loss: 3.6607213020324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 71.46400010484798, Tokens per sec: 177095.66466930986, Loss: 3.674797534942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 69.29411151426817, Tokens per sec: 171718.44170889212, Loss: 3.6739585399627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 71.43686040041425, Tokens per sec: 177028.4094920378, Loss: 3.6458733081817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 71.41547678742428, Tokens per sec: 176975.41854344922, Loss: 3.6631147861480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 70.29005364769786, Tokens per sec: 174186.49602761384, Loss: 3.6384472846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 71.47504738260409, Tokens per sec: 177123.04104054757, Loss: 3.62502384185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 71.45910678742082, Tokens per sec: 177083.5384896819, Loss: 3.650456190109253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 71.51864613385032, Tokens per sec: 177231.0835489347, Loss: 3.642221689224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 69.23121901295049, Tokens per sec: 171562.58716244597, Loss: 3.628877878189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 71.44511070850989, Tokens per sec: 177048.85466434524, Loss: 3.5958333015441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 71.47901542278375, Tokens per sec: 177132.87428122846, Loss: 3.586714267730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 69.45955644721093, Tokens per sec: 172128.43247798816, Loss: 3.6136727333068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 71.43428321082936, Tokens per sec: 177022.0229323456, Loss: 3.6126747131347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 71.46177821291145, Tokens per sec: 177090.15857632953, Loss: 3.6169023513793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 71.41867455256093, Tokens per sec: 176983.34295772074, Loss: 3.605381488800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 69.53235650892086, Tokens per sec: 172308.8390504942, Loss: 3.5720417499542236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 71.50705322883061, Tokens per sec: 177202.35505323194, Loss: 3.5816967487335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 71.49505406339412, Tokens per sec: 177172.6197994631, Loss: 3.5874738693237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 70.44938349974271, Tokens per sec: 174581.33295261313, Loss: 3.5861685276031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 70.55642451573921, Tokens per sec: 174846.59238179374, Loss: 3.554197311401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 71.53696271037957, Tokens per sec: 177276.4740433116, Loss: 3.5409228801727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 70.37466794086698, Tokens per sec: 174396.1795671208, Loss: 3.5564708709716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 71.49711484120665, Tokens per sec: 177177.72663389597, Loss: 3.55021333694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 70.41558321882276, Tokens per sec: 174497.57213308517, Loss: 3.5589826107025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 70.39276862432419, Tokens per sec: 174441.03505468267, Loss: 3.541677236557007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 71.49711381774716, Tokens per sec: 177177.7240976506, Loss: 3.537022590637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 70.58678780375412, Tokens per sec: 174921.835954287, Loss: 3.542119264602661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 71.51785487178067, Tokens per sec: 177229.12271436138, Loss: 3.5284647941589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 70.34874931326321, Tokens per sec: 174331.95035274615, Loss: 3.530156135559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 71.5044322853358, Tokens per sec: 177195.86006653102, Loss: 3.5238523483276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 70.48126417655712, Tokens per sec: 174660.3367817057, Loss: 3.5149388313293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 70.36993389328232, Tokens per sec: 174384.44807570762, Loss: 3.5277180671691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 71.51380209104465, Tokens per sec: 177219.07947724685, Loss: 3.4919309616088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 71.5144675970124, Tokens per sec: 177220.72867434646, Loss: 3.496771812438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 70.36789027449021, Tokens per sec: 174379.38376321754, Loss: 3.5015676021575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 70.29949172732658, Tokens per sec: 174209.88462862413, Loss: 3.4715311527252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 71.51230999777499, Tokens per sec: 177215.381905757, Loss: 3.5140106678009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 71.48241886448193, Tokens per sec: 177141.30838467664, Loss: 3.4855072498321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 69.30175081093994, Tokens per sec: 171737.37272758273, Loss: 3.4808943271636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 71.46393349710893, Tokens per sec: 177095.499607995, Loss: 3.493018627166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 71.48024409986873, Tokens per sec: 177135.919078395, Loss: 3.4790289402008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 69.20525496212659, Tokens per sec: 171498.245384328, Loss: 3.463843822479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 71.44574369849556, Tokens per sec: 177050.42328325936, Loss: 3.4851772785186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 71.47414762592723, Tokens per sec: 177120.81134438596, Loss: 3.4334261417388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 71.5569674759523, Tokens per sec: 177326.0480561077, Loss: 3.477524757385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 69.63845353647469, Tokens per sec: 172571.759172325, Loss: 3.4636032581329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 71.49230335218105, Tokens per sec: 177165.80323406126, Loss: 3.4568240642547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 71.50655415822112, Tokens per sec: 177201.11830128444, Loss: 3.4639859199523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 69.25395328703888, Tokens per sec: 171618.925198024, Loss: 3.4758243560791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 71.48040960976206, Tokens per sec: 177136.3292301431, Loss: 3.4495813846588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 71.51916026118343, Tokens per sec: 177232.35761310192, Loss: 3.4507360458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 70.44060466193136, Tokens per sec: 174559.5780254478, Loss: 3.4615280628204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 70.43722327980805, Tokens per sec: 174551.19858805623, Loss: 3.443114995956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 71.50621850275316, Tokens per sec: 177200.28651005932, Loss: 3.4166483879089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 71.47820238848801, Tokens per sec: 177130.85949268052, Loss: 3.440425157546997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 70.38864407751439, Tokens per sec: 174430.81397333162, Loss: 3.4152140617370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 70.3738331572838, Tokens per sec: 174394.11088144267, Loss: 3.42763090133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 71.45963916609334, Tokens per sec: 177084.85778267877, Loss: 3.4192633628845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 70.25964838239291, Tokens per sec: 174111.14843077172, Loss: 3.4253053665161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 71.5395100297917, Tokens per sec: 177282.7865814253, Loss: 3.3981454372406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 70.39633507792531, Tokens per sec: 174449.87311958353, Loss: 3.4083614349365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 70.37800563607121, Tokens per sec: 174404.45074351435, Loss: 3.4135823249816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 71.54519391723349, Tokens per sec: 177296.87188063795, Loss: 3.41562557220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 70.4299914620862, Tokens per sec: 174533.277347091, Loss: 3.409961700439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 71.50417503689722, Tokens per sec: 177195.22257656197, Loss: 3.4116716384887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 70.3571885702934, Tokens per sec: 174352.86376132784, Loss: 3.432392120361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 71.41527785236504, Tokens per sec: 176974.92556045792, Loss: 3.4200496673583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 70.5440999727204, Tokens per sec: 174816.05080653267, Loss: 3.3999876976013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 70.40120740007352, Tokens per sec: 174461.94727059736, Loss: 3.4134621620178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 71.54887227460922, Tokens per sec: 177305.98725541952, Loss: 3.381826877593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 71.53305691325927, Tokens per sec: 177266.79504778844, Loss: 3.411043643951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 69.39523503733177, Tokens per sec: 171969.0369387194, Loss: 3.3858087062835693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 71.56652998595426, Tokens per sec: 177349.74500929, Loss: 3.39111328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 71.52286123769404, Tokens per sec: 177241.52904059144, Loss: 3.384773015975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 71.46463287974183, Tokens per sec: 177097.23275520268, Loss: 3.3566763401031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 69.40317137902356, Tokens per sec: 171988.70406769184, Loss: 3.3744957447052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 71.50910677847368, Tokens per sec: 177207.44397546464, Loss: 3.3563859462738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 71.5146620135665, Tokens per sec: 177221.21045998426, Loss: 3.3540596961975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 69.45639317245666, Tokens per sec: 172120.59353468358, Loss: 3.393871307373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 71.4944834945008, Tokens per sec: 177171.2058668921, Loss: 3.378667116165161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 71.52152035080181, Tokens per sec: 177238.20617516155, Loss: 3.3776092529296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 71.55213047353551, Tokens per sec: 177314.0614312791, Loss: 3.3652167320251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 69.6644283561131, Tokens per sec: 172636.12763675163, Loss: 3.351569175720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 71.51640463239605, Tokens per sec: 177225.5288614092, Loss: 3.359278917312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 71.46077709821469, Tokens per sec: 177087.67770382905, Loss: 3.3694705963134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 69.2894615636984, Tokens per sec: 171706.91861914587, Loss: 3.3579201698303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 71.512634457179, Tokens per sec: 177216.18595190227, Loss: 3.3688340187072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 71.53625776469562, Tokens per sec: 177274.72711025062, Loss: 3.328540325164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 70.38836898516719, Tokens per sec: 174430.13226419134, Loss: 3.3241381645202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 70.55077150940703, Tokens per sec: 174832.58360937258, Loss: 3.337083578109741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 71.4990900715587, Tokens per sec: 177182.6214722979, Loss: 3.342658519744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 70.39433771605555, Tokens per sec: 174444.92343684268, Loss: 3.34505033493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 71.50397724908031, Tokens per sec: 177194.7324365635, Loss: 3.3296546936035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 69.95658147605211, Tokens per sec: 173360.1152512834, Loss: 3.321046829223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 71.48977959769306, Tokens per sec: 177159.54909242553, Loss: 3.3227128982543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 70.26255954391897, Tokens per sec: 174118.36260972416, Loss: 3.361607074737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 71.52282529428375, Tokens per sec: 177241.43996886123, Loss: 3.338367462158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 70.50889865936423, Tokens per sec: 174728.8180742913, Loss: 3.314939498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 70.33247759142371, Tokens per sec: 174291.6272335497, Loss: 3.3146491050720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 71.5128790449052, Tokens per sec: 177216.79206723138, Loss: 3.317182779312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 70.42477030591161, Tokens per sec: 174520.3387469325, Loss: 3.305738687515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 71.51312931414415, Tokens per sec: 177217.4122619749, Loss: 3.3195419311523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 70.41937494092045, Tokens per sec: 174506.9684381359, Loss: 3.3093678951263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 71.53748577163617, Tokens per sec: 177277.77024672544, Loss: 3.311990261077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 70.54031504258946, Tokens per sec: 174806.6713327237, Loss: 3.3091323375701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 70.40531228794897, Tokens per sec: 174472.11963494346, Loss: 3.2884154319763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 71.50184858607298, Tokens per sec: 177189.45737513978, Loss: 3.286916971206665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 71.49073476342669, Tokens per sec: 177161.91609833366, Loss: 3.32275652885437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 69.29744288083718, Tokens per sec: 171726.69720223043, Loss: 3.3258066177368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 71.51121758906143, Tokens per sec: 177212.67479662603, Loss: 3.2832179069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 71.45175709387021, Tokens per sec: 177065.32513942794, Loss: 3.2784385681152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 70.39777942965888, Tokens per sec: 174453.45238228777, Loss: 3.3362364768981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 70.5629396715647, Tokens per sec: 174862.737655633, Loss: 3.2857398986816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 71.53553215810433, Tokens per sec: 177272.92897718522, Loss: 3.30153226852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 71.50971313430135, Tokens per sec: 177208.9465920012, Loss: 3.2884202003479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 70.23958424650915, Tokens per sec: 174061.42729180472, Loss: 3.3012535572052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 71.55509519439977, Tokens per sec: 177321.4083361717, Loss: 3.285649538040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 71.52537950153658, Tokens per sec: 177247.76957580287, Loss: 3.2690305709838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 70.38268498564462, Tokens per sec: 174416.04668723018, Loss: 3.2918813228607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 70.55109854634942, Tokens per sec: 174833.39404294192, Loss: 3.302370071411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 71.52289578011569, Tokens per sec: 177241.61464051742, Loss: 3.276371479034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 71.44615031664298, Tokens per sec: 177051.43092782138, Loss: 3.2845921516418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 69.26091228849648, Tokens per sec: 171636.17037023325, Loss: 3.2722971439361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 71.44298027547237, Tokens per sec: 177043.5752165917, Loss: 3.2635443210601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 71.45108380600131, Tokens per sec: 177063.65665792, Loss: 3.2814266681671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 70.38109118906509, Tokens per sec: 174412.09708373493, Loss: 3.236046075820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 70.50676476445301, Tokens per sec: 174723.53004763337, Loss: 3.2638370990753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 71.49495148235718, Tokens per sec: 177172.36559235334, Loss: 3.2563657760620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 70.3325725097666, Tokens per sec: 174291.86245165952, Loss: 3.273975133895874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 71.55553238078984, Tokens per sec: 177322.49173220634, Loss: 3.2606661319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 70.47694892715396, Tokens per sec: 174649.6431183207, Loss: 3.2653071880340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 71.46629097539807, Tokens per sec: 177101.34169889317, Loss: 3.25689697265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 70.63619634615543, Tokens per sec: 175044.27576515774, Loss: 3.2586138248443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 71.52428089593295, Tokens per sec: 177245.0471100957, Loss: 3.2617313861846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 70.55819362330223, Tokens per sec: 174850.97642521825, Loss: 3.230813503265381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 70.39144622840463, Tokens per sec: 174437.75801192006, Loss: 3.2472872734069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 71.50665119998811, Tokens per sec: 177201.35878147307, Loss: 3.2503230571746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 70.46589508360165, Tokens per sec: 174622.25047631667, Loss: 3.2642009258270264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 70.29393471230412, Tokens per sec: 174196.11373324273, Loss: 3.2520816326141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 71.49485486818367, Tokens per sec: 177172.12617178843, Loss: 3.2566347122192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 71.50365459683916, Tokens per sec: 177193.9328687669, Loss: 3.228947162628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 70.50608948771664, Tokens per sec: 174721.85663749275, Loss: 3.2651045322418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 70.32742140865099, Tokens per sec: 174279.0974556544, Loss: 3.2482192516326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 71.54083060410593, Tokens per sec: 177286.05911004922, Loss: 3.216982364654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 71.52549682856352, Tokens per sec: 177248.0603251003, Loss: 3.243274211883545 +------------------------------------------------------------------ +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 69.3317936158482, Tokens per sec: 171811.82210763358, Loss: 3.2273778915405273 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/model.pt +[rank0]:[2024-08-26 02:31:36,201] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006479980000221985, 'preprocessing_with_comm': 0.006995947999712371, 'state_converting': 0.9873190550001709, : 1.0021964229999867}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200, took 4.39s + eval ppl=21.10089111328125, eval loss=3.0493152141571045 +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 69.35413184764829, Tokens per sec: 171867.17870679358, Loss: 3.221250057220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 71.49440237710226, Tokens per sec: 177171.00484904269, Loss: 3.222559690475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 68.48128142286102, Tokens per sec: 169704.15920175883, Loss: 3.2453701496124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 71.46130173707897, Tokens per sec: 177088.97781672847, Loss: 3.232313871383667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 71.46284627957176, Tokens per sec: 177092.80536317104, Loss: 3.2306270599365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 71.4627774457605, Tokens per sec: 177092.63478540405, Loss: 3.2017197608947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 69.57945297008483, Tokens per sec: 172425.549269362, Loss: 3.199495792388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 71.4916023562908, Tokens per sec: 177164.06608902413, Loss: 3.222843647003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 71.43208353080026, Tokens per sec: 177016.57188291868, Loss: 3.2207858562469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 69.28707616482036, Tokens per sec: 171701.00733795273, Loss: 3.239039897918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 71.51207962276567, Tokens per sec: 177214.81101110598, Loss: 3.2136435508728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 71.5162613679621, Tokens per sec: 177225.17383636, Loss: 3.218442678451538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 70.38634352532024, Tokens per sec: 174425.1129515678, Loss: 3.236889362335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 70.46558954417665, Tokens per sec: 174621.4933159636, Loss: 3.220334529876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 71.49666113393606, Tokens per sec: 177176.60229729943, Loss: 3.1642749309539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 71.48316147120497, Tokens per sec: 177143.1486459412, Loss: 3.197317600250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 70.24501247984028, Tokens per sec: 174074.87905196854, Loss: 3.2062010765075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 70.45691161659043, Tokens per sec: 174599.9884554523, Loss: 3.2118918895721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 71.50516667714571, Tokens per sec: 177197.67997033588, Loss: 3.2132253646850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 70.40485165917336, Tokens per sec: 174470.97814609506, Loss: 3.1983983516693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 71.49907303874964, Tokens per sec: 177182.5792631208, Loss: 3.177658796310425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 70.44037595741682, Tokens per sec: 174559.01127046687, Loss: 3.1849911212921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 71.53655877928182, Tokens per sec: 177275.47305755471, Loss: 3.2087507247924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 70.70384118861803, Tokens per sec: 175211.9071364746, Loss: 3.215726852416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 70.39727073971277, Tokens per sec: 174452.1917925645, Loss: 3.2085459232330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 71.51231389050035, Tokens per sec: 177215.39155235927, Loss: 3.2130866050720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 70.33752138144423, Tokens per sec: 174304.1263008399, Loss: 3.163285493850708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 71.4410881377441, Tokens per sec: 177038.8862908649, Loss: 3.168853282928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 70.45628956665755, Tokens per sec: 174598.44694719414, Loss: 3.1571903228759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 70.32683794517294, Tokens per sec: 174277.65156888196, Loss: 3.1923627853393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 71.48430992022021, Tokens per sec: 177145.99462911306, Loss: 3.1836354732513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 71.50005505524678, Tokens per sec: 177185.0128081787, Loss: 3.190718412399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 69.4238651369308, Tokens per sec: 172039.9854793916, Loss: 3.179842948913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 71.50881654186699, Tokens per sec: 177206.72473718104, Loss: 3.1787240505218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 71.43527622912183, Tokens per sec: 177024.4837410643, Loss: 3.1785836219787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 71.51379826578369, Tokens per sec: 177219.0699978288, Loss: 3.1913952827453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 69.32120981152035, Tokens per sec: 171785.59427460688, Loss: 3.156024694442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 71.46451194196261, Tokens per sec: 177096.9330580634, Loss: 3.1889820098876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 71.4983348229659, Tokens per sec: 177180.74988308747, Loss: 3.132686138153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 69.34756027362857, Tokens per sec: 171850.89362245408, Loss: 3.174607753753662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 71.49082006214566, Tokens per sec: 177162.12747795568, Loss: 3.1647238731384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 71.54875042304482, Tokens per sec: 177305.68529381987, Loss: 3.159632444381714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 71.49648334475046, Tokens per sec: 177176.16171611246, Loss: 3.1559739112854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 69.10324360431557, Tokens per sec: 171245.45000219354, Loss: 3.152059555053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 71.51447717967747, Tokens per sec: 177220.7524212461, Loss: 3.1552491188049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 71.54761456060082, Tokens per sec: 177302.87050155806, Loss: 3.1658873558044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 70.31389324869602, Tokens per sec: 174245.57318503613, Loss: 3.1694304943084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 70.43640092728707, Tokens per sec: 174549.1607079196, Loss: 3.157356023788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 71.47457927897348, Tokens per sec: 177121.88102818505, Loss: 3.176845073699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 70.29543811763482, Tokens per sec: 174199.83933726596, Loss: 3.172536849975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 70.41519748061336, Tokens per sec: 174496.6162313097, Loss: 3.1736090183258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 71.49272724063434, Tokens per sec: 177166.8536763437, Loss: 3.1665844917297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 71.50713072422836, Tokens per sec: 177202.54709536434, Loss: 3.1286356449127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 70.61603336654784, Tokens per sec: 174994.30968055455, Loss: 3.1735639572143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 70.4895680173051, Tokens per sec: 174680.91461382277, Loss: 3.177239179611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 71.49484156407988, Tokens per sec: 177172.09320275366, Loss: 3.144329786300659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 70.07661023484589, Tokens per sec: 173657.5597378337, Loss: 3.1554646492004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 71.5503480631975, Tokens per sec: 177309.6444221128, Loss: 3.152677297592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 70.4507681278303, Tokens per sec: 174584.76421354353, Loss: 3.1560049057006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 70.3656179097684, Tokens per sec: 174373.7525931153, Loss: 3.134113311767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 71.52496885998447, Tokens per sec: 177246.75196080084, Loss: 3.1319918632507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 70.52658026682772, Tokens per sec: 174772.63504537567, Loss: 3.136312484741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 71.49565330666238, Tokens per sec: 177174.10479029393, Loss: 3.136500358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 70.32503626373367, Tokens per sec: 174273.18680380387, Loss: 3.136805534362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 71.49832408595988, Tokens per sec: 177180.72327560376, Loss: 3.1408474445343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 70.52764182259683, Tokens per sec: 174775.26569751158, Loss: 3.1276791095733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 70.3289736870634, Tokens per sec: 174282.9441725579, Loss: 3.1080334186553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 71.45894617934876, Tokens per sec: 177083.1404851869, Loss: 3.1231634616851807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 71.50413969169763, Tokens per sec: 177195.1349872638, Loss: 3.1233227252960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 69.44235142028421, Tokens per sec: 172085.79652597356, Loss: 3.1569008827209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 71.52524196019988, Tokens per sec: 177247.42873321913, Loss: 3.137782096862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 71.45001753729089, Tokens per sec: 177061.0143265964, Loss: 3.128103494644165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 70.46350617320017, Tokens per sec: 174616.3304931816, Loss: 3.142050266265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 70.39580746400654, Tokens per sec: 174448.56563417133, Loss: 3.1267123222351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 71.51192839710802, Tokens per sec: 177214.43625726746, Loss: 3.1345479488372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 71.46380520159275, Tokens per sec: 177095.18167757298, Loss: 3.1232967376708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 69.47451844922259, Tokens per sec: 172165.50996717365, Loss: 3.122657299041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 71.55007022653099, Tokens per sec: 177308.95591224695, Loss: 3.116798162460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 71.5077844100231, Tokens per sec: 177204.1670007732, Loss: 3.1300573348999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 70.33612300797402, Tokens per sec: 174300.6609773372, Loss: 3.0863590240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 70.16454503360879, Tokens per sec: 173875.47185598593, Loss: 3.116863489151001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 71.44928759482397, Tokens per sec: 177059.20544875355, Loss: 3.1088674068450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 71.50845698978219, Tokens per sec: 177205.83372750695, Loss: 3.128765821456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 69.76434649445682, Tokens per sec: 172883.7357904589, Loss: 3.118283987045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 70.48766429307526, Tokens per sec: 174676.19697546866, Loss: 3.116234302520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 71.47802535694669, Tokens per sec: 177130.42078901952, Loss: 3.1102895736694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 70.15790418716868, Tokens per sec: 173859.01510695773, Loss: 3.0815658569335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 70.47255898367096, Tokens per sec: 174638.76435477764, Loss: 3.1009857654571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 71.51355623158425, Tokens per sec: 177218.47021042032, Loss: 3.1028523445129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 71.53278545438273, Tokens per sec: 177266.12234278876, Loss: 3.115802764892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 70.64807261761402, Tokens per sec: 175073.70647411142, Loss: 3.102414846420288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 70.56120677640196, Tokens per sec: 174858.44335052557, Loss: 3.107755422592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 71.49503821038151, Tokens per sec: 177172.5805139515, Loss: 3.099989175796509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 70.40594430826043, Tokens per sec: 174473.68585089804, Loss: 3.1058692932128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 71.53875828664245, Tokens per sec: 177280.9236790901, Loss: 3.1265034675598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 70.45542044224909, Tokens per sec: 174596.29316116805, Loss: 3.0886659622192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 70.35034882466145, Tokens per sec: 174335.9141181917, Loss: 3.0882482528686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 71.45217367431546, Tokens per sec: 177066.3574716614, Loss: 3.087311267852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 71.4695088933551, Tokens per sec: 177109.31605407363, Loss: 3.0925028324127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 69.08285506083007, Tokens per sec: 171194.92494545298, Loss: 3.0849857330322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 71.5405499025836, Tokens per sec: 177285.36350075505, Loss: 3.0822126865386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 71.46219739883966, Tokens per sec: 177091.1973652368, Loss: 3.0863754749298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 70.42163409647546, Tokens per sec: 174512.56687446774, Loss: 3.0748772621154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 70.34541384093585, Tokens per sec: 174323.6846848863, Loss: 3.0918846130371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 71.4752050042176, Tokens per sec: 177123.4316442693, Loss: 3.067885398864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 71.5436717809255, Tokens per sec: 177293.0998591896, Loss: 3.0715532302856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 69.43722221615897, Tokens per sec: 172073.08579312274, Loss: 3.069470167160034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 71.5018193171299, Tokens per sec: 177189.3848434746, Loss: 3.10295033454895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 71.45255567899348, Tokens per sec: 177067.30412133478, Loss: 3.0856616497039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 70.42367927377805, Tokens per sec: 174517.63504911843, Loss: 3.0755844116210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 70.35442045625945, Tokens per sec: 174346.00406981874, Loss: 3.0809895992279053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 71.53250092106569, Tokens per sec: 177265.41723788515, Loss: 3.06949520111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 71.48600053056309, Tokens per sec: 177150.18414777884, Loss: 3.091357707977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 69.45607866589063, Tokens per sec: 172119.81415276724, Loss: 3.0847995281219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 71.523592065837, Tokens per sec: 177243.34011323703, Loss: 3.072479724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 71.52427342674775, Tokens per sec: 177245.02860063198, Loss: 3.047231435775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 70.36598172590783, Tokens per sec: 174374.65416958646, Loss: 3.074632167816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 70.38827896017354, Tokens per sec: 174429.90917233998, Loss: 3.078115701675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 71.50134087804719, Tokens per sec: 177188.1992187232, Loss: 3.0713648796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 70.40392057061241, Tokens per sec: 174468.67080607315, Loss: 3.076369524002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 71.50682783251922, Tokens per sec: 177201.79649634287, Loss: 3.0633554458618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 70.40232485723705, Tokens per sec: 174464.71645254668, Loss: 3.0487561225891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 71.49803016156855, Tokens per sec: 177179.99489858362, Loss: 3.0730795860290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 70.31616084497027, Tokens per sec: 174251.19253840006, Loss: 3.07660174369812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 70.49793289839651, Tokens per sec: 174701.643710635, Loss: 3.0477113723754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 71.49402412451514, Tokens per sec: 177170.06749746384, Loss: 3.067629337310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 70.38966640938142, Tokens per sec: 174433.34742431727, Loss: 3.0580763816833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 71.5168159677831, Tokens per sec: 177226.54819581067, Loss: 3.0349905490875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 70.50378921390084, Tokens per sec: 174716.15630558113, Loss: 3.051729202270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 71.48077722782546, Tokens per sec: 177137.2402282007, Loss: 3.0349481105804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 70.34734211821403, Tokens per sec: 174328.4631683989, Loss: 3.0523688793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 71.45503931045923, Tokens per sec: 177073.4588337015, Loss: 3.0525434017181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 70.48369860938911, Tokens per sec: 174666.3695744383, Loss: 3.0542428493499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 70.35157988596016, Tokens per sec: 174338.96482370247, Loss: 3.036895990371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 71.42617782444297, Tokens per sec: 177001.9368919981, Loss: 3.0427732467651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 70.55202213700967, Tokens per sec: 174835.68280233993, Loss: 3.0295379161834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 70.29883852362741, Tokens per sec: 174208.26591790124, Loss: 3.0450246334075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 71.47112833286367, Tokens per sec: 177113.32920357186, Loss: 3.0511159896850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 71.52884600004876, Tokens per sec: 177256.3599409824, Loss: 3.0323305130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 70.52414473103137, Tokens per sec: 174766.59951937592, Loss: 3.0436925888061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 70.62430353865727, Tokens per sec: 175014.80407807635, Loss: 3.047546625137329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 71.53128897366702, Tokens per sec: 177262.41389872425, Loss: 3.0353739261627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 71.48551181129018, Tokens per sec: 177148.97304758368, Loss: 3.0465784072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 68.9508120244983, Tokens per sec: 170867.70775568165, Loss: 3.063524007797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 71.5280229799021, Tokens per sec: 177254.32040639588, Loss: 3.0568222999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 71.52994322861902, Tokens per sec: 177259.07899424923, Loss: 2.996908187866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 69.41382881651596, Tokens per sec: 172015.11437181165, Loss: 3.0614066123962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 71.52181720268227, Tokens per sec: 177238.9418068223, Loss: 3.035944700241089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 71.50600114687619, Tokens per sec: 177199.74787825244, Loss: 3.037625789642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 71.50426561154138, Tokens per sec: 177195.4470305078, Loss: 3.0366551876068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 69.37907354467048, Tokens per sec: 171928.9869795693, Loss: 3.0318803787231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 71.51308636767403, Tokens per sec: 177217.30583589102, Loss: 3.033582925796509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 71.40180791022341, Tokens per sec: 176941.54555999432, Loss: 3.0376181602478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 70.36825490910942, Tokens per sec: 174380.2873679718, Loss: 3.0342745780944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 70.53108635148693, Tokens per sec: 174783.80161954742, Loss: 3.027435064315796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 71.48793245259006, Tokens per sec: 177154.97166338042, Loss: 3.0296707153320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 71.49001326563328, Tokens per sec: 177160.12814732202, Loss: 3.0183255672454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 70.60156622537195, Tokens per sec: 174958.45851103353, Loss: 3.014592409133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 70.4568535770091, Tokens per sec: 174599.84462697618, Loss: 3.01664137840271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 71.39818412562364, Tokens per sec: 176932.5654225633, Loss: 3.0322518348693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 70.37297821685418, Tokens per sec: 174391.9922448786, Loss: 3.021397113800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 70.54147006999243, Tokens per sec: 174809.5336178651, Loss: 3.017801523208618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 71.51849606872234, Tokens per sec: 177230.7116710165, Loss: 3.019887924194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 70.38909661453317, Tokens per sec: 174431.93540991523, Loss: 3.025576114654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 71.52179279666053, Tokens per sec: 177238.8813260119, Loss: 3.030365228652954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 70.43873933379804, Tokens per sec: 174554.95553685055, Loss: 3.0068752765655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 70.39892754735796, Tokens per sec: 174456.29754442172, Loss: 3.0201070308685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 71.51673845139571, Tokens per sec: 177226.35610166355, Loss: 3.0131378173828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 71.51231220977164, Tokens per sec: 177215.38738732837, Loss: 3.010955333709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 70.56924715431863, Tokens per sec: 174878.36829272384, Loss: 3.011632204055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 70.32935392205651, Tokens per sec: 174283.8864367572, Loss: 3.006523609161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 71.47756395963405, Tokens per sec: 177129.27739565168, Loss: 3.01167368888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 71.46933502109185, Tokens per sec: 177108.88517945167, Loss: 3.013762950897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 69.31199717709939, Tokens per sec: 171762.76435165628, Loss: 3.027559518814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 71.48795515592431, Tokens per sec: 177155.0279247437, Loss: 3.0183990001678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 71.5000205252278, Tokens per sec: 177184.9272389879, Loss: 2.9902617931365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 70.57210983554104, Tokens per sec: 174885.46233214333, Loss: 3.0141940116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 70.37653636007252, Tokens per sec: 174400.8097157355, Loss: 2.9886374473571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 71.49663010235288, Tokens per sec: 177176.52539761792, Loss: 3.006110191345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 71.50849373005241, Tokens per sec: 177205.9247739438, Loss: 2.9944725036621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 69.32141684740536, Tokens per sec: 171786.1073323368, Loss: 2.996765375137329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 71.50843943095771, Tokens per sec: 177205.79021480563, Loss: 3.0084047317504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 71.44696443458588, Tokens per sec: 177053.44840176636, Loss: 3.0034420490264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 70.19236796002123, Tokens per sec: 173944.42013258435, Loss: 3.006997585296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 71.52560336245159, Tokens per sec: 177248.3243278106, Loss: 2.985704183578491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 71.53633986636676, Tokens per sec: 177274.93056723595, Loss: 3.0084197521209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 70.39464828184245, Tokens per sec: 174445.69305307427, Loss: 2.976226329803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 69.40002377226246, Tokens per sec: 171980.9039514003, Loss: 3.0079030990600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 71.51422680441104, Tokens per sec: 177220.13196375477, Loss: 2.9762043952941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 71.53121476014354, Tokens per sec: 177262.22998943774, Loss: 3.003453254699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 70.38351152067641, Tokens per sec: 174418.0949320893, Loss: 2.9957945346832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 70.4364235017853, Tokens per sec: 174549.21665001303, Loss: 2.9806573390960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 71.5145015613914, Tokens per sec: 177220.8128418191, Loss: 2.998204469680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 70.35204474457136, Tokens per sec: 174340.11679454942, Loss: 2.971815586090088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 71.51375413034692, Tokens per sec: 177218.96062535458, Loss: 3.0051093101501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 70.50676498980168, Tokens per sec: 174723.5306060722, Loss: 2.9726932048797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 71.56758111198864, Tokens per sec: 177352.3498153943, Loss: 3.027679204940796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 70.40620795614056, Tokens per sec: 174474.33919938823, Loss: 2.9990954399108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 70.43357335440085, Tokens per sec: 174542.1536708247, Loss: 2.9959161281585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 71.53294249956855, Tokens per sec: 177266.51151805915, Loss: 2.971723794937134 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/model.pt +[rank0]:[2024-08-26 02:56:45,032] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006395178000275337, 'preprocessing_with_comm': 0.0013657199997396674, 'state_converting': 0.9746936370002004, : 0.9838315139995757}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400, took 4.56s + eval ppl=16.409910202026367, eval loss=2.7978854179382324 +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 66.38406715267068, Tokens per sec: 164507.0312130035, Loss: 2.98765230178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 71.5135131493584, Tokens per sec: 177218.3634479188, Loss: 2.997295379638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 70.13446481489845, Tokens per sec: 173800.92976041208, Loss: 2.964998245239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 70.3811276782038, Tokens per sec: 174412.18750784022, Loss: 2.988830327987671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 71.45448476183395, Tokens per sec: 177072.0846011195, Loss: 2.9777989387512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 70.33194292721824, Tokens per sec: 174290.3022767504, Loss: 2.990023374557495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 70.37405055083079, Tokens per sec: 174394.6496066, Loss: 2.961353302001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 71.52799020096059, Tokens per sec: 177254.23917656788, Loss: 2.964566230773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 71.40177161652909, Tokens per sec: 176941.4556202217, Loss: 3.009058713912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 70.4425558849208, Tokens per sec: 174564.41337096054, Loss: 2.9680087566375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 70.59805111427647, Tokens per sec: 174949.74767851748, Loss: 2.965174436569214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 71.46529807861803, Tokens per sec: 177098.88119129563, Loss: 2.998861074447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 71.47286277132235, Tokens per sec: 177117.62733313665, Loss: 2.992979049682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 69.29225329470482, Tokens per sec: 171713.83683611432, Loss: 2.998638391494751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 71.4867107601668, Tokens per sec: 177151.94417497507, Loss: 2.96687388420105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 71.48537872807275, Tokens per sec: 177148.64325271058, Loss: 2.9998042583465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 70.37302375230063, Tokens per sec: 174392.10508673103, Loss: 2.97322678565979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 70.38350208900144, Tokens per sec: 174418.07155935976, Loss: 2.9903242588043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 71.53735029095513, Tokens per sec: 177277.4345106734, Loss: 2.948862075805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 71.54022211574743, Tokens per sec: 177284.55120886632, Loss: 2.994220733642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 69.3086094001198, Tokens per sec: 171754.36906710643, Loss: 2.953341007232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 71.50455817724297, Tokens per sec: 177196.17204054512, Loss: 2.9726920127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 71.53150018202201, Tokens per sec: 177262.93729629443, Loss: 2.954932689666748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 69.9916114753853, Tokens per sec: 173446.92344850404, Loss: 2.9786877632141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 70.48709642971674, Tokens per sec: 174674.78974751098, Loss: 2.9482903480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 71.49633533259026, Tokens per sec: 177175.7949256746, Loss: 2.9438059329986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 71.49605440082598, Tokens per sec: 177175.0987458155, Loss: 2.964695692062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 69.60320489957141, Tokens per sec: 172484.4091671208, Loss: 2.932880163192749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 71.5424458044551, Tokens per sec: 177290.06175444345, Loss: 2.9348745346069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 71.52379982755448, Tokens per sec: 177243.85496965982, Loss: 2.932685613632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 70.37750443537958, Tokens per sec: 174403.20871298868, Loss: 2.969935894012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 70.41831559280342, Tokens per sec: 174504.34325680506, Loss: 2.974128484725952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 71.53604907034286, Tokens per sec: 177274.20994265488, Loss: 2.961108922958374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 70.34626348501432, Tokens per sec: 174325.79019650878, Loss: 2.960587501525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 71.53912773278121, Tokens per sec: 177281.8392073198, Loss: 2.97963285446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 70.47062365756518, Tokens per sec: 174633.96840349372, Loss: 2.9367575645446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 70.39016197162314, Tokens per sec: 174434.57548214297, Loss: 2.931633949279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 71.49859403776964, Tokens per sec: 177181.39224592602, Loss: 2.9480459690093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 71.44156656131919, Tokens per sec: 177040.07187718665, Loss: 2.94234037399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 70.42726490850697, Tokens per sec: 174526.520646967, Loss: 2.968292474746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 70.40669864849527, Tokens per sec: 174475.55518909745, Loss: 2.9542453289031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 71.50516353847878, Tokens per sec: 177197.67219237352, Loss: 2.9687886238098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 70.48782464708623, Tokens per sec: 174676.5943503723, Loss: 2.9720897674560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 70.43769577455856, Tokens per sec: 174552.3694820976, Loss: 2.959134578704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 71.51668823585008, Tokens per sec: 177226.2316620098, Loss: 2.907881021499634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 71.55067223932565, Tokens per sec: 177310.4477662693, Loss: 2.943277359008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 70.54610753444457, Tokens per sec: 174821.02576563635, Loss: 2.9588510990142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 70.34002152561948, Tokens per sec: 174310.32193351976, Loss: 2.935063600540161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 71.55588418596443, Tokens per sec: 177323.36354418352, Loss: 2.928619861602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 71.51015217134828, Tokens per sec: 177210.03457416737, Loss: 2.968322515487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 69.34225794501663, Tokens per sec: 171837.753868054, Loss: 2.9441685676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 71.5281840145975, Tokens per sec: 177254.71946811047, Loss: 2.9362030029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 71.5278406474431, Tokens per sec: 177253.8685664769, Loss: 2.9381630420684814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 69.37707333446807, Tokens per sec: 171924.03023834637, Loss: 2.959545373916626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 71.50316374340697, Tokens per sec: 177192.71647988993, Loss: 2.95563006401062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 71.42329952940473, Tokens per sec: 176994.80415982296, Loss: 2.955315351486206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 70.38236182860281, Tokens per sec: 174415.24586848187, Loss: 2.9312548637390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 70.47308432478225, Tokens per sec: 174640.0662079225, Loss: 2.9258642196655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 71.53116578064142, Tokens per sec: 177262.10861283596, Loss: 2.917935609817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 71.49525608855386, Tokens per sec: 177173.12044006385, Loss: 2.9556612968444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 70.39786962872896, Tokens per sec: 174453.6759055197, Loss: 2.929582118988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 70.54265734622123, Tokens per sec: 174812.47581914932, Loss: 2.9361824989318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 71.49872951652746, Tokens per sec: 177181.72797721208, Loss: 2.9236316680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 70.37126110255127, Tokens per sec: 174387.73704648024, Loss: 2.9251036643981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 70.39576245132253, Tokens per sec: 174448.45408778178, Loss: 2.957470178604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 71.51813646759648, Tokens per sec: 177229.82053981326, Loss: 2.91837215423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 71.55400624575631, Tokens per sec: 177318.70980147558, Loss: 2.9198052883148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 70.37888943903674, Tokens per sec: 174406.64090462186, Loss: 2.9256153106689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 70.5813088239471, Tokens per sec: 174908.25843876577, Loss: 2.9404778480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 71.54727474870413, Tokens per sec: 177302.02841024467, Loss: 2.932952404022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 70.39000619814546, Tokens per sec: 174434.1894583054, Loss: 2.906942844390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 71.52723233171852, Tokens per sec: 177252.3610930998, Loss: 2.940828323364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 70.38640796168382, Tokens per sec: 174425.27263197498, Loss: 2.9090347290039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 70.37455375095658, Tokens per sec: 174395.89659194375, Loss: 2.9122719764709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 71.48987832188043, Tokens per sec: 177159.79374183714, Loss: 2.930323839187622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 71.55693662410496, Tokens per sec: 177325.97160183138, Loss: 2.9236695766448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 70.52611870752439, Tokens per sec: 174771.49125057706, Loss: 2.9368021488189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 70.65955080798288, Tokens per sec: 175102.15069427248, Loss: 2.931122303009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 71.49057676439202, Tokens per sec: 177161.52455932068, Loss: 2.9192311763763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 70.43838054708225, Tokens per sec: 174554.06642384522, Loss: 2.9466207027435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 70.33250242582392, Tokens per sec: 174291.68877592916, Loss: 2.914832830429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 71.54574385131261, Tokens per sec: 177298.23467786622, Loss: 2.910162925720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 71.46244864460583, Tokens per sec: 177091.81997992235, Loss: 2.9231560230255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 69.46277296022097, Tokens per sec: 172136.4033515552, Loss: 2.9167611598968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 71.47961319090528, Tokens per sec: 177134.35561648326, Loss: 2.930227518081665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 71.47212201880203, Tokens per sec: 177115.7916667906, Loss: 2.9135401248931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 71.50515813997566, Tokens per sec: 177197.6588142881, Loss: 2.9062087535858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 69.3246574695821, Tokens per sec: 171794.1379510748, Loss: 2.9222445487976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 71.5186712429847, Tokens per sec: 177231.14577213564, Loss: 2.9374144077301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 71.54803742279367, Tokens per sec: 177303.91840065405, Loss: 2.9342691898345947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 70.41960181375717, Tokens per sec: 174507.53065401744, Loss: 2.9087486267089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 70.4092591365314, Tokens per sec: 174481.900360512, Loss: 2.9130635261535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 71.53319812835669, Tokens per sec: 177267.14499435594, Loss: 2.9081833362579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 71.53718480724454, Tokens per sec: 177277.02442380902, Loss: 2.8847150802612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 69.67030565960475, Tokens per sec: 172650.6922422423, Loss: 2.89837384223938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 71.50999303368366, Tokens per sec: 177209.640213504, Loss: 2.9179770946502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 71.55680657275256, Tokens per sec: 177325.64932025378, Loss: 2.9210641384124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 70.37686195207104, Tokens per sec: 174401.61656857474, Loss: 2.9089860916137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 70.48792443349942, Tokens per sec: 174676.8416320966, Loss: 2.9066858291625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 71.53025564215422, Tokens per sec: 177259.85318940465, Loss: 2.903109312057495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 70.37177409352536, Tokens per sec: 174389.00829462465, Loss: 2.887580394744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 70.57458464165512, Tokens per sec: 174891.5951743164, Loss: 2.8930845260620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 71.53115247269352, Tokens per sec: 177262.07563427507, Loss: 2.9130091667175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 70.38053386429743, Tokens per sec: 174410.71597156624, Loss: 2.9135899543762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 71.53757146228264, Tokens per sec: 177277.982597587, Loss: 2.904261589050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 69.89758022337733, Tokens per sec: 173213.9036476308, Loss: 2.9046080112457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 71.50035760956118, Tokens per sec: 177185.7625710989, Loss: 2.916684865951538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 70.39422447315104, Tokens per sec: 174444.64280845056, Loss: 2.895310640335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 71.53786266698333, Tokens per sec: 177278.70423491413, Loss: 2.9025707244873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 70.56071086231509, Tokens per sec: 174857.21442078886, Loss: 2.8963537216186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 70.42660259239275, Tokens per sec: 174524.87935467582, Loss: 2.902979850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 71.49678772261936, Tokens per sec: 177176.91599800155, Loss: 2.8708701133728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 71.5531721348067, Tokens per sec: 177316.64278265755, Loss: 2.9106087684631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 69.81588830765676, Tokens per sec: 173011.46208136782, Loss: 2.9069979190826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 70.11521361738677, Tokens per sec: 173753.22317228367, Loss: 2.8846089839935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 71.50201389668393, Tokens per sec: 177189.8670330442, Loss: 2.901155948638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 69.9060444622906, Tokens per sec: 173234.87896407073, Loss: 2.914303779602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 70.30695843218065, Tokens per sec: 174228.38794577794, Loss: 2.9024970531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 71.47800365387273, Tokens per sec: 177130.36700641146, Loss: 2.884505271911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 71.53254583349364, Tokens per sec: 177265.52853582913, Loss: 2.889988899230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 67.78861331512354, Tokens per sec: 167987.65132709427, Loss: 2.915022373199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 71.46628670180836, Tokens per sec: 177101.3311084671, Loss: 2.914032459259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 71.51677714193855, Tokens per sec: 177226.45198109085, Loss: 2.8997433185577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 71.51216769636235, Tokens per sec: 177215.0292671807, Loss: 2.8791821002960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 69.30239945969487, Tokens per sec: 171738.98015065203, Loss: 2.895416259765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 71.49263693657218, Tokens per sec: 177166.62989292986, Loss: 2.8774185180664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 71.53056338089729, Tokens per sec: 177260.61579991019, Loss: 2.886341094970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 69.37240652581477, Tokens per sec: 171912.4653724121, Loss: 2.913628339767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 71.46736645096229, Tokens per sec: 177104.00684581473, Loss: 2.892096519470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 71.50570331564757, Tokens per sec: 177199.0098196591, Loss: 2.884869337081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 70.41101837152084, Tokens per sec: 174486.25993861156, Loss: 2.88411283493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 70.53510937349569, Tokens per sec: 174793.77111125755, Loss: 2.859245777130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 71.4579514130403, Tokens per sec: 177080.675344692, Loss: 2.9001991748809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 71.50858739665122, Tokens per sec: 177206.15689009393, Loss: 2.895128011703491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 70.3686597648701, Tokens per sec: 174381.29064514532, Loss: 2.87845778465271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 70.46131640593877, Tokens per sec: 174610.9040086851, Loss: 2.886939764022827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 71.5177083941386, Tokens per sec: 177228.75972662846, Loss: 2.9012093544006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 70.38930205817505, Tokens per sec: 174432.44452189127, Loss: 2.8742055892944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 71.5614218619348, Tokens per sec: 177337.08651526348, Loss: 2.869269847869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 70.46119386064129, Tokens per sec: 174610.60032793845, Loss: 2.8856472969055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 70.39037274656413, Tokens per sec: 174435.0978056655, Loss: 2.879040479660034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 71.51887718881082, Tokens per sec: 177231.6561285796, Loss: 2.8736953735351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 70.59700159607517, Tokens per sec: 174947.14685680068, Loss: 2.8899126052856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 71.56697057226309, Tokens per sec: 177350.83683069787, Loss: 2.8970091342926025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 70.36320898297564, Tokens per sec: 174367.78300715762, Loss: 2.8866701126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 71.49474593498145, Tokens per sec: 177171.85622331334, Loss: 2.8787190914154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 70.40977913332246, Tokens per sec: 174483.18896984262, Loss: 2.8843600749969482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 70.39494698206079, Tokens per sec: 174446.43326511985, Loss: 2.8530657291412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 71.54487318426796, Tokens per sec: 177296.0770690177, Loss: 2.8759310245513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 70.64173535255755, Tokens per sec: 175058.0020331941, Loss: 2.864777088165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 70.3844843728647, Tokens per sec: 174420.5057669804, Loss: 2.857875347137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 71.50728231592028, Tokens per sec: 177202.92275627612, Loss: 2.8806912899017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 71.50510449299962, Tokens per sec: 177197.52587117173, Loss: 2.8570363521575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 70.487144280752, Tokens per sec: 174674.90832764757, Loss: 2.864553451538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 70.17432622946383, Tokens per sec: 173899.7107368024, Loss: 2.860213041305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 71.5298065008459, Tokens per sec: 177258.74016776562, Loss: 2.883296489715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 71.56854279848706, Tokens per sec: 177354.73298047588, Loss: 2.8787028789520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 69.35335317843087, Tokens per sec: 171865.2490786953, Loss: 2.8737010955810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 71.51841936021356, Tokens per sec: 177230.5215788816, Loss: 2.8535594940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 71.53007028310019, Tokens per sec: 177259.39384924975, Loss: 2.857077121734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 71.48959036503446, Tokens per sec: 177159.0801530518, Loss: 2.8701095581054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 69.66644416195122, Tokens per sec: 172641.1230256778, Loss: 2.8732166290283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 71.45894045989724, Tokens per sec: 177083.12631175597, Loss: 2.8721232414245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 71.52122236130006, Tokens per sec: 177237.46772435, Loss: 2.8883538246154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 69.46479864934237, Tokens per sec: 172141.4232323461, Loss: 2.858917236328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 71.47547092170883, Tokens per sec: 177124.09061710583, Loss: 2.876152515411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 71.45495514828606, Tokens per sec: 177073.2502705652, Loss: 2.8906912803649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 70.38442445012983, Tokens per sec: 174420.35727184234, Loss: 2.847412586212158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 70.47239830512856, Tokens per sec: 174638.36617564937, Loss: 2.8676793575286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 71.54836581305497, Tokens per sec: 177304.7321878967, Loss: 2.8742728233337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 70.29349814405278, Tokens per sec: 174195.03186902395, Loss: 2.8844738006591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 71.54105472608904, Tokens per sec: 177286.61450901243, Loss: 2.846588373184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 70.50216773372965, Tokens per sec: 174712.13809909613, Loss: 2.84910249710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 71.45327355681532, Tokens per sec: 177069.0831016581, Loss: 2.857940435409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 70.3931216210493, Tokens per sec: 174441.90981945326, Loss: 2.8458597660064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 70.45857247988926, Tokens per sec: 174604.10425766598, Loss: 2.8512935638427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 71.49955903703574, Tokens per sec: 177183.78362041083, Loss: 2.8465216159820557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 70.40364863559593, Tokens per sec: 174467.9969211462, Loss: 2.8399033546447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 71.50147753564427, Tokens per sec: 177188.53787130286, Loss: 2.864633083343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 70.56457573286347, Tokens per sec: 174866.7919957584, Loss: 2.851546049118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 71.48771025602194, Tokens per sec: 177154.42103580767, Loss: 2.8559839725494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 70.27404485938573, Tokens per sec: 174146.82448666278, Loss: 2.8545236587524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 71.53174639683127, Tokens per sec: 177263.54744371455, Loss: 2.8691132068634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 70.45129308786433, Tokens per sec: 174586.06512233696, Loss: 2.8598039150238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 70.3939186376956, Tokens per sec: 174443.88491450145, Loss: 2.871298313140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 71.50661029909897, Tokens per sec: 177201.25742456416, Loss: 2.8579959869384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 70.55074158491253, Tokens per sec: 174832.50945317876, Loss: 2.8554258346557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 70.39020736084842, Tokens per sec: 174434.68796164324, Loss: 2.8586301803588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 71.49735772965487, Tokens per sec: 177178.328538227, Loss: 2.8550851345062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 71.51233648374867, Tokens per sec: 177215.4475409174, Loss: 2.8590006828308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 69.49401061646238, Tokens per sec: 172213.8137048333, Loss: 2.841176748275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 71.54540116635823, Tokens per sec: 177297.3854667994, Loss: 2.862338066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 71.49094301970636, Tokens per sec: 177162.43218033598, Loss: 2.8436272144317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 71.48304082780683, Tokens per sec: 177142.84967831065, Loss: 2.859729528427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 69.35115782858838, Tokens per sec: 171859.8087599479, Loss: 2.8719985485076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 71.47808590173301, Tokens per sec: 177130.57082566957, Loss: 2.847724437713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 71.50514929377768, Tokens per sec: 177197.636892435, Loss: 2.8691000938415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 70.33038650036755, Tokens per sec: 174286.44527955988, Loss: 2.862637996673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 70.47822217649097, Tokens per sec: 174652.7983704378, Loss: 2.842245101928711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 71.53269173404122, Tokens per sec: 177265.89009345832, Loss: 2.845792293548584 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/model.pt +[rank0]:[2024-08-26 03:21:46,090] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00642666299972916, 'preprocessing_with_comm': 0.0013907759994253865, 'state_converting': 0.9866728769993642, : 0.9958642649999092}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600, took 4.57s + eval ppl=13.94437026977539, eval loss=2.635075807571411 +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 67.25898097768358, Tokens per sec: 166675.16405109965, Loss: 2.8413078784942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 68.92130206298724, Tokens per sec: 170794.5788202666, Loss: 2.8618898391723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 71.52883929334429, Tokens per sec: 177256.3433210299, Loss: 2.8480114936828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 71.42849699260317, Tokens per sec: 177007.68404617655, Loss: 2.8548099994659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 70.3931457963395, Tokens per sec: 174441.96972848548, Loss: 2.8383030891418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 70.40374627393552, Tokens per sec: 174468.23887970747, Loss: 2.8311562538146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 71.49338751763288, Tokens per sec: 177168.4899154816, Loss: 2.8551111221313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 71.46691810872314, Tokens per sec: 177102.89580435713, Loss: 2.8559958934783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 70.63756919563865, Tokens per sec: 175047.677837409, Loss: 2.8514387607574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 70.39161688904005, Tokens per sec: 174438.18092777443, Loss: 2.8476195335388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 71.45941952771044, Tokens per sec: 177084.3134945708, Loss: 2.8476970195770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 70.08848800499109, Tokens per sec: 173686.9941606979, Loss: 2.838834285736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 71.53704922137808, Tokens per sec: 177276.6884270959, Loss: 2.8425683975219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 70.3923169760003, Tokens per sec: 174439.91582038288, Loss: 2.8555896282196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 70.22481373906206, Tokens per sec: 174024.8243472451, Loss: 2.836991310119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 71.46620618860678, Tokens per sec: 177101.1315878843, Loss: 2.845487356185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 70.45843530700888, Tokens per sec: 174603.7643281576, Loss: 2.8493916988372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 70.42696625248477, Tokens per sec: 174525.7805444443, Loss: 2.8420650959014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 71.53854720942694, Tokens per sec: 177280.400606498, Loss: 2.843674659729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 71.50052527902514, Tokens per sec: 177186.17807450093, Loss: 2.8449831008911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 70.41118622418846, Tokens per sec: 174486.6758960124, Loss: 2.8606510162353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 70.40576214623022, Tokens per sec: 174473.23443332038, Loss: 2.853416919708252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 71.4748577068734, Tokens per sec: 177122.57100319242, Loss: 2.8443708419799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 71.54030046608082, Tokens per sec: 177284.74536962347, Loss: 2.8337979316711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 69.39536178904154, Tokens per sec: 171969.35104341904, Loss: 2.8341891765594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 71.54221982727644, Tokens per sec: 177289.50175810128, Loss: 2.8422510623931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 71.50256346650303, Tokens per sec: 177191.228927596, Loss: 2.8493852615356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 71.50236979168456, Tokens per sec: 177190.7489800608, Loss: 2.8281052112579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 69.21996123965181, Tokens per sec: 171534.6891600657, Loss: 2.8142943382263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 71.50339659109625, Tokens per sec: 177193.29350211425, Loss: 2.8276755809783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 71.48022856224223, Tokens per sec: 177135.88057444498, Loss: 2.8256664276123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 68.34093889350132, Tokens per sec: 169356.37495399965, Loss: 2.8281991481781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 71.47300582334724, Tokens per sec: 177117.9818318128, Loss: 2.8388564586639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 71.50716461402924, Tokens per sec: 177202.6310780242, Loss: 2.8396928310394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 70.36083163979072, Tokens per sec: 174361.89168885944, Loss: 2.831038475036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 70.50255876287224, Tokens per sec: 174713.1071123869, Loss: 2.856199026107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 71.48885558848117, Tokens per sec: 177157.25929581045, Loss: 2.8555731773376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 70.3859257024649, Tokens per sec: 174424.07754050527, Loss: 2.8478519916534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 70.56351740997563, Tokens per sec: 174864.16935505843, Loss: 2.8535523414611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 71.46627577183345, Tokens per sec: 177101.30402278525, Loss: 2.822423219680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 71.503297916485, Tokens per sec: 177193.04897555767, Loss: 2.8431243896484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 70.36783498152228, Tokens per sec: 174379.24674115202, Loss: 2.850090742111206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 70.44342531993972, Tokens per sec: 174566.56792671361, Loss: 2.823699474334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 71.53417430713, Tokens per sec: 177269.56407289123, Loss: 2.841972589492798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 70.3854247913222, Tokens per sec: 174422.8362275137, Loss: 2.8412694931030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 71.51266430488329, Tokens per sec: 177216.2599178015, Loss: 2.8290557861328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 70.42676030787823, Tokens per sec: 174525.2701910226, Loss: 2.8168981075286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 71.49260161242361, Tokens per sec: 177166.54235579842, Loss: 2.8380298614501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 69.71241978798945, Tokens per sec: 172755.0556916338, Loss: 2.8416481018066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 71.49117693659521, Tokens per sec: 177163.01185215462, Loss: 2.846808671951294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 70.53931876174317, Tokens per sec: 174804.2024390372, Loss: 2.8181865215301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 70.37485727598295, Tokens per sec: 174396.64876039515, Loss: 2.8233590126037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 71.47250173782886, Tokens per sec: 177116.73265236864, Loss: 2.8187437057495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 70.4682819236956, Tokens per sec: 174628.16532900164, Loss: 2.8287954330444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 70.40698082217877, Tokens per sec: 174476.25444656913, Loss: 2.8477866649627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 71.47401774989474, Tokens per sec: 177120.4894972704, Loss: 2.813514232635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 71.49590914209416, Tokens per sec: 177174.7387786765, Loss: 2.8385910987854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 70.5167417534939, Tokens per sec: 174748.25412553275, Loss: 2.8061881065368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 70.59095142676904, Tokens per sec: 174932.15387077894, Loss: 2.8523685932159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 71.5515006060769, Tokens per sec: 177312.50055033102, Loss: 2.837116003036499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 71.48634956744512, Tokens per sec: 177151.04909962206, Loss: 2.812930107116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 69.33087110922163, Tokens per sec: 171809.53603459045, Loss: 2.838563919067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 71.49700212592002, Tokens per sec: 177177.44731299908, Loss: 2.815164089202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 71.4417377341765, Tokens per sec: 177040.4960623833, Loss: 2.833871603012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 69.4062822300514, Tokens per sec: 171996.4130992274, Loss: 2.801852226257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 71.5193774458767, Tokens per sec: 177232.89582069672, Loss: 2.815142869949341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 71.47341658246276, Tokens per sec: 177118.99973814987, Loss: 2.8284459114074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 71.4814416753281, Tokens per sec: 177138.88680230716, Loss: 2.8125572204589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 69.4108881139797, Tokens per sec: 172007.82698698167, Loss: 2.802917718887329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 71.4919515251168, Tokens per sec: 177164.9313678391, Loss: 2.836388111114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 71.44734122519453, Tokens per sec: 177054.38213040153, Loss: 2.828320026397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 70.36217814315279, Tokens per sec: 174365.22847252025, Loss: 2.8223025798797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 70.51509038431499, Tokens per sec: 174744.16185079407, Loss: 2.8101933002471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 71.50647859562727, Tokens per sec: 177200.93104885294, Loss: 2.823931932449341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 71.44062513493202, Tokens per sec: 177037.73891889257, Loss: 2.8374106884002686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 69.6515982119889, Tokens per sec: 172604.33312625476, Loss: 2.8088879585266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 71.48169959021651, Tokens per sec: 177139.5259438123, Loss: 2.84076189994812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 71.5302943763208, Tokens per sec: 177259.94917693635, Loss: 2.8085007667541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 70.43214545788638, Tokens per sec: 174538.61518596607, Loss: 2.8169851303100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 70.43319678910223, Tokens per sec: 174541.22050053263, Loss: 2.8136184215545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 71.46798871193404, Tokens per sec: 177105.54887705043, Loss: 2.8190817832946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 70.36620605706499, Tokens per sec: 174375.21008690598, Loss: 2.8195626735687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 71.48074662501483, Tokens per sec: 177137.16439106475, Loss: 2.8052823543548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 70.51558091439055, Tokens per sec: 174745.3774383577, Loss: 2.8053948879241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 70.35737255673192, Tokens per sec: 174353.31969999027, Loss: 2.820549488067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 71.50787125610714, Tokens per sec: 177204.38221493628, Loss: 2.808905839920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 71.45896263363895, Tokens per sec: 177083.18126073055, Loss: 2.8001022338867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 70.47346656992617, Tokens per sec: 174641.0134534969, Loss: 2.807490587234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 70.4094163155259, Tokens per sec: 174482.28986737508, Loss: 2.826524257659912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 71.49124849918755, Tokens per sec: 177163.1891921417, Loss: 2.834059238433838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 70.55884285399652, Tokens per sec: 174852.5852903973, Loss: 2.814868688583374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 70.35226382648833, Tokens per sec: 174340.65970367356, Loss: 2.8002545833587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 71.49985893174586, Tokens per sec: 177184.52679253864, Loss: 2.8279149532318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 71.49192442672943, Tokens per sec: 177164.86421505007, Loss: 2.8108396530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 70.41228017490988, Tokens per sec: 174489.38682640874, Loss: 2.778142213821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 70.37274125207804, Tokens per sec: 174391.40502005952, Loss: 2.8172593116760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 71.4793793786381, Tokens per sec: 177133.7762039286, Loss: 2.8200318813323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 71.48564978865319, Tokens per sec: 177149.31497068872, Loss: 2.8081846237182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 69.46005983160593, Tokens per sec: 172129.67991997136, Loss: 2.838066816329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 71.47018049280013, Tokens per sec: 177110.98035148115, Loss: 2.8086230754852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 71.49712695856334, Tokens per sec: 177177.75666204066, Loss: 2.7966794967651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 70.2989009983396, Tokens per sec: 174208.42073712024, Loss: 2.8156378269195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 70.43813572890096, Tokens per sec: 174553.45973742305, Loss: 2.8224942684173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 71.48779090456155, Tokens per sec: 177154.62089177297, Loss: 2.8102686405181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 70.31396833825698, Tokens per sec: 174245.75926524, Loss: 2.821500301361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 70.53837583353878, Tokens per sec: 174801.86575907463, Loss: 2.832624912261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 71.5245986324589, Tokens per sec: 177245.83449620858, Loss: 2.7802867889404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 71.55076540727005, Tokens per sec: 177310.6786466988, Loss: 2.8070015907287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 70.312469649615, Tokens per sec: 174242.04534969098, Loss: 2.8210065364837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 70.53285544776651, Tokens per sec: 174788.18563501022, Loss: 2.800395965576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 71.53696742739255, Tokens per sec: 177276.4857325894, Loss: 2.8111279010772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 70.31303938115444, Tokens per sec: 174243.4572072068, Loss: 2.8015551567077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 70.6010996827893, Tokens per sec: 174957.30236711982, Loss: 2.8167619705200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 71.53577263991696, Tokens per sec: 177273.52491760819, Loss: 2.8210792541503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 71.53594592701342, Tokens per sec: 177273.95434212236, Loss: 2.7925472259521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 70.35552706054719, Tokens per sec: 174348.7463571463, Loss: 2.812446117401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 70.47208145283312, Tokens per sec: 174637.5809807571, Loss: 2.7852001190185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 71.4204377583654, Tokens per sec: 176987.71237593735, Loss: 2.810316562652588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 70.35773179962567, Tokens per sec: 174354.20994345486, Loss: 2.78228759765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 71.49736270215926, Tokens per sec: 177178.3408606407, Loss: 2.7864091396331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 70.54128133172703, Tokens per sec: 174809.06590365217, Loss: 2.8050949573516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 70.28949221133932, Tokens per sec: 174185.10472647008, Loss: 2.8015189170837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 71.4525536057089, Tokens per sec: 177067.2989835072, Loss: 2.8016860485076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 71.48726256583262, Tokens per sec: 177153.3116102014, Loss: 2.8200364112854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 70.45772569977473, Tokens per sec: 174602.00584326166, Loss: 2.7770981788635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 70.57948598891112, Tokens per sec: 174903.74125274477, Loss: 2.782637596130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 71.52483314648832, Tokens per sec: 177246.41564780733, Loss: 2.807544469833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 70.58262745718098, Tokens per sec: 174911.52615717874, Loss: 2.8108971118927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 70.36661831500284, Tokens per sec: 174376.23170748996, Loss: 2.7962052822113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 71.40977042789528, Tokens per sec: 176961.277555929, Loss: 2.7923903465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 71.4603901154354, Tokens per sec: 177086.71871787228, Loss: 2.7978155612945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 69.43120487679654, Tokens per sec: 172058.174163318, Loss: 2.811159372329712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 71.47270061552037, Tokens per sec: 177117.22549319637, Loss: 2.7914929389953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 71.49837032657908, Tokens per sec: 177180.837864952, Loss: 2.792794942855835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 71.51605478995819, Tokens per sec: 177224.6619133099, Loss: 2.799143075942993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 69.20896218315076, Tokens per sec: 171507.43228641, Loss: 2.8149068355560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 71.49552211664856, Tokens per sec: 177173.77968698792, Loss: 2.795915126800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 71.52279538012, Tokens per sec: 177241.36583826813, Loss: 2.785897970199585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 69.44699925037179, Tokens per sec: 172097.3143609313, Loss: 2.801553249359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 71.51258278882847, Tokens per sec: 177216.0579120379, Loss: 2.794882297515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 71.51192474590788, Tokens per sec: 177214.42720919132, Loss: 2.7939014434814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 71.5283962772903, Tokens per sec: 177255.24547844593, Loss: 2.7894465923309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 69.71563326026899, Tokens per sec: 172763.0190299357, Loss: 2.8005154132843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 71.482190979004, Tokens per sec: 177140.74365935865, Loss: 2.7763447761535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 71.49774632951215, Tokens per sec: 177179.29153148117, Loss: 2.79837703704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 70.34692183261825, Tokens per sec: 174327.4216543943, Loss: 2.8002772331237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 70.5101618980554, Tokens per sec: 174731.94851892526, Loss: 2.793710947036743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 71.49074332610891, Tokens per sec: 177161.93731760344, Loss: 2.7926063537597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 70.39489819845681, Tokens per sec: 174446.31237397523, Loss: 2.81640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 70.43700390270259, Tokens per sec: 174550.65494742247, Loss: 2.7953672409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 71.47383384007011, Tokens per sec: 177120.03374846565, Loss: 2.792682409286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 69.92801687989757, Tokens per sec: 173289.32903536243, Loss: 2.777474880218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 71.46851303220276, Tokens per sec: 177106.84820043496, Loss: 2.7939612865448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 70.56045150092602, Tokens per sec: 174856.5716946957, Loss: 2.7776522636413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 71.55226086568764, Tokens per sec: 177314.38455739597, Loss: 2.7813282012939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 70.33428634429367, Tokens per sec: 174296.10952241204, Loss: 2.8095340728759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 71.53648059179656, Tokens per sec: 177275.27930035317, Loss: 2.7798349857330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 70.45033995891268, Tokens per sec: 174583.703163799, Loss: 2.7911109924316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 70.32166054080943, Tokens per sec: 174264.82139053105, Loss: 2.7786738872528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 71.5042172966697, Tokens per sec: 177195.32730093345, Loss: 2.786897897720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 71.51696264464482, Tokens per sec: 177226.91167723178, Loss: 2.791109323501587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 70.54280182145646, Tokens per sec: 174812.83384469748, Loss: 2.7733588218688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 70.3747375756588, Tokens per sec: 174396.35212980583, Loss: 2.804492950439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 71.52638203094625, Tokens per sec: 177250.2539541178, Loss: 2.7763569355010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 70.51219606804345, Tokens per sec: 174736.98941623795, Loss: 2.7971222400665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 70.37612154428955, Tokens per sec: 174399.78175652953, Loss: 2.7848775386810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 71.5335826622931, Tokens per sec: 177268.09791181033, Loss: 2.786891222000122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 71.51475077938716, Tokens per sec: 177221.43043146623, Loss: 2.785243511199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 69.32295903588711, Tokens per sec: 171789.9290452807, Loss: 2.779892921447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 71.5371589891544, Tokens per sec: 177276.96044373757, Loss: 2.780827283859253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 71.51491102015262, Tokens per sec: 177221.82752573502, Loss: 2.7890000343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 71.4856506380343, Tokens per sec: 177149.31707554866, Loss: 2.8016817569732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 70.44974788260589, Tokens per sec: 174582.23593348815, Loss: 2.786698818206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 71.41885918058549, Tokens per sec: 176983.80048630404, Loss: 2.761279821395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 71.50733455609108, Tokens per sec: 177203.05221317412, Loss: 2.7847912311553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 70.30610774866281, Tokens per sec: 174226.2798583102, Loss: 2.799706220626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 70.40977716692552, Tokens per sec: 174483.1840968944, Loss: 2.7788565158843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 71.48073772694273, Tokens per sec: 177137.14234066193, Loss: 2.797764539718628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 70.39518610880047, Tokens per sec: 174447.02584752277, Loss: 2.7885241508483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 70.54966668923771, Tokens per sec: 174829.845743287, Loss: 2.7936089038848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 71.55629569515796, Tokens per sec: 177324.38330929654, Loss: 2.788184881210327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 71.47060089647961, Tokens per sec: 177112.02215811025, Loss: 2.7821240425109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 69.59226158689911, Tokens per sec: 172457.2904328134, Loss: 2.791776418685913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 70.41007603299508, Tokens per sec: 174483.92471993755, Loss: 2.7906906604766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 71.49808934020066, Tokens per sec: 177180.14154975305, Loss: 2.776413917541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 70.21223559414717, Tokens per sec: 173993.65431854897, Loss: 2.785998582839966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 70.48589262083186, Tokens per sec: 174671.80657648374, Loss: 2.786271810531616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 71.45718386858569, Tokens per sec: 177078.77328498143, Loss: 2.7824389934539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 70.2946655705105, Tokens per sec: 174197.9248803888, Loss: 2.7697768211364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 71.51942362677251, Tokens per sec: 177233.01026204374, Loss: 2.7717339992523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 70.337957104333, Tokens per sec: 174305.20607015712, Loss: 2.786055326461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 71.493359375501, Tokens per sec: 177168.42017617865, Loss: 2.788188934326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 70.34786661296144, Tokens per sec: 174329.76292416087, Loss: 2.801084280014038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 71.47704595264766, Tokens per sec: 177127.99371727623, Loss: 2.7903647422790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 70.07844355759904, Tokens per sec: 173662.10291357318, Loss: 2.7905161380767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 70.34725576875726, Tokens per sec: 174328.24918493288, Loss: 2.7800536155700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 71.47330051655666, Tokens per sec: 177118.7121140489, Loss: 2.7814877033233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 71.52166331747206, Tokens per sec: 177238.56046231955, Loss: 2.7696750164031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 68.59855216320467, Tokens per sec: 169994.7690147689, Loss: 2.7790637016296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 71.53797016568767, Tokens per sec: 177278.97062854544, Loss: 2.7667882442474365 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/model.pt +[rank0]:[2024-08-26 03:46:47,035] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0064265950004482875, 'preprocessing_with_comm': 0.001614970000446192, 'state_converting': 0.9440371640002923, : 0.9534755049999148}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800, took 4.21s + eval ppl=13.86946964263916, eval loss=2.629689931869507 +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 66.78681805002525, Tokens per sec: 165505.09230332205, Loss: 2.7723512649536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 70.20846502818914, Tokens per sec: 173984.31044074282, Loss: 2.787625551223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 70.25751297037895, Tokens per sec: 174105.85664456605, Loss: 2.7912449836730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 71.52882774507684, Tokens per sec: 177256.31470315106, Loss: 2.761833667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 71.50480279665103, Tokens per sec: 177196.77823438533, Loss: 2.8191967010498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 69.41678620564916, Tokens per sec: 172022.44310786703, Loss: 2.767381429672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 71.53727328100965, Tokens per sec: 177277.24367154515, Loss: 2.7885446548461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 71.4485545817866, Tokens per sec: 177057.38896169892, Loss: 2.7828781604766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 69.8034942660123, Tokens per sec: 172980.7482808564, Loss: 2.756115198135376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 71.48569203617893, Tokens per sec: 177149.4196647114, Loss: 2.7973573207855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 71.51274404378998, Tokens per sec: 177216.45751959624, Loss: 2.7820162773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 71.53713298742517, Tokens per sec: 177276.89600858817, Loss: 2.772798538208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 69.79790400780865, Tokens per sec: 172966.89500521, Loss: 2.77253794670105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 71.52586839888843, Tokens per sec: 177248.98111729728, Loss: 2.768599271774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 71.5373612523426, Tokens per sec: 177277.46167419903, Loss: 2.784391164779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 70.32896642807188, Tokens per sec: 174282.9261839773, Loss: 2.7689287662506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 70.46836952721017, Tokens per sec: 174628.38242016104, Loss: 2.763171672821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 71.54895544951185, Tokens per sec: 177306.19337199067, Loss: 2.767293930053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 70.41867676958371, Tokens per sec: 174505.23829265352, Loss: 2.7784762382507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 70.55579173723932, Tokens per sec: 174845.02428696473, Loss: 2.789660692214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 71.51130243438925, Tokens per sec: 177212.88505269485, Loss: 2.7910921573638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 71.54003239362255, Tokens per sec: 177284.08105654127, Loss: 2.7703888416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 70.35602744338519, Tokens per sec: 174349.9863609406, Loss: 2.7929160594940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 70.30098783709994, Tokens per sec: 174213.5921534526, Loss: 2.7808127403259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 71.55220692558629, Tokens per sec: 177314.25088788284, Loss: 2.7558579444885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 70.35958175010005, Tokens per sec: 174358.7943245192, Loss: 2.7858636379241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 71.4905488079003, Tokens per sec: 177161.45528005454, Loss: 2.797921657562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 70.55099650327524, Tokens per sec: 174833.14116896313, Loss: 2.7743237018585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 71.50349572950996, Tokens per sec: 177193.53917802445, Loss: 2.777952194213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 70.61856969237536, Tokens per sec: 175000.5949753545, Loss: 2.7715554237365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 71.49784145489924, Tokens per sec: 177179.52726266932, Loss: 2.766045093536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 70.40719679138503, Tokens per sec: 174476.78964205322, Loss: 2.76427960395813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 70.37813507712075, Tokens per sec: 174404.77151269422, Loss: 2.7918028831481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 71.55594305421354, Tokens per sec: 177323.50942618967, Loss: 2.766291856765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 70.52734416548947, Tokens per sec: 174774.52807040987, Loss: 2.776815891265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 70.35245603261302, Tokens per sec: 174341.1360116238, Loss: 2.767784595489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 71.44904861014797, Tokens per sec: 177058.6132184001, Loss: 2.7734477519989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 71.46793584425767, Tokens per sec: 177105.41786512453, Loss: 2.7561917304992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 69.41699392203448, Tokens per sec: 172022.95785195174, Loss: 2.7709083557128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 71.55549282626485, Tokens per sec: 177322.39371173613, Loss: 2.7762436866760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 71.55757374425527, Tokens per sec: 177327.55045574842, Loss: 2.7737786769866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 71.54615292343038, Tokens per sec: 177299.2484036371, Loss: 2.759455680847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 69.35714281074725, Tokens per sec: 171874.64020503775, Loss: 2.7929346561431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 71.52694869296131, Tokens per sec: 177251.6582050141, Loss: 2.7831499576568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 71.53164629093153, Tokens per sec: 177263.29937026743, Loss: 2.758770704269409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 70.24116910577875, Tokens per sec: 174065.35474766145, Loss: 2.761695146560669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 70.45849803758539, Tokens per sec: 174603.9197814365, Loss: 2.785271167755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 71.5227553206898, Tokens per sec: 177241.2665665872, Loss: 2.7612204551696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 71.49556156628542, Tokens per sec: 177173.87744753377, Loss: 2.758002281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 69.39171875554327, Tokens per sec: 171960.32320509705, Loss: 2.7483410835266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 71.50554401428757, Tokens per sec: 177198.61505334056, Loss: 2.7915449142456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 71.49609113799877, Tokens per sec: 177175.18978457662, Loss: 2.768324851989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 70.35891984616167, Tokens per sec: 174357.15405364518, Loss: 2.7863519191741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 70.44055780188575, Tokens per sec: 174559.46190109267, Loss: 2.7615249156951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 71.54857863064865, Tokens per sec: 177305.25957333765, Loss: 2.7470054626464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 70.23924917923746, Tokens per sec: 174060.5969581948, Loss: 2.7617335319519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 70.57685671184773, Tokens per sec: 174897.22561454287, Loss: 2.785428285598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 71.48245752755548, Tokens per sec: 177141.40419603197, Loss: 2.775282382965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 70.42691375209516, Tokens per sec: 174525.65044269548, Loss: 2.7635483741760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 71.49094759550356, Tokens per sec: 177162.44351966548, Loss: 2.7595107555389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 70.42088747286081, Tokens per sec: 174510.7166589033, Loss: 2.739877462387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 71.4845402514737, Tokens per sec: 177146.5654153323, Loss: 2.759147882461548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 70.39594911064835, Tokens per sec: 174448.9166501534, Loss: 2.7552595138549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 71.50227897077295, Tokens per sec: 177190.52391583708, Loss: 2.760451316833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 70.5389097219172, Tokens per sec: 174803.18879328898, Loss: 2.7732160091400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 70.4136590277107, Tokens per sec: 174492.80377554233, Loss: 2.743811845779419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 71.38839874387321, Tokens per sec: 176908.31616863693, Loss: 2.759781837463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 71.47606593832317, Tokens per sec: 177125.56513382265, Loss: 2.7547030448913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 70.54812402824986, Tokens per sec: 174826.022859421, Loss: 2.756758213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 70.3997029495607, Tokens per sec: 174458.2190764977, Loss: 2.7410385608673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 71.51120146808806, Tokens per sec: 177212.63484707812, Loss: 2.74900484085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 70.47776502145331, Tokens per sec: 174651.6654898947, Loss: 2.757175922393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 70.41245736065476, Tokens per sec: 174489.82591220314, Loss: 2.755720615386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 71.5283369228555, Tokens per sec: 177255.09839161806, Loss: 2.749119281768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 71.4787767796186, Tokens per sec: 177132.28289717654, Loss: 2.7738401889801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 69.49079629942311, Tokens per sec: 172205.84827312376, Loss: 2.7726643085479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 71.49654854559935, Tokens per sec: 177176.32329099847, Loss: 2.7555158138275146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 71.53882888120886, Tokens per sec: 177281.0986202023, Loss: 2.776303768157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 71.5616393323878, Tokens per sec: 177337.62543100232, Loss: 2.7769296169281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 70.2963401487687, Tokens per sec: 174202.07466978268, Loss: 2.7575368881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 71.50676213002394, Tokens per sec: 177201.63367832178, Loss: 2.7751848697662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 71.54452132300081, Tokens per sec: 177295.20511803695, Loss: 2.7785799503326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 69.43368815578623, Tokens per sec: 172064.32800221018, Loss: 2.783169984817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 71.50484614845587, Tokens per sec: 177196.88566493333, Loss: 2.7706964015960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 71.52268519263097, Tokens per sec: 177241.0927815322, Loss: 2.761296272277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 70.39880272897875, Tokens per sec: 174455.98823072817, Loss: 2.762794017791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 70.48687758645637, Tokens per sec: 174674.2474298042, Loss: 2.767157793045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 71.54939451410497, Tokens per sec: 177307.28142241944, Loss: 2.768030881881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 71.54391040323918, Tokens per sec: 177293.6911915693, Loss: 2.7664616107940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 70.41261416997283, Tokens per sec: 174490.21450296728, Loss: 2.759462833404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 70.52812898527338, Tokens per sec: 174776.47294028944, Loss: 2.7550718784332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 71.55430188774973, Tokens per sec: 177319.44243490294, Loss: 2.753561496734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 70.4213602449013, Tokens per sec: 174511.88824009974, Loss: 2.749216318130493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 70.24170500941614, Tokens per sec: 174066.68277590955, Loss: 2.7728989124298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 71.49764862638267, Tokens per sec: 177179.04941236344, Loss: 2.790282964706421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 71.50627835165537, Tokens per sec: 177200.43482223188, Loss: 2.7770533561706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 70.4872022294711, Tokens per sec: 174675.05193095704, Loss: 2.7689449787139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 70.61215771528646, Tokens per sec: 174984.70538979064, Loss: 2.7785377502441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 71.45175081627212, Tokens per sec: 177065.30958284833, Loss: 2.7610867023468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 70.41918925321691, Tokens per sec: 174506.50828355135, Loss: 2.7807304859161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 71.58277961335304, Tokens per sec: 177390.0133760284, Loss: 2.7567543983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 70.57220369911614, Tokens per sec: 174885.69493642237, Loss: 2.742504119873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 70.00068356518183, Tokens per sec: 173469.40508639306, Loss: 2.758387804031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 71.56836384864448, Tokens per sec: 177354.289523053, Loss: 2.7458183765411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 71.5233949725581, Tokens per sec: 177242.85169438028, Loss: 2.7602035999298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 69.31529268423373, Tokens per sec: 171770.93098136527, Loss: 2.77036714553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 71.55639153734163, Tokens per sec: 177324.62081678555, Loss: 2.7719902992248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 71.5148677720621, Tokens per sec: 177221.72035220254, Loss: 2.7407615184783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 70.57188060553716, Tokens per sec: 174884.89427494197, Loss: 2.761312961578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 70.4045319311728, Tokens per sec: 174470.18582488873, Loss: 2.7705330848693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 71.55981880407633, Tokens per sec: 177333.11396129875, Loss: 2.762998342514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 71.55965157214135, Tokens per sec: 177332.69954214175, Loss: 2.773693561553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 70.39298192317973, Tokens per sec: 174441.56363274346, Loss: 2.7509689331054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 71.5488945912333, Tokens per sec: 177306.04255847237, Loss: 2.7504711151123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 71.55476748702266, Tokens per sec: 177320.59624119126, Loss: 2.740090847015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 71.42670992457057, Tokens per sec: 177003.25549472994, Loss: 2.7746121883392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 69.41162518638974, Tokens per sec: 172009.65353360903, Loss: 2.7436575889587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 71.42840686999381, Tokens per sec: 177007.4607124227, Loss: 2.7597665786743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 71.5084024299718, Tokens per sec: 177205.69852228655, Loss: 2.7545907497406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 70.19054073178965, Tokens per sec: 173939.89205974067, Loss: 2.7791852951049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 70.39928712111775, Tokens per sec: 174457.18860780878, Loss: 2.741243839263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 71.49661085968152, Tokens per sec: 177176.47771215855, Loss: 2.7757837772369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 70.40952114250068, Tokens per sec: 174482.5496401662, Loss: 2.7495498657226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 70.5569757908262, Tokens per sec: 174847.9585021873, Loss: 2.76511287689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 71.5548893977658, Tokens per sec: 177320.89834944234, Loss: 2.7321503162384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 71.5477727731846, Tokens per sec: 177303.26256976923, Loss: 2.747899293899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 70.65478545656184, Tokens per sec: 175090.34162850433, Loss: 2.7430262565612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 70.49717958701413, Tokens per sec: 174699.7769220454, Loss: 2.7551004886627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 71.51464328284801, Tokens per sec: 177221.16404320052, Loss: 2.760477304458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 70.36192908532227, Tokens per sec: 174364.61127978022, Loss: 2.7409467697143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 70.560718009358, Tokens per sec: 174857.2321319485, Loss: 2.761198043823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 71.49067340563471, Tokens per sec: 177161.76404696598, Loss: 2.7441632747650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 70.40402172991237, Tokens per sec: 174468.9214899621, Loss: 2.762443780899048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 71.49002149992816, Tokens per sec: 177160.14855281182, Loss: 2.767000198364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 70.54312497779125, Tokens per sec: 174813.63466169368, Loss: 2.7613234519958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 71.51310945403925, Tokens per sec: 177217.36304644722, Loss: 2.782405138015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 70.27347831536486, Tokens per sec: 174145.42052816908, Loss: 2.7474091053009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 71.51396907977224, Tokens per sec: 177219.49329370915, Loss: 2.755502462387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 70.39717158422556, Tokens per sec: 174451.94607434433, Loss: 2.747875452041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 70.38337169250995, Tokens per sec: 174417.7484224895, Loss: 2.7379653453826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 71.49353773807329, Tokens per sec: 177168.86217828107, Loss: 2.7475388050079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 71.51267110514846, Tokens per sec: 177216.2767696077, Loss: 2.74084734916687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 68.39802822914722, Tokens per sec: 169497.84861663965, Loss: 2.738332748413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 71.51038021582464, Tokens per sec: 177210.599693501, Loss: 2.7325422763824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 71.53132304101862, Tokens per sec: 177262.4983213743, Loss: 2.760727643966675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 70.0751273699103, Tokens per sec: 173653.88503517228, Loss: 2.7454466819763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 70.10767988242449, Tokens per sec: 173734.5537471377, Loss: 2.747209072113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 71.4314373963123, Tokens per sec: 177014.97069047997, Loss: 2.7497050762176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 71.57711412864377, Tokens per sec: 177375.97368081418, Loss: 2.746781349182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 70.34031247305154, Tokens per sec: 174311.04293330692, Loss: 2.6975607872009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 71.52570024334568, Tokens per sec: 177248.5644093385, Loss: 2.7462902069091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 71.48681793144432, Tokens per sec: 177152.20975720667, Loss: 2.756803035736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 69.62661205649285, Tokens per sec: 172542.41468048329, Loss: 2.740285873413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 70.40514980932271, Tokens per sec: 174471.71699500913, Loss: 2.7711358070373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 71.55150304289641, Tokens per sec: 177312.5065890382, Loss: 2.7284233570098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 71.53501783403571, Tokens per sec: 177271.65442548573, Loss: 2.757842779159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 69.42021949252744, Tokens per sec: 172030.9511709592, Loss: 2.752523422241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 71.54483901700918, Tokens per sec: 177295.9923987866, Loss: 2.7471861839294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 71.56729510219014, Tokens per sec: 177351.64105160703, Loss: 2.755908966064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 70.38690967287013, Tokens per sec: 174426.51592756272, Loss: 2.7491753101348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 70.49857815742385, Tokens per sec: 174703.24273358588, Loss: 2.748134136199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 71.51811623760774, Tokens per sec: 177229.77040767265, Loss: 2.748091220855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 71.54237468070272, Tokens per sec: 177289.88550195014, Loss: 2.731018304824829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 70.67188723801364, Tokens per sec: 175132.72172685867, Loss: 2.735421895980835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 70.51400261885404, Tokens per sec: 174741.4662481546, Loss: 2.7644248008728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 71.54885087657982, Tokens per sec: 177305.93422874546, Loss: 2.7401795387268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 70.36104834522342, Tokens per sec: 174362.42870879374, Loss: 2.7370808124542236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 70.46532284414285, Tokens per sec: 174620.8324039004, Loss: 2.7226877212524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 71.49164836617705, Tokens per sec: 177164.18010659068, Loss: 2.76497745513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 70.34419388395256, Tokens per sec: 174320.66149709234, Loss: 2.744952440261841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 71.49145017462206, Tokens per sec: 177163.6889660848, Loss: 2.753033399581909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 70.54118001255733, Tokens per sec: 174808.81482358841, Loss: 2.74716854095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 70.48991761204553, Tokens per sec: 174681.7809481006, Loss: 2.7459936141967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 71.51756728849068, Tokens per sec: 177228.41005128887, Loss: 2.739957809448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 71.51799370825036, Tokens per sec: 177229.46676642736, Loss: 2.7413477897644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 70.57138235903565, Tokens per sec: 174883.65956522484, Loss: 2.7337968349456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 70.42709207012012, Tokens per sec: 174526.09233440465, Loss: 2.7336220741271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 71.45663221853881, Tokens per sec: 177077.40623539602, Loss: 2.73956561088562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 71.51574863475015, Tokens per sec: 177223.90322697858, Loss: 2.731217622756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 68.11102514334604, Tokens per sec: 168786.62335402446, Loss: 2.7312300205230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 71.51712372614818, Tokens per sec: 177227.31085494175, Loss: 2.738753318786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 71.48731074197333, Tokens per sec: 177153.43099598514, Loss: 2.7582054138183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 70.55531800469038, Tokens per sec: 174843.8503255225, Loss: 2.744159460067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 69.65806395273181, Tokens per sec: 172620.35594407583, Loss: 2.723186731338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 71.52462945643181, Tokens per sec: 177245.910881409, Loss: 2.7601168155670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 71.5264269743388, Tokens per sec: 177250.36532879548, Loss: 2.736025810241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 69.38104692096479, Tokens per sec: 171933.87722341198, Loss: 2.7332451343536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 71.58228589217826, Tokens per sec: 177388.7898805694, Loss: 2.7448322772979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 71.50991515540031, Tokens per sec: 177209.44722253887, Loss: 2.730435609817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 70.37583301343453, Tokens per sec: 174399.06674528655, Loss: 2.7330052852630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 70.51854377509382, Tokens per sec: 174752.71973356642, Loss: 2.7287607192993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 71.53123501688071, Tokens per sec: 177262.28018786394, Loss: 2.7540316581726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 70.40466387597536, Tokens per sec: 174470.5127986444, Loss: 2.752962350845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 71.47363228331123, Tokens per sec: 177119.53426861382, Loss: 2.74418044090271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 70.40654207149814, Tokens per sec: 174475.16717405015, Loss: 2.740152597427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 71.49995109835616, Tokens per sec: 177184.755191552, Loss: 2.7337396144866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 70.39827994565735, Tokens per sec: 174454.69271606838, Loss: 2.7320284843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 70.57775544224096, Tokens per sec: 174899.45276746064, Loss: 2.722580671310425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 71.48765933798651, Tokens per sec: 177154.2948553068, Loss: 2.735715627670288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 70.41296569885527, Tokens per sec: 174491.08563026213, Loss: 2.747447967529297 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000, took 4.55s +[rank0]:[2024-08-26 04:11:47,890] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006429709999792976, 'preprocessing_with_comm': 0.0012574530010169838, 'state_converting': 0.9817426709996653, : 0.9907831880009326}) + eval ppl=12.73641300201416, eval loss=2.5444650650024414 +------------------------------------------------------------------ +iteration: 1001 , TFLOPS: 67.36156948362608, Tokens per sec: 166929.38966988202, Loss: 2.728109836578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1002 , TFLOPS: 69.91569381124397, Tokens per sec: 173258.79111374074, Loss: 2.7283833026885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1003 , TFLOPS: 71.47956521815603, Tokens per sec: 177134.23673472594, Loss: 2.737713098526001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1004 , TFLOPS: 70.34533819817945, Tokens per sec: 174323.4972338032, Loss: 2.7420895099639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1005 , TFLOPS: 71.50132655761874, Tokens per sec: 177188.163731124, Loss: 2.722437620162964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1006 , TFLOPS: 70.55378262767495, Tokens per sec: 174840.04549214253, Loss: 2.733297109603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1007 , TFLOPS: 70.4158651733746, Tokens per sec: 174498.27084752446, Loss: 2.7210235595703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1008 , TFLOPS: 71.51557713603735, Tokens per sec: 177223.47823427612, Loss: 2.7303314208984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1009 , TFLOPS: 70.49038182292128, Tokens per sec: 174682.93131378773, Loss: 2.734342336654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1010 , TFLOPS: 71.46791459108468, Tokens per sec: 177105.36519742062, Loss: 2.7411718368530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1011 , TFLOPS: 70.6146490536133, Tokens per sec: 174990.87920060812, Loss: 2.727367877960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1012 , TFLOPS: 71.52884496600808, Tokens per sec: 177256.35737851574, Loss: 2.7264294624328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1013 , TFLOPS: 70.55081858200967, Tokens per sec: 174832.7002604674, Loss: 2.744854211807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1014 , TFLOPS: 70.39060690081067, Tokens per sec: 174435.67806568264, Loss: 2.758805274963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1015 , TFLOPS: 71.53560614860902, Tokens per sec: 177273.11233380658, Loss: 2.7368953227996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1016 , TFLOPS: 70.43746550519454, Tokens per sec: 174551.79884924728, Loss: 2.720203399658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1017 , TFLOPS: 70.39395597314088, Tokens per sec: 174443.97743584763, Loss: 2.7246809005737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1018 , TFLOPS: 71.46113740561727, Tokens per sec: 177088.57058526375, Loss: 2.733308792114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1019 , TFLOPS: 71.51266099170296, Tokens per sec: 177216.25170737566, Loss: 2.744868278503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1020 , TFLOPS: 70.36127477691633, Tokens per sec: 174362.98983147228, Loss: 2.742438316345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1021 , TFLOPS: 71.49031512680149, Tokens per sec: 177160.8761925496, Loss: 2.7344560623168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1022 , TFLOPS: 71.44588972722505, Tokens per sec: 177050.78515853744, Loss: 2.7448501586914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1023 , TFLOPS: 71.53771188323334, Tokens per sec: 177278.3305761715, Loss: 2.7353439331054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1024 , TFLOPS: 69.39456344744787, Tokens per sec: 171967.37266500533, Loss: 2.7150046825408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1025 , TFLOPS: 71.5202074842038, Tokens per sec: 177234.9527471075, Loss: 2.7371392250061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1026 , TFLOPS: 71.53436749702034, Tokens per sec: 177270.0428187211, Loss: 2.727785110473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1027 , TFLOPS: 70.39099999709465, Tokens per sec: 174436.65220157747, Loss: 2.7379298210144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1028 , TFLOPS: 70.53976105792354, Tokens per sec: 174805.29849769516, Loss: 2.7385034561157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1029 , TFLOPS: 71.53380685485116, Tokens per sec: 177268.65348566603, Loss: 2.7276296615600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1030 , TFLOPS: 71.55498985886445, Tokens per sec: 177321.14730311147, Loss: 2.7635891437530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1031 , TFLOPS: 69.34270872523116, Tokens per sec: 171838.87095108314, Loss: 2.7401790618896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1032 , TFLOPS: 71.48206441304626, Tokens per sec: 177140.43001497298, Loss: 2.7451841831207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1033 , TFLOPS: 71.49961065760975, Tokens per sec: 177183.91154187976, Loss: 2.727808952331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1034 , TFLOPS: 70.38315585072915, Tokens per sec: 174417.2135427797, Loss: 2.717583656311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1035 , TFLOPS: 70.5597044604915, Tokens per sec: 174854.72044620573, Loss: 2.7389943599700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1036 , TFLOPS: 71.50080687357996, Tokens per sec: 177186.87589682796, Loss: 2.7255706787109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1037 , TFLOPS: 69.94016201032419, Tokens per sec: 173319.42600645617, Loss: 2.714827299118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1038 , TFLOPS: 71.50450786292365, Tokens per sec: 177196.04735611926, Loss: 2.729779005050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1039 , TFLOPS: 70.21398209191065, Tokens per sec: 173997.9823323995, Loss: 2.7411863803863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1040 , TFLOPS: 70.37598521613491, Tokens per sec: 174399.44392034458, Loss: 2.717369318008423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1041 , TFLOPS: 71.45795888766484, Tokens per sec: 177080.69386763504, Loss: 2.7474257946014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1042 , TFLOPS: 71.4980396336108, Tokens per sec: 177180.01837134775, Loss: 2.7411880493164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1043 , TFLOPS: 70.08721832325452, Tokens per sec: 173683.84774948773, Loss: 2.697674512863159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1044 , TFLOPS: 70.47465133297585, Tokens per sec: 174643.94942684646, Loss: 2.700070858001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1045 , TFLOPS: 71.57034617182215, Tokens per sec: 177359.2019382582, Loss: 2.74465274810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1046 , TFLOPS: 69.92950337138036, Tokens per sec: 173293.01272500737, Loss: 2.730510711669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1047 , TFLOPS: 70.37379903297453, Tokens per sec: 174394.0263176452, Loss: 2.7278683185577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1048 , TFLOPS: 71.5230451003753, Tokens per sec: 177241.98467256976, Loss: 2.72864031791687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1049 , TFLOPS: 71.53696389930087, Tokens per sec: 177276.47698958957, Loss: 2.7275617122650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1050 , TFLOPS: 70.52089696661443, Tokens per sec: 174758.55120137997, Loss: 2.714266061782837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1051 , TFLOPS: 70.36396469890131, Tokens per sec: 174369.65575447044, Loss: 2.737393379211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1052 , TFLOPS: 71.51499354618, Tokens per sec: 177222.03203432186, Loss: 2.7379844188690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1053 , TFLOPS: 70.53661109333241, Tokens per sec: 174797.49253844106, Loss: 2.729919195175171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1054 , TFLOPS: 70.37009228104262, Tokens per sec: 174384.84057802547, Loss: 2.732311964035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1055 , TFLOPS: 71.52082835654187, Tokens per sec: 177236.491337156, Loss: 2.735626459121704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1056 , TFLOPS: 71.51801829523019, Tokens per sec: 177229.52769567183, Loss: 2.732442855834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1057 , TFLOPS: 70.32140581995439, Tokens per sec: 174264.1901641924, Loss: 2.713162660598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1058 , TFLOPS: 71.5521513900078, Tokens per sec: 177314.11326460156, Loss: 2.7324728965759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1059 , TFLOPS: 71.5524711728578, Tokens per sec: 177314.90572173084, Loss: 2.726351022720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1060 , TFLOPS: 71.50367815997903, Tokens per sec: 177193.99126082318, Loss: 2.691227436065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1061 , TFLOPS: 69.69665740737675, Tokens per sec: 172715.99477610597, Loss: 2.728778600692749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1062 , TFLOPS: 71.50795706897846, Tokens per sec: 177204.59486868448, Loss: 2.7198522090911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1063 , TFLOPS: 71.51926217744216, Tokens per sec: 177232.61017281804, Loss: 2.742183208465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1064 , TFLOPS: 70.38806177815692, Tokens per sec: 174429.37097137826, Loss: 2.745680332183838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1065 , TFLOPS: 70.44839533525113, Tokens per sec: 174578.8841721477, Loss: 2.741644859313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1066 , TFLOPS: 71.51779538026291, Tokens per sec: 177228.97528782682, Loss: 2.7253799438476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1067 , TFLOPS: 70.38219969916615, Tokens per sec: 174414.84409387782, Loss: 2.7300875186920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1068 , TFLOPS: 70.57569636701157, Tokens per sec: 174894.3501522164, Loss: 2.7561941146850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1069 , TFLOPS: 71.53718436261828, Tokens per sec: 177277.02332197616, Loss: 2.719111919403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1070 , TFLOPS: 71.55396513593925, Tokens per sec: 177318.6079268203, Loss: 2.724332332611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1071 , TFLOPS: 70.3492723426961, Tokens per sec: 174333.24647729725, Loss: 2.7200305461883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1072 , TFLOPS: 70.45989469852934, Tokens per sec: 174607.38086111052, Loss: 2.735846996307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1073 , TFLOPS: 71.53384304126598, Tokens per sec: 177268.74315958822, Loss: 2.7154645919799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1074 , TFLOPS: 70.35973133919548, Tokens per sec: 174359.1650227763, Loss: 2.720764398574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1075 , TFLOPS: 71.49640582414546, Tokens per sec: 177175.96961151363, Loss: 2.723905563354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1076 , TFLOPS: 70.43220449682342, Tokens per sec: 174538.76149095572, Loss: 2.7232067584991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1077 , TFLOPS: 71.54226114596962, Tokens per sec: 177289.60415037465, Loss: 2.70406174659729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1078 , TFLOPS: 70.65853107958637, Tokens per sec: 175099.6236949736, Loss: 2.714078903198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1079 , TFLOPS: 71.50889605970096, Tokens per sec: 177206.9217911332, Loss: 2.7386279106140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1080 , TFLOPS: 70.56399683913581, Tokens per sec: 174865.35743333085, Loss: 2.7031984329223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1081 , TFLOPS: 70.37472939049081, Tokens per sec: 174396.3318460579, Loss: 2.7291324138641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1082 , TFLOPS: 71.53560540435706, Tokens per sec: 177273.11048946824, Loss: 2.713252067565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1083 , TFLOPS: 70.50215486224408, Tokens per sec: 174712.106202137, Loss: 2.6953084468841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1084 , TFLOPS: 70.365744948536, Tokens per sec: 174374.06740917588, Loss: 2.717154026031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1085 , TFLOPS: 71.45708301645963, Tokens per sec: 177078.52336230333, Loss: 2.7154641151428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1086 , TFLOPS: 71.50114231407964, Tokens per sec: 177187.70715533802, Loss: 2.712592840194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1087 , TFLOPS: 69.41232560899437, Tokens per sec: 172011.38925798115, Loss: 2.71811580657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1088 , TFLOPS: 71.52333630278859, Tokens per sec: 177242.7063042285, Loss: 2.7212448120117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1089 , TFLOPS: 71.5163902479708, Tokens per sec: 177225.49321521888, Loss: 2.74473237991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1090 , TFLOPS: 70.40445565563347, Tokens per sec: 174469.9968056998, Loss: 2.7203586101531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1091 , TFLOPS: 69.82890627153645, Tokens per sec: 173043.7220299088, Loss: 2.7080235481262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1092 , TFLOPS: 71.50025256065379, Tokens per sec: 177185.5022483343, Loss: 2.7450144290924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1093 , TFLOPS: 71.4850203092775, Tokens per sec: 177147.75505145284, Loss: 2.73099946975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1094 , TFLOPS: 69.4880799507051, Tokens per sec: 172199.11686177002, Loss: 2.6970460414886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1095 , TFLOPS: 71.52924094356703, Tokens per sec: 177257.33865452732, Loss: 2.7498598098754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1096 , TFLOPS: 71.47308868687385, Tokens per sec: 177118.1871767599, Loss: 2.7247045040130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1097 , TFLOPS: 71.52667497093015, Tokens per sec: 177250.9798916679, Loss: 2.7239418029785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1098 , TFLOPS: 69.35659056352213, Tokens per sec: 171873.27167557887, Loss: 2.725552797317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1099 , TFLOPS: 71.41620448823636, Tokens per sec: 176977.2218662244, Loss: 2.718292474746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1100 , TFLOPS: 71.45947244063147, Tokens per sec: 177084.44461861794, Loss: 2.7378389835357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1101 , TFLOPS: 70.36911863883361, Tokens per sec: 174382.42778537603, Loss: 2.7080585956573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1102 , TFLOPS: 70.48360727067029, Tokens per sec: 174666.14322703125, Loss: 2.725267171859741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1103 , TFLOPS: 71.48855231100019, Tokens per sec: 177156.5077408038, Loss: 2.7393648624420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1104 , TFLOPS: 70.24864931032992, Tokens per sec: 174083.8915185518, Loss: 2.70407772064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1105 , TFLOPS: 70.48188838466251, Tokens per sec: 174661.8836381531, Loss: 2.7050209045410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1106 , TFLOPS: 71.51812988852171, Tokens per sec: 177229.80423614127, Loss: 2.7211503982543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1107 , TFLOPS: 70.37009311346121, Tokens per sec: 174384.84264085046, Loss: 2.6983327865600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1108 , TFLOPS: 71.53245136960247, Tokens per sec: 177265.29444390076, Loss: 2.695960521697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1109 , TFLOPS: 70.5839638779228, Tokens per sec: 174914.8379549959, Loss: 2.7306063175201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1110 , TFLOPS: 71.53475550375846, Tokens per sec: 177271.00434216062, Loss: 2.709075689315796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1111 , TFLOPS: 70.38498865792549, Tokens per sec: 174421.75544091262, Loss: 2.7460169792175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1112 , TFLOPS: 71.54342402418362, Tokens per sec: 177292.48589069053, Loss: 2.7003581523895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1113 , TFLOPS: 70.47152172930814, Tokens per sec: 174636.19392420357, Loss: 2.7316293716430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1114 , TFLOPS: 70.3193987540541, Tokens per sec: 174259.21643379494, Loss: 2.7106730937957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1115 , TFLOPS: 71.51345544243733, Tokens per sec: 177218.2204438114, Loss: 2.7381861209869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1116 , TFLOPS: 71.47893297013357, Tokens per sec: 177132.66995447842, Loss: 2.714043617248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1117 , TFLOPS: 70.51542189920727, Tokens per sec: 174744.98338121653, Loss: 2.7379446029663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1118 , TFLOPS: 70.31443754661748, Tokens per sec: 174246.9220152449, Loss: 2.7374815940856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1119 , TFLOPS: 71.55094700568118, Tokens per sec: 177311.12866756629, Loss: 2.718172550201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1120 , TFLOPS: 70.45348353534472, Tokens per sec: 174591.49329249107, Loss: 2.727018117904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1121 , TFLOPS: 70.08747942246252, Tokens per sec: 173684.4947820876, Loss: 2.703141689300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1122 , TFLOPS: 71.48197023684493, Tokens per sec: 177140.19663597172, Loss: 2.687474489212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1123 , TFLOPS: 71.48753008852022, Tokens per sec: 177153.97456089134, Loss: 2.7275538444519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1124 , TFLOPS: 69.41074377899368, Tokens per sec: 172007.4693089865, Loss: 2.7195019721984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1125 , TFLOPS: 71.5295263883496, Tokens per sec: 177258.04601814286, Loss: 2.7351412773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1126 , TFLOPS: 71.52746830288943, Tokens per sec: 177252.94585565574, Loss: 2.7102177143096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1127 , TFLOPS: 70.63564592456322, Tokens per sec: 175042.9117598183, Loss: 2.6894631385803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1128 , TFLOPS: 70.30951717960798, Tokens per sec: 174234.72880377216, Loss: 2.7114293575286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1129 , TFLOPS: 71.53504417251203, Tokens per sec: 177271.71969513144, Loss: 2.7082016468048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1130 , TFLOPS: 71.4855568674162, Tokens per sec: 177149.0847016273, Loss: 2.708388328552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1131 , TFLOPS: 69.32392364996032, Tokens per sec: 171792.3194652151, Loss: 2.70158314704895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1132 , TFLOPS: 71.53358734986094, Tokens per sec: 177268.1095281198, Loss: 2.710369110107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1133 , TFLOPS: 71.54791234463202, Tokens per sec: 177303.608443191, Loss: 2.6974306106567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1134 , TFLOPS: 70.45708647664017, Tokens per sec: 174600.421777918, Loss: 2.688305377960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1135 , TFLOPS: 70.58188516946431, Tokens per sec: 174909.68668644712, Loss: 2.724273443222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1136 , TFLOPS: 71.53200904330384, Tokens per sec: 177264.19831060647, Loss: 2.713829278945923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1137 , TFLOPS: 71.47091666077311, Tokens per sec: 177112.80465681385, Loss: 2.734304904937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1138 , TFLOPS: 70.34562376847444, Tokens per sec: 174324.20490845235, Loss: 2.714574098587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1139 , TFLOPS: 70.11294567603069, Tokens per sec: 173747.60296376894, Loss: 2.6935324668884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1140 , TFLOPS: 71.54802331570798, Tokens per sec: 177303.88344174167, Loss: 2.70992112159729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1141 , TFLOPS: 70.35058039438843, Tokens per sec: 174336.48797348468, Loss: 2.6972484588623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1142 , TFLOPS: 70.55076317040732, Tokens per sec: 174832.56294441267, Loss: 2.704882860183716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1143 , TFLOPS: 71.5132613222289, Tokens per sec: 177217.7393925508, Loss: 2.736166477203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1144 , TFLOPS: 71.54887064050293, Tokens per sec: 177305.9832059241, Loss: 2.7005906105041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1145 , TFLOPS: 70.64115601216622, Tokens per sec: 175056.56636388478, Loss: 2.7246320247650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1146 , TFLOPS: 70.43191536535849, Tokens per sec: 174538.04499133516, Loss: 2.7072086334228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1147 , TFLOPS: 71.48861756495457, Tokens per sec: 177156.66944729106, Loss: 2.703864812850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1148 , TFLOPS: 69.96380613259974, Tokens per sec: 173378.01874607007, Loss: 2.7058653831481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1149 , TFLOPS: 71.47992526145728, Tokens per sec: 177135.1289616885, Loss: 2.7221128940582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1150 , TFLOPS: 70.55213169500874, Tokens per sec: 174835.95429913042, Loss: 2.703336238861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1151 , TFLOPS: 70.36472996990608, Tokens per sec: 174371.5521803218, Loss: 2.7054550647735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1152 , TFLOPS: 71.54103214367876, Tokens per sec: 177286.5585473121, Loss: 2.682528018951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1153 , TFLOPS: 71.51873872569857, Tokens per sec: 177231.31300173444, Loss: 2.715745210647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1154 , TFLOPS: 69.218803236739, Tokens per sec: 171531.81950128323, Loss: 2.7156758308410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1155 , TFLOPS: 71.49051491454927, Tokens per sec: 177161.37128859703, Loss: 2.722482681274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1156 , TFLOPS: 71.50763131920436, Tokens per sec: 177203.78762485998, Loss: 2.732466220855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1157 , TFLOPS: 70.5540719237559, Tokens per sec: 174840.7623996997, Loss: 2.7211086750030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1158 , TFLOPS: 70.40840965547437, Tokens per sec: 174479.7952528746, Loss: 2.7062840461730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1159 , TFLOPS: 71.52807098872232, Tokens per sec: 177254.43937754096, Loss: 2.7115323543548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1160 , TFLOPS: 71.50898851038265, Tokens per sec: 177207.15089410674, Loss: 2.7425081729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 70.22489532644079, Tokens per sec: 174025.0265297572, Loss: 2.6755335330963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1162 , TFLOPS: 71.52241459510157, Tokens per sec: 177240.42221104554, Loss: 2.7136950492858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1163 , TFLOPS: 71.52138925907194, Tokens per sec: 177237.88131541404, Loss: 2.6878538131713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1164 , TFLOPS: 70.51650671234866, Tokens per sec: 174747.6716676829, Loss: 2.7052485942840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1165 , TFLOPS: 70.3738309777358, Tokens per sec: 174394.10548028265, Loss: 2.7095839977264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1166 , TFLOPS: 71.4985979290063, Tokens per sec: 177181.4018888391, Loss: 2.705432653427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1167 , TFLOPS: 71.51877915739561, Tokens per sec: 177231.41319593365, Loss: 2.6990795135498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1168 , TFLOPS: 69.47510666396359, Tokens per sec: 172166.96762809902, Loss: 2.704984664916992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1169 , TFLOPS: 71.51156642791997, Tokens per sec: 177213.53925774532, Loss: 2.7056448459625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1170 , TFLOPS: 71.5562181801949, Tokens per sec: 177324.19121867913, Loss: 2.7090961933135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1171 , TFLOPS: 70.29763853220851, Tokens per sec: 174205.29220697525, Loss: 2.711637020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1172 , TFLOPS: 70.58758931550702, Tokens per sec: 174923.8221887045, Loss: 2.712541103363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1173 , TFLOPS: 71.5254493353188, Tokens per sec: 177247.94263160805, Loss: 2.7076308727264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1174 , TFLOPS: 71.5322156554822, Tokens per sec: 177264.71031834467, Loss: 2.7518229484558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1175 , TFLOPS: 70.74036584321088, Tokens per sec: 175302.41925407853, Loss: 2.720877170562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1176 , TFLOPS: 70.56506678236155, Tokens per sec: 174868.0088705082, Loss: 2.71578311920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1177 , TFLOPS: 71.49075093309428, Tokens per sec: 177161.95616855114, Loss: 2.701367139816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1178 , TFLOPS: 70.28363563124095, Tokens per sec: 174170.59147583076, Loss: 2.6798741817474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1179 , TFLOPS: 70.44696855310048, Tokens per sec: 174575.3484488053, Loss: 2.6873059272766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1180 , TFLOPS: 71.54387311415627, Tokens per sec: 177293.59878511413, Loss: 2.7108213901519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1181 , TFLOPS: 70.46554877515052, Tokens per sec: 174621.39228582577, Loss: 2.7098066806793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1182 , TFLOPS: 71.47618631693275, Tokens per sec: 177125.863445278, Loss: 2.6983747482299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1183 , TFLOPS: 70.57951331267775, Tokens per sec: 174903.80896404842, Loss: 2.7033438682556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1184 , TFLOPS: 71.50971379108763, Tokens per sec: 177208.94821958995, Loss: 2.7349610328674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1185 , TFLOPS: 70.44425809053858, Tokens per sec: 174568.63162399488, Loss: 2.7073516845703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1186 , TFLOPS: 71.5455008563759, Tokens per sec: 177297.63250964493, Loss: 2.699655771255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1187 , TFLOPS: 70.46039134387907, Tokens per sec: 174608.61160299715, Loss: 2.7341394424438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1188 , TFLOPS: 70.384414690695, Tokens per sec: 174420.33308688775, Loss: 2.7036445140838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1189 , TFLOPS: 71.55653535913457, Tokens per sec: 177324.97722303178, Loss: 2.7026097774505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1190 , TFLOPS: 71.54269520759848, Tokens per sec: 177290.6798029068, Loss: 2.7086780071258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1191 , TFLOPS: 69.41981714796854, Tokens per sec: 172029.95411682033, Loss: 2.7397892475128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1192 , TFLOPS: 71.4908459336843, Tokens per sec: 177162.1915904784, Loss: 2.6927807331085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1193 , TFLOPS: 71.56490065795565, Tokens per sec: 177345.70735502415, Loss: 2.7181179523468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1194 , TFLOPS: 70.45206521903518, Tokens per sec: 174587.9785484356, Loss: 2.68916916847229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1195 , TFLOPS: 70.38413597282515, Tokens per sec: 174419.64239330296, Loss: 2.707839250564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1196 , TFLOPS: 71.50054256358858, Tokens per sec: 177186.2209075532, Loss: 2.7003443241119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1197 , TFLOPS: 71.44236497650938, Tokens per sec: 177042.05043798062, Loss: 2.717088460922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1198 , TFLOPS: 69.48412263828654, Tokens per sec: 172189.31020566286, Loss: 2.7139244079589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1199 , TFLOPS: 71.55509279601007, Tokens per sec: 177321.4023926978, Loss: 2.692202568054199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1200 , TFLOPS: 71.50562005777881, Tokens per sec: 177198.80349748876, Loss: 2.7274394035339355 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/optimizer.pt +[rank0]:[2024-08-26 04:36:48,305] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00636672600012389, 'preprocessing_with_comm': 0.0013175639996916289, 'state_converting': 0.9881797220004955, : 0.9972303859995009}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200, took 4.50s + eval ppl=11.835589408874512, eval loss=2.4711110591888428 +------------------------------------------------------------------ +iteration: 1201 , TFLOPS: 65.9000825194928, Tokens per sec: 163307.66397668008, Loss: 2.708641529083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1202 , TFLOPS: 71.46021610427489, Tokens per sec: 177086.28749904764, Loss: 2.7012012004852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1203 , TFLOPS: 71.46494476162431, Tokens per sec: 177098.0056328641, Loss: 2.7103514671325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1204 , TFLOPS: 71.49495909989268, Tokens per sec: 177172.38446944545, Loss: 2.7083537578582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1205 , TFLOPS: 69.27785564827258, Tokens per sec: 171678.15788222355, Loss: 2.7138538360595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1206 , TFLOPS: 71.48572718941011, Tokens per sec: 177149.50677829076, Loss: 2.712775230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1207 , TFLOPS: 71.52345243463581, Tokens per sec: 177242.99409173886, Loss: 2.698054790496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1208 , TFLOPS: 70.43207145558263, Tokens per sec: 174538.43180010535, Loss: 2.7221834659576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1209 , TFLOPS: 70.35482441427051, Tokens per sec: 174347.00512226982, Loss: 2.719228982925415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1210 , TFLOPS: 71.42191563376895, Tokens per sec: 176991.37471398877, Loss: 2.7102162837982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1211 , TFLOPS: 70.3681522251798, Tokens per sec: 174380.0329058826, Loss: 2.714421033859253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1212 , TFLOPS: 70.39490297199254, Tokens per sec: 174446.3242033226, Loss: 2.693444013595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1213 , TFLOPS: 71.4749288701214, Tokens per sec: 177122.74735356026, Loss: 2.6834564208984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1214 , TFLOPS: 71.5268494025296, Tokens per sec: 177251.4121523866, Loss: 2.698718309402466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1215 , TFLOPS: 70.63896862006986, Tokens per sec: 175051.14576530887, Loss: 2.6953322887420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1216 , TFLOPS: 70.46650062746991, Tokens per sec: 174623.7510807284, Loss: 2.708526849746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1217 , TFLOPS: 71.50889175203926, Tokens per sec: 177206.911116273, Loss: 2.702444553375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1218 , TFLOPS: 70.41341177792116, Tokens per sec: 174492.19106332705, Loss: 2.699876546859741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1219 , TFLOPS: 71.56091944957585, Tokens per sec: 177335.84148209268, Loss: 2.710386276245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1220 , TFLOPS: 70.39348984916641, Tokens per sec: 174442.82232929114, Loss: 2.690082311630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1221 , TFLOPS: 69.87724785263421, Tokens per sec: 173163.5177931338, Loss: 2.7220330238342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1222 , TFLOPS: 71.499095903656, Tokens per sec: 177182.6359248775, Loss: 2.7296085357666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1223 , TFLOPS: 71.5295203386809, Tokens per sec: 177258.03102639737, Loss: 2.696113109588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1224 , TFLOPS: 70.432157067017, Tokens per sec: 174538.64395467058, Loss: 2.687171697616577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1225 , TFLOPS: 70.42711714002469, Tokens per sec: 174526.15446038987, Loss: 2.6938695907592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1226 , TFLOPS: 71.48513255243903, Tokens per sec: 177148.0332023717, Loss: 2.7039904594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1227 , TFLOPS: 70.45403955964555, Tokens per sec: 174592.8711819597, Loss: 2.7204174995422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1228 , TFLOPS: 70.34541793479035, Tokens per sec: 174323.69482990872, Loss: 2.707799196243286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1229 , TFLOPS: 70.56679889836091, Tokens per sec: 174872.30124476296, Loss: 2.6901021003723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1230 , TFLOPS: 71.5041789576994, Tokens per sec: 177195.23229274168, Loss: 2.682431221008301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1231 , TFLOPS: 70.38021395482005, Tokens per sec: 174409.9232006404, Loss: 2.702075958251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1232 , TFLOPS: 70.68400373625096, Tokens per sec: 175162.7477442332, Loss: 2.7177305221557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1233 , TFLOPS: 71.51030558205314, Tokens per sec: 177210.41474279363, Loss: 2.7036244869232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1234 , TFLOPS: 70.38180486731875, Tokens per sec: 174413.86565706565, Loss: 2.7215874195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1235 , TFLOPS: 70.31427724277313, Tokens per sec: 174246.52476465967, Loss: 2.7315709590911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1236 , TFLOPS: 71.45982443038396, Tokens per sec: 177085.31688799948, Loss: 2.6904709339141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1237 , TFLOPS: 71.53683248995996, Tokens per sec: 177276.15134276685, Loss: 2.704852819442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1238 , TFLOPS: 69.37446206415449, Tokens per sec: 171917.55922285418, Loss: 2.692072629928589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1239 , TFLOPS: 71.47587871783008, Tokens per sec: 177125.1011808167, Loss: 2.7182974815368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1240 , TFLOPS: 71.54575955279037, Tokens per sec: 177298.27358785778, Loss: 2.7018866539001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1241 , TFLOPS: 70.38542025302401, Tokens per sec: 174422.82498111087, Loss: 2.7054476737976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1242 , TFLOPS: 70.39288632693139, Tokens per sec: 174441.32673470932, Loss: 2.7039778232574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1243 , TFLOPS: 71.48243545505349, Tokens per sec: 177141.34949794057, Loss: 2.719075918197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1244 , TFLOPS: 71.48458749734891, Tokens per sec: 177146.68249581565, Loss: 2.707207202911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1245 , TFLOPS: 70.44821194108452, Tokens per sec: 174578.4297012003, Loss: 2.712860345840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1246 , TFLOPS: 70.47680073431876, Tokens per sec: 174649.27588015026, Loss: 2.712164878845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1247 , TFLOPS: 71.5169380100751, Tokens per sec: 177226.8506300543, Loss: 2.6909427642822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1248 , TFLOPS: 70.40948955859379, Tokens per sec: 174482.47137176569, Loss: 2.7070693969726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1249 , TFLOPS: 70.37119841496606, Tokens per sec: 174387.58169973848, Loss: 2.719202995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1250 , TFLOPS: 71.4833689691486, Tokens per sec: 177143.66284870333, Loss: 2.7004926204681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1251 , TFLOPS: 71.50108750471253, Tokens per sec: 177187.57133168852, Loss: 2.6980135440826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1252 , TFLOPS: 70.41496247906863, Tokens per sec: 174496.0338715943, Loss: 2.6905677318573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1253 , TFLOPS: 70.38599169139414, Tokens per sec: 174424.24106834122, Loss: 2.6988799571990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1254 , TFLOPS: 71.53004329110466, Tokens per sec: 177259.32696011144, Loss: 2.702345848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1255 , TFLOPS: 70.4495567006531, Tokens per sec: 174581.7621635487, Loss: 2.697626829147339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1256 , TFLOPS: 71.49914517719476, Tokens per sec: 177182.7580301345, Loss: 2.6708221435546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1257 , TFLOPS: 70.42444593793724, Tokens per sec: 174519.53492735996, Loss: 2.6974551677703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1258 , TFLOPS: 70.3528653154971, Tokens per sec: 174342.15025969676, Loss: 2.694295883178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1259 , TFLOPS: 71.43866433459604, Tokens per sec: 177032.87983966002, Loss: 2.699432849884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1260 , TFLOPS: 71.51497388791238, Tokens per sec: 177221.98331896914, Loss: 2.6915969848632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1261 , TFLOPS: 69.3183895178171, Tokens per sec: 171778.60527612918, Loss: 2.7081480026245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1262 , TFLOPS: 71.4973326451392, Tokens per sec: 177178.2663760339, Loss: 2.704850196838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1263 , TFLOPS: 71.5044092431968, Tokens per sec: 177195.80296557225, Loss: 2.7107794284820557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1264 , TFLOPS: 70.39469173001866, Tokens per sec: 174445.8007224411, Loss: 2.6923646926879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1265 , TFLOPS: 70.43553331488948, Tokens per sec: 174547.010668823, Loss: 2.6827192306518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1266 , TFLOPS: 71.51135283556064, Tokens per sec: 177213.0099523497, Loss: 2.6924734115600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1267 , TFLOPS: 71.52198065392017, Tokens per sec: 177239.34685699546, Loss: 2.6910717487335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1268 , TFLOPS: 69.26943266072216, Tokens per sec: 171657.28479120496, Loss: 2.6979191303253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1269 , TFLOPS: 71.52004056295885, Tokens per sec: 177234.53909787466, Loss: 2.6915740966796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1270 , TFLOPS: 71.53464656492984, Tokens per sec: 177270.7343797427, Loss: 2.682840585708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1271 , TFLOPS: 70.46309035969234, Tokens per sec: 174615.30006150357, Loss: 2.688894271850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1272 , TFLOPS: 70.48375578300397, Tokens per sec: 174666.5112569541, Loss: 2.6858370304107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1273 , TFLOPS: 71.49086942081317, Tokens per sec: 177162.24979417108, Loss: 2.7005832195281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1274 , TFLOPS: 71.52269973418164, Tokens per sec: 177241.12881709664, Loss: 2.6819698810577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1275 , TFLOPS: 69.33353260582632, Tokens per sec: 171816.13151636493, Loss: 2.697965383529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1276 , TFLOPS: 71.53997372555075, Tokens per sec: 177283.93567059652, Loss: 2.7063369750976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1277 , TFLOPS: 71.55872890889434, Tokens per sec: 177330.41308097594, Loss: 2.7078027725219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1278 , TFLOPS: 70.4910843544886, Tokens per sec: 174684.67226440183, Loss: 2.6864280700683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1279 , TFLOPS: 70.31830609977062, Tokens per sec: 174256.50871611468, Loss: 2.6883387565612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1280 , TFLOPS: 71.51221879420949, Tokens per sec: 177215.15589327476, Loss: 2.683474063873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1281 , TFLOPS: 71.49570311611922, Tokens per sec: 177174.22822361495, Loss: 2.6795105934143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1282 , TFLOPS: 70.65459895900302, Tokens per sec: 175089.87946700922, Loss: 2.6910417079925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1283 , TFLOPS: 70.45758620481017, Tokens per sec: 174601.66015937284, Loss: 2.717373847961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1284 , TFLOPS: 71.51897529776569, Tokens per sec: 177231.89925337734, Loss: 2.703571081161499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1285 , TFLOPS: 70.44819434601334, Tokens per sec: 174578.38609867563, Loss: 2.6865451335906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1286 , TFLOPS: 70.38998558125664, Tokens per sec: 174434.13836738374, Loss: 2.6743052005767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1287 , TFLOPS: 71.4902438170635, Tokens per sec: 177160.69947916345, Loss: 2.690436363220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1288 , TFLOPS: 70.32348521463747, Tokens per sec: 174269.34313328133, Loss: 2.6662120819091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1289 , TFLOPS: 71.54282538165981, Tokens per sec: 177291.00238857063, Loss: 2.6987485885620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1290 , TFLOPS: 70.39058026593749, Tokens per sec: 174435.61206153283, Loss: 2.691760778427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1291 , TFLOPS: 71.52118097031467, Tokens per sec: 177237.36515292866, Loss: 2.6806159019470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1292 , TFLOPS: 70.33136249992265, Tokens per sec: 174288.8639139726, Loss: 2.692873477935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1293 , TFLOPS: 71.52938375747813, Tokens per sec: 177257.69256313125, Loss: 2.6985013484954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1294 , TFLOPS: 70.39809697145708, Tokens per sec: 174454.23928584383, Loss: 2.7030467987060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1295 , TFLOPS: 70.41669800426469, Tokens per sec: 174500.334694214, Loss: 2.6720011234283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1296 , TFLOPS: 71.50541496090524, Tokens per sec: 177198.29524484274, Loss: 2.6832633018493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1297 , TFLOPS: 71.5338126829352, Tokens per sec: 177268.6679283003, Loss: 2.7086169719696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1298 , TFLOPS: 70.41683210196803, Tokens per sec: 174500.66700309498, Loss: 2.684354782104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1299 , TFLOPS: 70.35444141692662, Tokens per sec: 174346.05601266102, Loss: 2.726032257080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1300 , TFLOPS: 71.5209536685381, Tokens per sec: 177236.8018740869, Loss: 2.709386110305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1301 , TFLOPS: 70.37860989484852, Tokens per sec: 174405.94816332913, Loss: 2.688960313796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1302 , TFLOPS: 70.33649545235613, Tokens per sec: 174301.58393554416, Loss: 2.6859021186828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1303 , TFLOPS: 71.50936509585465, Tokens per sec: 177208.08411439066, Loss: 2.695580005645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1304 , TFLOPS: 71.50281573009548, Tokens per sec: 177191.85406456722, Loss: 2.691582441329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1305 , TFLOPS: 69.33458047818843, Tokens per sec: 171818.7282595109, Loss: 2.7069308757781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1306 , TFLOPS: 71.5346500057921, Tokens per sec: 177270.74290657847, Loss: 2.6733455657958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1307 , TFLOPS: 71.51421843895777, Tokens per sec: 177220.11123324005, Loss: 2.7000651359558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1308 , TFLOPS: 70.30002973479588, Tokens per sec: 174211.21787039944, Loss: 2.693570137023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1309 , TFLOPS: 71.50547800510986, Tokens per sec: 177198.4514753267, Loss: 2.7014100551605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1310 , TFLOPS: 71.52381621510123, Tokens per sec: 177243.8955798059, Loss: 2.694956064224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1311 , TFLOPS: 71.53581311989468, Tokens per sec: 177273.62523145223, Loss: 2.6802055835723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1312 , TFLOPS: 68.99892468126538, Tokens per sec: 170986.93621919714, Loss: 2.7112581729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1313 , TFLOPS: 71.54726299142717, Tokens per sec: 177301.99927441723, Loss: 2.6974804401397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1314 , TFLOPS: 71.50704215117288, Tokens per sec: 177202.32760157576, Loss: 2.6878714561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1315 , TFLOPS: 70.38907559577846, Tokens per sec: 174431.88332312566, Loss: 2.694218873977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1316 , TFLOPS: 70.44400652986837, Tokens per sec: 174568.0082289424, Loss: 2.6725053787231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1317 , TFLOPS: 71.53261281081039, Tokens per sec: 177265.69451299822, Loss: 2.694157361984253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1318 , TFLOPS: 71.53490730925319, Tokens per sec: 177271.38053289932, Loss: 2.6766765117645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1319 , TFLOPS: 70.42732955096128, Tokens per sec: 174526.6808380897, Loss: 2.680549383163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1320 , TFLOPS: 70.45691844258963, Tokens per sec: 174600.00537103025, Loss: 2.6872048377990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1321 , TFLOPS: 71.52726928454601, Tokens per sec: 177252.45266627707, Loss: 2.70039701461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1322 , TFLOPS: 70.37549724369988, Tokens per sec: 174398.23467089597, Loss: 2.6987898349761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1323 , TFLOPS: 70.39938336193126, Tokens per sec: 174457.42710314647, Loss: 2.693326473236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1324 , TFLOPS: 71.52116991725983, Tokens per sec: 177237.33776224114, Loss: 2.7099051475524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1325 , TFLOPS: 70.39135210569931, Tokens per sec: 174437.5247654878, Loss: 2.6758546829223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1326 , TFLOPS: 71.50387022915672, Tokens per sec: 177194.46722940364, Loss: 2.6887166500091553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1327 , TFLOPS: 70.34362103047744, Tokens per sec: 174319.24190307598, Loss: 2.6823792457580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1328 , TFLOPS: 71.52753880752763, Tokens per sec: 177253.12057391595, Loss: 2.702540636062622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1329 , TFLOPS: 70.69137227400898, Tokens per sec: 175181.00779250904, Loss: 2.692488193511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1330 , TFLOPS: 71.51814995427078, Tokens per sec: 177229.85396127787, Loss: 2.6873412132263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1331 , TFLOPS: 70.37650858516804, Tokens per sec: 174400.74088646256, Loss: 2.70108699798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1332 , TFLOPS: 70.33393182573725, Tokens per sec: 174295.2309863758, Loss: 2.705242395401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1333 , TFLOPS: 71.51219356403568, Tokens per sec: 177215.09337012476, Loss: 2.674088478088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1334 , TFLOPS: 71.48984342520698, Tokens per sec: 177159.70726403617, Loss: 2.694967746734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1335 , TFLOPS: 69.36402544604151, Tokens per sec: 171891.6961334818, Loss: 2.6760354042053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1336 , TFLOPS: 71.53716354186733, Tokens per sec: 177276.9717258617, Loss: 2.678378105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1337 , TFLOPS: 71.47662969971228, Tokens per sec: 177126.9621966483, Loss: 2.6999995708465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1338 , TFLOPS: 70.36936580211264, Tokens per sec: 174383.0402832087, Loss: 2.6665585041046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1339 , TFLOPS: 69.8895098482978, Tokens per sec: 173193.9043691585, Loss: 2.6881325244903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1340 , TFLOPS: 71.49691637795117, Tokens per sec: 177177.23482008636, Loss: 2.67875599861145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1341 , TFLOPS: 71.5378264765192, Tokens per sec: 177278.6145509573, Loss: 2.683882713317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1342 , TFLOPS: 69.35945842702041, Tokens per sec: 171880.37855724886, Loss: 2.687940835952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1343 , TFLOPS: 71.52104548372628, Tokens per sec: 177237.02940223753, Loss: 2.680903434753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1344 , TFLOPS: 71.51058650991854, Tokens per sec: 177211.11091299084, Loss: 2.6727640628814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1345 , TFLOPS: 70.26456918540013, Tokens per sec: 174123.3427226948, Loss: 2.694638252258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1346 , TFLOPS: 71.43365930062379, Tokens per sec: 177020.4768141268, Loss: 2.672513723373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1347 , TFLOPS: 71.53598055940012, Tokens per sec: 177274.0401649917, Loss: 2.6916491985321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1348 , TFLOPS: 71.5498808872968, Tokens per sec: 177308.48670876716, Loss: 2.683779001235962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1349 , TFLOPS: 69.34045365682219, Tokens per sec: 171833.28264314937, Loss: 2.679044485092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1350 , TFLOPS: 71.51762621931307, Tokens per sec: 177228.5560883585, Loss: 2.7020649909973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1351 , TFLOPS: 71.54432444306708, Tokens per sec: 177294.7172278731, Loss: 2.6894607543945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1352 , TFLOPS: 70.37842801457931, Tokens per sec: 174405.49744398627, Loss: 2.6728134155273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1353 , TFLOPS: 70.41078001745562, Tokens per sec: 174485.6692709811, Loss: 2.6887545585632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1354 , TFLOPS: 71.50185909238314, Tokens per sec: 177189.48341093372, Loss: 2.6863791942596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1355 , TFLOPS: 70.36472625722887, Tokens per sec: 174371.5429798987, Loss: 2.6827645301818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1356 , TFLOPS: 71.53212567688689, Tokens per sec: 177264.48734147355, Loss: 2.6745777130126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1357 , TFLOPS: 70.39478382120936, Tokens per sec: 174446.02893455635, Loss: 2.676807403564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1358 , TFLOPS: 71.47000904855845, Tokens per sec: 177110.5554937642, Loss: 2.6799535751342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1359 , TFLOPS: 70.43263995997626, Tokens per sec: 174539.84061661866, Loss: 2.674987316131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1360 , TFLOPS: 70.40052010173073, Tokens per sec: 174460.2440695918, Loss: 2.6908223628997803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1361 , TFLOPS: 71.47613413191701, Tokens per sec: 177125.73412506032, Loss: 2.693419933319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1362 , TFLOPS: 70.33742537510683, Tokens per sec: 174303.88838655985, Loss: 2.686129331588745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1363 , TFLOPS: 71.48232605782893, Tokens per sec: 177141.078399567, Loss: 2.695408582687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1364 , TFLOPS: 70.36973251907688, Tokens per sec: 174383.94904824335, Loss: 2.6894161701202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1365 , TFLOPS: 71.48337387213415, Tokens per sec: 177143.6749988416, Loss: 2.686018943786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1366 , TFLOPS: 70.6815620904447, Tokens per sec: 175156.6970769558, Loss: 2.664257764816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1367 , TFLOPS: 71.54050854723027, Tokens per sec: 177285.26101763386, Loss: 2.6627306938171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1368 , TFLOPS: 70.42781452311966, Tokens per sec: 174527.8826525223, Loss: 2.698918342590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1369 , TFLOPS: 70.42323615583905, Tokens per sec: 174516.53695405214, Loss: 2.668862819671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1370 , TFLOPS: 71.50195709672299, Tokens per sec: 177189.72627648394, Loss: 2.6875712871551514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1371 , TFLOPS: 71.4838883225019, Tokens per sec: 177144.94986352426, Loss: 2.679710626602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1372 , TFLOPS: 68.96592148352481, Tokens per sec: 170905.15065959364, Loss: 2.685100793838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1373 , TFLOPS: 71.46824182177889, Tokens per sec: 177106.1761111283, Loss: 2.7107691764831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1374 , TFLOPS: 71.51740105582257, Tokens per sec: 177227.99810842526, Loss: 2.677196979522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1375 , TFLOPS: 69.37288259570497, Tokens per sec: 171913.64512604344, Loss: 2.690535545349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1376 , TFLOPS: 71.49077418847813, Tokens per sec: 177162.0137979541, Loss: 2.7109968662261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1377 , TFLOPS: 71.553555278196, Tokens per sec: 177317.59225418296, Loss: 2.6820716857910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1378 , TFLOPS: 71.53458543134506, Tokens per sec: 177270.58288398522, Loss: 2.6711061000823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1379 , TFLOPS: 69.32501575702656, Tokens per sec: 171795.02582683077, Loss: 2.7128047943115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1380 , TFLOPS: 71.52231362640109, Tokens per sec: 177240.1719994831, Loss: 2.6806888580322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1381 , TFLOPS: 71.52891082505968, Tokens per sec: 177256.52058450057, Loss: 2.6809704303741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1382 , TFLOPS: 69.197988816225, Tokens per sec: 171480.23907435182, Loss: 2.7055740356445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1383 , TFLOPS: 71.54981626600137, Tokens per sec: 177308.32657007847, Loss: 2.661391258239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1384 , TFLOPS: 71.5337028287863, Tokens per sec: 177268.3956976179, Loss: 2.673225164413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1385 , TFLOPS: 71.51980937488462, Tokens per sec: 177233.96618835925, Loss: 2.709527015686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1386 , TFLOPS: 69.29567372958326, Tokens per sec: 171722.3130505351, Loss: 2.6982529163360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1387 , TFLOPS: 71.49794427596728, Tokens per sec: 177179.7820646026, Loss: 2.6691269874572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1388 , TFLOPS: 71.52727229954853, Tokens per sec: 177252.4601377854, Loss: 2.6634879112243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1389 , TFLOPS: 70.24599682031167, Tokens per sec: 174077.31835609072, Loss: 2.6814839839935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1390 , TFLOPS: 70.63333213857841, Tokens per sec: 175037.17794326338, Loss: 2.6840052604675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1391 , TFLOPS: 71.51732733483189, Tokens per sec: 177227.8154196893, Loss: 2.6874170303344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1392 , TFLOPS: 70.39297010232089, Tokens per sec: 174441.53433935312, Loss: 2.6792757511138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1393 , TFLOPS: 71.50827683358482, Tokens per sec: 177205.38728060402, Loss: 2.6526029109954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1394 , TFLOPS: 70.46224938836714, Tokens per sec: 174613.21604191908, Loss: 2.688286304473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1395 , TFLOPS: 70.44201811182286, Tokens per sec: 174563.08070998325, Loss: 2.6846067905426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1396 , TFLOPS: 71.52712448842, Tokens per sec: 177252.0938455262, Loss: 2.689772129058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1397 , TFLOPS: 70.57497053818523, Tokens per sec: 174892.55146842805, Loss: 2.679419755935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1398 , TFLOPS: 71.46100650621362, Tokens per sec: 177088.24620212178, Loss: 2.6933040618896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1399 , TFLOPS: 70.3059838287077, Tokens per sec: 174225.97277101054, Loss: 2.6661438941955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1400 , TFLOPS: 71.50315227102173, Tokens per sec: 177192.68805005553, Loss: 2.70222806930542 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/optimizer.pt +[rank0]:[2024-08-26 05:01:50,073] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006396287000825396, 'preprocessing_with_comm': 0.0012815520003641723, 'state_converting': 0.967197068999667, : 0.9762561279985675}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400, took 4.49s + eval ppl=12.776593208312988, eval loss=2.547614812850952 +------------------------------------------------------------------ +iteration: 1401 , TFLOPS: 64.69346827260672, Tokens per sec: 160317.54095336428, Loss: 2.7061963081359863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1402 , TFLOPS: 69.5806144299804, Tokens per sec: 172428.4274949286, Loss: 2.6969401836395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1403 , TFLOPS: 71.45246539923238, Tokens per sec: 177067.08039814167, Loss: 2.666541814804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1404 , TFLOPS: 71.52081134235875, Tokens per sec: 177236.44917413598, Loss: 2.6613645553588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1405 , TFLOPS: 68.42792189215423, Tokens per sec: 169571.92840662986, Loss: 2.6948862075805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1406 , TFLOPS: 71.48700613590823, Tokens per sec: 177152.67614860064, Loss: 2.70902943611145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1407 , TFLOPS: 71.51891008499842, Tokens per sec: 177231.73764895627, Loss: 2.6956536769866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1408 , TFLOPS: 70.43713185533242, Tokens per sec: 174550.9720281344, Loss: 2.679354667663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1409 , TFLOPS: 69.51849874263013, Tokens per sec: 172274.49797906677, Loss: 2.6790075302124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1410 , TFLOPS: 71.48955369246067, Tokens per sec: 177158.98927437418, Loss: 2.690619707107544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1411 , TFLOPS: 71.46496216403136, Tokens per sec: 177098.04875794583, Loss: 2.702465295791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1412 , TFLOPS: 69.26542487132245, Tokens per sec: 171647.353047578, Loss: 2.6738851070404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1413 , TFLOPS: 71.48395948446544, Tokens per sec: 177145.12621070907, Loss: 2.663151979446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1414 , TFLOPS: 71.5234293222676, Tokens per sec: 177242.9368167444, Loss: 2.6957902908325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1415 , TFLOPS: 70.68155988236184, Tokens per sec: 175156.69160508327, Loss: 2.6733179092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1416 , TFLOPS: 69.34946049431142, Tokens per sec: 171855.60257863274, Loss: 2.6716699600219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1417 , TFLOPS: 71.42665883321456, Tokens per sec: 177003.12888472158, Loss: 2.657731056213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1418 , TFLOPS: 71.46414927576156, Tokens per sec: 177096.03433126622, Loss: 2.675180196762085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1419 , TFLOPS: 69.6002716262592, Tokens per sec: 172477.14019272642, Loss: 2.6786227226257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1420 , TFLOPS: 70.40005105438752, Tokens per sec: 174459.08171860548, Loss: 2.687411308288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1421 , TFLOPS: 71.51147087962936, Tokens per sec: 177213.30247855562, Loss: 2.6688284873962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1422 , TFLOPS: 69.49211445150206, Tokens per sec: 172209.11479918184, Loss: 2.68569016456604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1423 , TFLOPS: 70.48439691645913, Tokens per sec: 174668.10005628268, Loss: 2.681431770324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1424 , TFLOPS: 71.49520638592092, Tokens per sec: 177172.99727146424, Loss: 2.675217390060425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1425 , TFLOPS: 70.36552803649793, Tokens per sec: 174373.52987725072, Loss: 2.670449733734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1426 , TFLOPS: 70.96153940822842, Tokens per sec: 175850.5117124715, Loss: 2.686176300048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1427 , TFLOPS: 70.45434765225056, Tokens per sec: 174593.6346693741, Loss: 2.6631739139556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1428 , TFLOPS: 71.5025056613639, Tokens per sec: 177191.08568009347, Loss: 2.683074712753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1429 , TFLOPS: 69.47637977452962, Tokens per sec: 172170.12253632635, Loss: 2.6811776161193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1430 , TFLOPS: 71.50591507793268, Tokens per sec: 177199.53458992916, Loss: 2.673778533935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1431 , TFLOPS: 70.37025362513678, Tokens per sec: 174385.24040646435, Loss: 2.6796324253082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1432 , TFLOPS: 70.3527208038683, Tokens per sec: 174341.79214396133, Loss: 2.666877031326294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1433 , TFLOPS: 71.5090924839051, Tokens per sec: 177207.40855194902, Loss: 2.6903913021087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1434 , TFLOPS: 70.56843796044375, Tokens per sec: 174876.3630211618, Loss: 2.6569674015045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1435 , TFLOPS: 70.58529155981363, Tokens per sec: 174918.1280969776, Loss: 2.7051422595977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1436 , TFLOPS: 69.90510791231495, Tokens per sec: 173232.55809006162, Loss: 2.678124189376831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1437 , TFLOPS: 71.51855262444766, Tokens per sec: 177230.85182233396, Loss: 2.681593179702759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1438 , TFLOPS: 70.426520891644, Tokens per sec: 174524.6768912203, Loss: 2.6779890060424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1439 , TFLOPS: 69.56165960003293, Tokens per sec: 172381.45533827186, Loss: 2.669689893722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1440 , TFLOPS: 71.52415655787875, Tokens per sec: 177244.73898670045, Loss: 2.6476259231567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1441 , TFLOPS: 71.50277477565379, Tokens per sec: 177191.75257494923, Loss: 2.668996572494507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1442 , TFLOPS: 68.52513893296572, Tokens per sec: 169812.84294310186, Loss: 2.6676697731018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1443 , TFLOPS: 71.48922852541102, Tokens per sec: 177158.183474605, Loss: 2.6708078384399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1444 , TFLOPS: 71.50635899380282, Tokens per sec: 177200.63466235672, Loss: 2.6823930740356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1445 , TFLOPS: 70.569697454456, Tokens per sec: 174879.4841860689, Loss: 2.698810338973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1446 , TFLOPS: 69.71651723191192, Tokens per sec: 172765.20960904442, Loss: 2.671205997467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1447 , TFLOPS: 71.38341576476832, Tokens per sec: 176895.96779749452, Loss: 2.690627098083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1448 , TFLOPS: 71.44059632097053, Tokens per sec: 177037.66751472186, Loss: 2.6498451232910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1449 , TFLOPS: 68.54256715468914, Tokens per sec: 169856.0319964096, Loss: 2.6705923080444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1450 , TFLOPS: 71.5265474163615, Tokens per sec: 177250.66379739548, Loss: 2.6786048412323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1451 , TFLOPS: 71.50341155936755, Tokens per sec: 177193.33059513944, Loss: 2.6755378246307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1452 , TFLOPS: 70.15146727035399, Tokens per sec: 173843.06371800601, Loss: 2.6591498851776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1453 , TFLOPS: 70.53804106722882, Tokens per sec: 174801.03617128098, Loss: 2.6518630981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1454 , TFLOPS: 71.4874720181178, Tokens per sec: 177153.83065603714, Loss: 2.671241283416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1455 , TFLOPS: 70.70229175899836, Tokens per sec: 175208.0674791933, Loss: 2.6431198120117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1456 , TFLOPS: 70.30194002837452, Tokens per sec: 174215.95178832355, Loss: 2.667607545852661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1457 , TFLOPS: 70.4264903698521, Tokens per sec: 174524.60125485776, Loss: 2.6786293983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1458 , TFLOPS: 71.54719491344473, Tokens per sec: 177301.83056967732, Loss: 2.6789636611938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1459 , TFLOPS: 69.43034087271876, Tokens per sec: 172056.03306603536, Loss: 2.6732304096221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1460 , TFLOPS: 70.57391772675066, Tokens per sec: 174889.94248571643, Loss: 2.707926034927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1461 , TFLOPS: 71.48374377918873, Tokens per sec: 177144.59166927147, Loss: 2.683542013168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1462 , TFLOPS: 70.68742487010043, Tokens per sec: 175171.22569078123, Loss: 2.688368558883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1463 , TFLOPS: 70.65547647441483, Tokens per sec: 175092.05404686596, Loss: 2.6668314933776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1464 , TFLOPS: 68.57728160329111, Tokens per sec: 169942.0582241575, Loss: 2.6881771087646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1465 , TFLOPS: 71.49649997080738, Tokens per sec: 177176.202917313, Loss: 2.6611835956573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1466 , TFLOPS: 69.40112621137952, Tokens per sec: 171983.6359169761, Loss: 2.6968185901641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1467 , TFLOPS: 71.52332723951429, Tokens per sec: 177242.68384443634, Loss: 2.677621364593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1468 , TFLOPS: 70.21031609440992, Tokens per sec: 173988.89758674972, Loss: 2.68800950050354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1469 , TFLOPS: 69.5983405545009, Tokens per sec: 172472.3547841842, Loss: 2.6636581420898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1470 , TFLOPS: 71.50866929960019, Tokens per sec: 177206.35985462382, Loss: 2.6665565967559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1471 , TFLOPS: 71.49595005972006, Tokens per sec: 177174.84017706092, Loss: 2.663930654525757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1472 , TFLOPS: 69.26241399207798, Tokens per sec: 171639.8917571345, Loss: 2.679600715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1473 , TFLOPS: 71.49497231713148, Tokens per sec: 177172.41722321918, Loss: 2.668288230895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1474 , TFLOPS: 71.50668678912295, Tokens per sec: 177201.44697526965, Loss: 2.6751158237457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1475 , TFLOPS: 70.54272909970861, Tokens per sec: 174812.65363219535, Loss: 2.6738040447235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1476 , TFLOPS: 69.79963354238737, Tokens per sec: 172971.18098242002, Loss: 2.647791862487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1477 , TFLOPS: 71.52319184144253, Tokens per sec: 177242.34831309918, Loss: 2.671642303466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1478 , TFLOPS: 71.48746684429871, Tokens per sec: 177153.8178347435, Loss: 2.695143461227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1479 , TFLOPS: 68.51550826064587, Tokens per sec: 169788.97707618715, Loss: 2.663872718811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1480 , TFLOPS: 71.47860368253468, Tokens per sec: 177131.85394353437, Loss: 2.6606390476226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1481 , TFLOPS: 71.48859248656264, Tokens per sec: 177156.60730027323, Loss: 2.651026725769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1482 , TFLOPS: 69.85502704715267, Tokens per sec: 173108.45218932672, Loss: 2.6655054092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1483 , TFLOPS: 70.34713333946324, Tokens per sec: 174327.9457916556, Loss: 2.67900013923645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1484 , TFLOPS: 71.49948623149687, Tokens per sec: 177183.60320026535, Loss: 2.6897881031036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1485 , TFLOPS: 71.53610503522006, Tokens per sec: 177274.34862978564, Loss: 2.683755397796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1486 , TFLOPS: 68.46715107154063, Tokens per sec: 169669.1426346002, Loss: 2.6634299755096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1487 , TFLOPS: 71.465395278553, Tokens per sec: 177099.12206344196, Loss: 2.6671836376190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1488 , TFLOPS: 71.46888484039978, Tokens per sec: 177107.76958210525, Loss: 2.6785781383514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1489 , TFLOPS: 69.62666185048451, Tokens per sec: 172542.53807547988, Loss: 2.6725234985351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1490 , TFLOPS: 70.46477332854899, Tokens per sec: 174619.47064372466, Loss: 2.6751279830932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1491 , TFLOPS: 71.5182468536182, Tokens per sec: 177230.09408853512, Loss: 2.651212215423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1492 , TFLOPS: 71.49712750892049, Tokens per sec: 177177.75802588632, Loss: 2.653498888015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1493 , TFLOPS: 70.58767955929288, Tokens per sec: 174924.04582274702, Loss: 2.6902577877044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1494 , TFLOPS: 70.48778617032187, Tokens per sec: 174676.4990007117, Loss: 2.667789936065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1495 , TFLOPS: 71.50876139233408, Tokens per sec: 177206.58807056324, Loss: 2.670522689819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1496 , TFLOPS: 69.57626086283418, Tokens per sec: 172417.63887595525, Loss: 2.6580631732940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1497 , TFLOPS: 70.57690782720972, Tokens per sec: 174897.35228404074, Loss: 2.6727378368377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1498 , TFLOPS: 71.5183094953556, Tokens per sec: 177230.24932166093, Loss: 2.6528773307800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1499 , TFLOPS: 69.53275032389378, Tokens per sec: 172309.8149673794, Loss: 2.671260118484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1500 , TFLOPS: 71.50118322804387, Tokens per sec: 177187.8085446484, Loss: 2.6543872356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1501 , TFLOPS: 70.45362973627593, Tokens per sec: 174591.855594504, Loss: 2.680521011352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1502 , TFLOPS: 70.65323086897507, Tokens per sec: 175086.48918921239, Loss: 2.6704258918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1503 , TFLOPS: 70.36042375096908, Tokens per sec: 174360.88089542673, Loss: 2.6689627170562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1504 , TFLOPS: 71.48553673307951, Tokens per sec: 177149.03480652304, Loss: 2.6729190349578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1505 , TFLOPS: 70.46851133471996, Tokens per sec: 174628.73383479172, Loss: 2.6699330806732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1506 , TFLOPS: 69.463459067803, Tokens per sec: 172138.1036017245, Loss: 2.6815311908721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1507 , TFLOPS: 71.51136585601806, Tokens per sec: 177213.04221847752, Loss: 2.6583755016326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1508 , TFLOPS: 70.50050851341601, Tokens per sec: 174708.02636838073, Loss: 2.6456809043884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1509 , TFLOPS: 69.4569829756179, Tokens per sec: 172122.05513188936, Loss: 2.6537227630615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1510 , TFLOPS: 71.52758193563999, Tokens per sec: 177253.22745012928, Loss: 2.666405439376831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1511 , TFLOPS: 71.50193865270077, Tokens per sec: 177189.6805701651, Loss: 2.657752275466919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1512 , TFLOPS: 70.50947292144768, Tokens per sec: 174730.2411589943, Loss: 2.6778035163879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1513 , TFLOPS: 70.36116376301308, Tokens per sec: 174362.71472679084, Loss: 2.6677088737487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1514 , TFLOPS: 71.4246088598627, Tokens per sec: 176998.04882493243, Loss: 2.676060199737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1515 , TFLOPS: 71.5219882385646, Tokens per sec: 177239.36565257987, Loss: 2.668393850326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1516 , TFLOPS: 68.66166229366009, Tokens per sec: 170151.1628701922, Loss: 2.67696213722229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1517 , TFLOPS: 71.52610173268421, Tokens per sec: 177249.55934414704, Loss: 2.6578943729400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1518 , TFLOPS: 71.49576265779876, Tokens per sec: 177174.3757744559, Loss: 2.6893646717071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1519 , TFLOPS: 69.34905477280758, Tokens per sec: 171854.59715605233, Loss: 2.6609737873077393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1520 , TFLOPS: 71.49301334017267, Tokens per sec: 177167.5626625161, Loss: 2.681272029876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1521 , TFLOPS: 71.5431229711586, Tokens per sec: 177291.73984813082, Loss: 2.6749932765960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1522 , TFLOPS: 70.65373687206574, Tokens per sec: 175087.74312061188, Loss: 2.667140245437622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1523 , TFLOPS: 69.42908567355329, Tokens per sec: 172052.9225442317, Loss: 2.681838274002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1524 , TFLOPS: 71.48638456687812, Tokens per sec: 177151.13583207256, Loss: 2.673466205596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1525 , TFLOPS: 71.53768192762183, Tokens per sec: 177278.25634286625, Loss: 2.6672136783599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1526 , TFLOPS: 69.60983257622276, Tokens per sec: 172500.83327996166, Loss: 2.660902738571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1527 , TFLOPS: 70.47676517791685, Tokens per sec: 174649.1877674695, Loss: 2.6608996391296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1528 , TFLOPS: 71.51913779887423, Tokens per sec: 177232.30194902528, Loss: 2.6605541706085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1529 , TFLOPS: 70.58113156838657, Tokens per sec: 174907.81917996047, Loss: 2.655738353729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1530 , TFLOPS: 70.61287442984704, Tokens per sec: 174986.48148742437, Loss: 2.669813394546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1531 , TFLOPS: 69.92409342411416, Tokens per sec: 173279.60627972684, Loss: 2.681907892227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1532 , TFLOPS: 71.52071777899519, Tokens per sec: 177236.2173138142, Loss: 2.646475315093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1533 , TFLOPS: 70.4301781062172, Tokens per sec: 174533.73987180815, Loss: 2.647613286972046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1534 , TFLOPS: 70.47356242265921, Tokens per sec: 174641.25098712844, Loss: 2.658942937850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1535 , TFLOPS: 71.47370920455573, Tokens per sec: 177119.72488793125, Loss: 2.657761812210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1536 , TFLOPS: 69.44113924991832, Tokens per sec: 172082.7926342753, Loss: 2.6850087642669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1537 , TFLOPS: 71.50751167490073, Tokens per sec: 177203.49113309567, Loss: 2.653041362762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1538 , TFLOPS: 70.59130335499225, Tokens per sec: 174933.02598768414, Loss: 2.668362617492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1539 , TFLOPS: 69.6761514299368, Tokens per sec: 172665.1787050899, Loss: 2.6447644233703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1540 , TFLOPS: 71.5100265108717, Tokens per sec: 177209.7231736638, Loss: 2.668487787246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1541 , TFLOPS: 71.49463401884033, Tokens per sec: 177171.57888278706, Loss: 2.6820173263549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1542 , TFLOPS: 70.38700550552157, Tokens per sec: 174426.75341142982, Loss: 2.663908004760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1543 , TFLOPS: 69.7625109817434, Tokens per sec: 172879.18718775664, Loss: 2.6658856868743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1544 , TFLOPS: 71.48753781064381, Tokens per sec: 177153.9936971643, Loss: 2.6583240032196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1545 , TFLOPS: 71.53256758903132, Tokens per sec: 177265.58244844805, Loss: 2.6845505237579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1546 , TFLOPS: 68.70881566005563, Tokens per sec: 170268.01410649248, Loss: 2.6815195083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1547 , TFLOPS: 71.46047922884287, Tokens per sec: 177086.93955071268, Loss: 2.6526525020599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1548 , TFLOPS: 71.47610081187595, Tokens per sec: 177125.65155432804, Loss: 2.6606693267822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1549 , TFLOPS: 69.561676873868, Tokens per sec: 172381.49814473803, Loss: 2.670191526412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1550 , TFLOPS: 70.37609655746412, Tokens per sec: 174399.7198364236, Loss: 2.6479969024658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1551 , TFLOPS: 71.53609486681252, Tokens per sec: 177274.3234313516, Loss: 2.65206241607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1552 , TFLOPS: 71.49788909494818, Tokens per sec: 177179.64531995857, Loss: 2.6566953659057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1553 , TFLOPS: 70.33207074136247, Tokens per sec: 174290.6190142797, Loss: 2.6742451190948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1554 , TFLOPS: 71.49833173317613, Tokens per sec: 177180.74222624805, Loss: 2.6554460525512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1555 , TFLOPS: 71.4799190754225, Tokens per sec: 177135.1136320129, Loss: 2.6646475791931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1556 , TFLOPS: 69.41262186433093, Tokens per sec: 172012.1234113404, Loss: 2.664403200149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1557 , TFLOPS: 71.48833125927302, Tokens per sec: 177155.959950273, Loss: 2.674525022506714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1558 , TFLOPS: 71.44372052045746, Tokens per sec: 177045.4096252095, Loss: 2.6789684295654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1559 , TFLOPS: 70.33167765831027, Tokens per sec: 174289.6449111747, Loss: 2.647233247756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1560 , TFLOPS: 69.58116988474686, Tokens per sec: 172429.80397302826, Loss: 2.6626715660095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1561 , TFLOPS: 71.47294935893157, Tokens per sec: 177117.8419067707, Loss: 2.6566386222839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1562 , TFLOPS: 71.54360579800336, Tokens per sec: 177292.9363462399, Loss: 2.6765246391296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1563 , TFLOPS: 69.44653874171165, Tokens per sec: 172096.1731697423, Loss: 2.6787450313568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1564 , TFLOPS: 70.27309749970735, Tokens per sec: 174144.47682501946, Loss: 2.6716420650482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1565 , TFLOPS: 71.47321275094646, Tokens per sec: 177118.4946211986, Loss: 2.654480457305908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1566 , TFLOPS: 69.05241772460525, Tokens per sec: 171119.49787333852, Loss: 2.6658709049224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1567 , TFLOPS: 71.50401239132194, Tokens per sec: 177194.81952290953, Loss: 2.6588006019592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1568 , TFLOPS: 69.73770341650155, Tokens per sec: 172817.7113083084, Loss: 2.6289970874786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1569 , TFLOPS: 70.5866660262303, Tokens per sec: 174921.53417616806, Loss: 2.6493825912475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1570 , TFLOPS: 70.34394941160326, Tokens per sec: 174320.05566767996, Loss: 2.65136456489563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1571 , TFLOPS: 70.53756985670348, Tokens per sec: 174799.8684596911, Loss: 2.6858723163604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1572 , TFLOPS: 71.48844556892085, Tokens per sec: 177156.24322217258, Loss: 2.673063039779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1573 , TFLOPS: 70.35228429976922, Tokens per sec: 174340.71043871893, Loss: 2.66812801361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1574 , TFLOPS: 71.50889545121258, Tokens per sec: 177206.920283232, Loss: 2.670358896255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1575 , TFLOPS: 70.47112105453223, Tokens per sec: 174635.2010079709, Loss: 2.6927590370178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1576 , TFLOPS: 69.42548316104968, Tokens per sec: 172043.99512139827, Loss: 2.675497531890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1577 , TFLOPS: 71.48522252869083, Tokens per sec: 177148.2561734354, Loss: 2.6951727867126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1578 , TFLOPS: 71.53590532123656, Tokens per sec: 177273.85371653418, Loss: 2.6506996154785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1579 , TFLOPS: 70.49567736796756, Tokens per sec: 174696.0542577649, Loss: 2.6617977619171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1580 , TFLOPS: 69.84020710572184, Tokens per sec: 173071.7267419098, Loss: 2.6698074340820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1581 , TFLOPS: 71.49543494332521, Tokens per sec: 177173.56366188973, Loss: 2.6589317321777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1582 , TFLOPS: 70.53185635201798, Tokens per sec: 174785.70976568593, Loss: 2.6722960472106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1583 , TFLOPS: 69.44533188147504, Tokens per sec: 172093.18243712935, Loss: 2.667128562927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1584 , TFLOPS: 71.47799561535467, Tokens per sec: 177130.34708607817, Loss: 2.6635634899139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1585 , TFLOPS: 71.47171320425133, Tokens per sec: 177114.77857929937, Loss: 2.6746344566345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1586 , TFLOPS: 68.55489398954825, Tokens per sec: 169886.57925110342, Loss: 2.691114902496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1587 , TFLOPS: 71.47456323239165, Tokens per sec: 177121.8412629876, Loss: 2.662973165512085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1588 , TFLOPS: 71.46948934691909, Tokens per sec: 177109.26761585195, Loss: 2.6590871810913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1589 , TFLOPS: 71.48438140149466, Tokens per sec: 177146.17176758545, Loss: 2.6713109016418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1590 , TFLOPS: 68.74658091769913, Tokens per sec: 170361.60057511964, Loss: 2.6745824813842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1591 , TFLOPS: 71.48142230573626, Tokens per sec: 177138.83880232484, Loss: 2.6545324325561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1592 , TFLOPS: 71.51648157809704, Tokens per sec: 177225.71954133254, Loss: 2.654498815536499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1593 , TFLOPS: 69.33132093333504, Tokens per sec: 171810.65074829568, Loss: 2.667048215866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1594 , TFLOPS: 71.51340509627695, Tokens per sec: 177218.09568047992, Loss: 2.678532600402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1595 , TFLOPS: 71.54763735987616, Tokens per sec: 177302.9270006739, Loss: 2.6583337783813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1596 , TFLOPS: 70.60644514414213, Tokens per sec: 174970.5490092033, Loss: 2.6445155143737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1597 , TFLOPS: 69.68543635464191, Tokens per sec: 172688.18777132803, Loss: 2.671447515487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1598 , TFLOPS: 71.37526742556503, Tokens per sec: 176875.7753153351, Loss: 2.668846607208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1599 , TFLOPS: 71.43793133707794, Tokens per sec: 177031.06339106386, Loss: 2.648801565170288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1600 , TFLOPS: 69.55387430975672, Tokens per sec: 172362.1625313469, Loss: 2.661712884902954 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/optimizer.pt +[rank0]:[2024-08-26 05:26:55,720] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006332795001071645, 'preprocessing_with_comm': 0.001282651999645168, 'state_converting': 0.9721380530008901, : 0.981109447999188}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600, took 4.53s + eval ppl=11.910943031311035, eval loss=2.4774575233459473 +------------------------------------------------------------------ +iteration: 1601 , TFLOPS: 65.12220962202764, Tokens per sec: 161380.00924698787, Loss: 2.657534122467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1602 , TFLOPS: 71.52006051384386, Tokens per sec: 177234.58853836547, Loss: 2.6662495136260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1603 , TFLOPS: 69.50374597517128, Tokens per sec: 172237.93899615225, Loss: 2.6646480560302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1604 , TFLOPS: 71.51977060374428, Tokens per sec: 177233.8701092025, Loss: 2.660512685775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1605 , TFLOPS: 70.3904800249414, Tokens per sec: 174435.36365330208, Loss: 2.6601006984710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1606 , TFLOPS: 69.43014839113305, Tokens per sec: 172055.55607546237, Loss: 2.685183048248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1607 , TFLOPS: 71.48320699730434, Tokens per sec: 177143.26146463054, Loss: 2.6558382511138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1608 , TFLOPS: 70.39407943590705, Tokens per sec: 174444.28339018274, Loss: 2.6512999534606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1609 , TFLOPS: 70.6897724987481, Tokens per sec: 175177.04337318277, Loss: 2.6638078689575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1610 , TFLOPS: 70.2994257324457, Tokens per sec: 174209.72108603935, Loss: 2.6610491275787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1611 , TFLOPS: 71.4711186942753, Tokens per sec: 177113.3053180881, Loss: 2.6640336513519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1612 , TFLOPS: 70.33163025170062, Tokens per sec: 174289.52743237372, Loss: 2.660778284072876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1613 , TFLOPS: 70.25026829430219, Tokens per sec: 174087.90353918087, Loss: 2.661963939666748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1614 , TFLOPS: 71.47859567282772, Tokens per sec: 177131.8340945982, Loss: 2.6681840419769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1615 , TFLOPS: 70.38570380012733, Tokens per sec: 174423.52764206822, Loss: 2.64536190032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1616 , TFLOPS: 69.6077703238471, Tokens per sec: 172495.72279139896, Loss: 2.642444610595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1617 , TFLOPS: 71.46423346924952, Tokens per sec: 177096.24297200405, Loss: 2.6477699279785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1618 , TFLOPS: 71.51043072192917, Tokens per sec: 177210.7248531917, Loss: 2.659660577774048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1619 , TFLOPS: 69.31177083346877, Tokens per sec: 171762.20344720577, Loss: 2.6634392738342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1620 , TFLOPS: 70.86600847230979, Tokens per sec: 175613.77552966378, Loss: 2.658169746398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1621 , TFLOPS: 71.49907008408002, Tokens per sec: 177182.57194112396, Loss: 2.675381660461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1622 , TFLOPS: 71.48173257766202, Tokens per sec: 177139.60769033615, Loss: 2.6590957641601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1623 , TFLOPS: 68.56861280927845, Tokens per sec: 169920.57599764763, Loss: 2.6772994995117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1624 , TFLOPS: 71.38700781653435, Tokens per sec: 176904.86929746796, Loss: 2.6737029552459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1625 , TFLOPS: 71.51012799544958, Tokens per sec: 177209.97466362722, Loss: 2.641874313354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1626 , TFLOPS: 69.47174063671359, Tokens per sec: 172158.62624177505, Loss: 2.656559944152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1627 , TFLOPS: 70.39841622915209, Tokens per sec: 174455.0304415812, Loss: 2.647475004196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1628 , TFLOPS: 71.4881679811041, Tokens per sec: 177155.55532898384, Loss: 2.6764721870422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1629 , TFLOPS: 70.66862323915404, Tokens per sec: 175124.63317812546, Loss: 2.671705961227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1630 , TFLOPS: 69.24525123279363, Tokens per sec: 171597.3605489939, Loss: 2.6649293899536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1631 , TFLOPS: 71.5057206799744, Tokens per sec: 177199.05285037388, Loss: 2.6793224811553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1632 , TFLOPS: 71.487561614156, Tokens per sec: 177154.05268488958, Loss: 2.6564440727233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1633 , TFLOPS: 70.40422517288685, Tokens per sec: 174469.42564406394, Loss: 2.658940076828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1634 , TFLOPS: 70.40095986327916, Tokens per sec: 174461.3338471525, Loss: 2.6566364765167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1635 , TFLOPS: 71.4679514537275, Tokens per sec: 177105.45654711028, Loss: 2.666471481323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1636 , TFLOPS: 69.53327818452706, Tokens per sec: 172311.1230641773, Loss: 2.656761646270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1637 , TFLOPS: 71.5297645490733, Tokens per sec: 177258.63620665178, Loss: 2.6818535327911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1638 , TFLOPS: 70.4561612912629, Tokens per sec: 174598.12906663553, Loss: 2.662243604660034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1639 , TFLOPS: 71.5346225850763, Tokens per sec: 177270.67495502403, Loss: 2.687520980834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1640 , TFLOPS: 69.73147240079328, Tokens per sec: 172802.2701649803, Loss: 2.6516988277435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1641 , TFLOPS: 71.52885636932955, Tokens per sec: 177256.3856372025, Loss: 2.6777966022491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1642 , TFLOPS: 70.38268249679204, Tokens per sec: 174416.04051957928, Loss: 2.6617627143859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1643 , TFLOPS: 69.56397287936994, Tokens per sec: 172387.18789929774, Loss: 2.6646361351013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1644 , TFLOPS: 71.5067513228736, Tokens per sec: 177201.60689701274, Loss: 2.646994113922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1645 , TFLOPS: 70.44925714309835, Tokens per sec: 174581.01982692906, Loss: 2.6699342727661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1646 , TFLOPS: 69.12767631832772, Tokens per sec: 171305.9969590013, Loss: 2.6705665588378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1647 , TFLOPS: 71.47090626003059, Tokens per sec: 177112.77888262813, Loss: 2.6643407344818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1648 , TFLOPS: 71.47305616088833, Tokens per sec: 177118.10657378475, Loss: 2.680062770843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1649 , TFLOPS: 70.45975015009323, Tokens per sec: 174607.0226541625, Loss: 2.6755120754241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1650 , TFLOPS: 69.8570723609565, Tokens per sec: 173113.52070224253, Loss: 2.6601338386535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1651 , TFLOPS: 71.51777167255875, Tokens per sec: 177228.9165375243, Loss: 2.6521077156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1652 , TFLOPS: 70.57620662275795, Tokens per sec: 174895.61462216522, Loss: 2.6576497554779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1653 , TFLOPS: 70.29561846788201, Tokens per sec: 174200.28626504572, Loss: 2.660304069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1654 , TFLOPS: 71.48195449579136, Tokens per sec: 177140.15762790697, Loss: 2.658238410949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1655 , TFLOPS: 71.45998996146588, Tokens per sec: 177085.72709225523, Loss: 2.6438283920288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1656 , TFLOPS: 69.57042088927538, Tokens per sec: 172403.16677815036, Loss: 2.652616262435913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1657 , TFLOPS: 70.65391762288404, Tokens per sec: 175088.191041051, Loss: 2.6610162258148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1658 , TFLOPS: 71.49768551002784, Tokens per sec: 177179.14081409923, Loss: 2.658557415008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1659 , TFLOPS: 71.49499189682678, Tokens per sec: 177172.465743861, Loss: 2.656223773956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1660 , TFLOPS: 68.49568909147966, Tokens per sec: 169739.86299173156, Loss: 2.660006046295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1661 , TFLOPS: 71.48997308846512, Tokens per sec: 177160.02858387347, Loss: 2.6583030223846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1662 , TFLOPS: 71.46758255906492, Tokens per sec: 177104.54238549934, Loss: 2.6404600143432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1663 , TFLOPS: 69.5917164472102, Tokens per sec: 172455.9395166122, Loss: 2.654402494430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1664 , TFLOPS: 70.54335837292095, Tokens per sec: 174814.21304053566, Loss: 2.6538586616516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1665 , TFLOPS: 71.48237195925115, Tokens per sec: 177141.1921483477, Loss: 2.6520206928253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1666 , TFLOPS: 69.47560377334229, Tokens per sec: 172168.19951990055, Loss: 2.6381542682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1667 , TFLOPS: 70.44399628527103, Tokens per sec: 174567.98284170163, Loss: 2.673353672027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1668 , TFLOPS: 71.47204287218389, Tokens per sec: 177115.5955327522, Loss: 2.6537961959838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1669 , TFLOPS: 71.4938945238047, Tokens per sec: 177169.74633262667, Loss: 2.642825126647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1670 , TFLOPS: 69.53272291881228, Tokens per sec: 172309.74705456852, Loss: 2.650801181793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1671 , TFLOPS: 70.48349319613604, Tokens per sec: 174665.860537769, Loss: 2.6451144218444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1672 , TFLOPS: 71.50911585746118, Tokens per sec: 177207.46647419577, Loss: 2.6650092601776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1673 , TFLOPS: 70.3222873799551, Tokens per sec: 174266.37476698603, Loss: 2.674197196960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1674 , TFLOPS: 71.56234357897324, Tokens per sec: 177339.37063162026, Loss: 2.6537604331970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1675 , TFLOPS: 70.42183884022874, Tokens per sec: 174513.07425204277, Loss: 2.6359925270080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1676 , TFLOPS: 70.67433376279952, Tokens per sec: 175138.78448478505, Loss: 2.6476633548736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1677 , TFLOPS: 70.34394095320924, Tokens per sec: 174320.0347068478, Loss: 2.6635897159576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1678 , TFLOPS: 71.49472775535546, Tokens per sec: 177171.8111721975, Loss: 2.6586122512817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1679 , TFLOPS: 70.46794710954877, Tokens per sec: 174627.33562266297, Loss: 2.6460211277008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1680 , TFLOPS: 69.50111985072172, Tokens per sec: 172231.43117047477, Loss: 2.6322367191314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1681 , TFLOPS: 71.52537237031949, Tokens per sec: 177247.75190386138, Loss: 2.6507115364074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1682 , TFLOPS: 70.42314947946987, Tokens per sec: 174516.32216046113, Loss: 2.641571044921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1683 , TFLOPS: 68.9707820326586, Tokens per sec: 170917.1956357809, Loss: 2.646617889404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1684 , TFLOPS: 71.53143698561006, Tokens per sec: 177262.78068862393, Loss: 2.6568374633789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1685 , TFLOPS: 71.46500016192226, Tokens per sec: 177098.14292090526, Loss: 2.658465623855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1686 , TFLOPS: 69.41164746371264, Tokens per sec: 172009.70873926918, Loss: 2.6451363563537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1687 , TFLOPS: 70.86766990610818, Tokens per sec: 175617.89274563827, Loss: 2.6461760997772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1688 , TFLOPS: 71.52129174252734, Tokens per sec: 177237.6396586742, Loss: 2.6513118743896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1689 , TFLOPS: 70.51962396898678, Tokens per sec: 174755.39657302637, Loss: 2.6349048614501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1690 , TFLOPS: 69.54805528749104, Tokens per sec: 172347.74235315432, Loss: 2.64986252784729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1691 , TFLOPS: 71.49160862159923, Tokens per sec: 177164.08161514858, Loss: 2.6650969982147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1692 , TFLOPS: 71.54051069328601, Tokens per sec: 177285.26633579633, Loss: 2.661541700363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1693 , TFLOPS: 69.28234567909728, Tokens per sec: 171689.2846732249, Loss: 2.6419849395751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1694 , TFLOPS: 71.49834415992085, Tokens per sec: 177180.77302109037, Loss: 2.676173210144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1695 , TFLOPS: 71.50523908862283, Tokens per sec: 177197.859413953, Loss: 2.6730399131774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1696 , TFLOPS: 70.56350751617437, Tokens per sec: 174864.144837129, Loss: 2.636188268661499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1697 , TFLOPS: 69.35892455194016, Tokens per sec: 171879.0555559888, Loss: 2.650848150253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1698 , TFLOPS: 71.50973375553755, Tokens per sec: 177208.9976936961, Loss: 2.65798020362854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1699 , TFLOPS: 71.45187337912351, Tokens per sec: 177065.6133070955, Loss: 2.6505558490753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1700 , TFLOPS: 69.60726156572676, Tokens per sec: 172494.46203273244, Loss: 2.6468939781188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1701 , TFLOPS: 70.36860176772669, Tokens per sec: 174381.146921835, Loss: 2.6653616428375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1702 , TFLOPS: 71.5027385530549, Tokens per sec: 177191.66281135892, Loss: 2.6497397422790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1703 , TFLOPS: 69.02909221130263, Tokens per sec: 171061.69468185722, Loss: 2.654231548309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1704 , TFLOPS: 70.52712556077053, Tokens per sec: 174773.98634383504, Loss: 2.6628286838531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1705 , TFLOPS: 71.50099755612537, Tokens per sec: 177187.3484291809, Loss: 2.629479169845581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1706 , TFLOPS: 71.51249914788491, Tokens per sec: 177215.850640566, Loss: 2.6417911052703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1707 , TFLOPS: 69.93550414147927, Tokens per sec: 173307.88329435186, Loss: 2.6651811599731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1708 , TFLOPS: 70.43361373036493, Tokens per sec: 174542.25372691147, Loss: 2.6362316608428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1709 , TFLOPS: 71.50471463497396, Tokens per sec: 177196.55976003775, Loss: 2.646761178970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1710 , TFLOPS: 69.42140744749074, Tokens per sec: 172033.8950542223, Loss: 2.654353618621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1711 , TFLOPS: 71.52075653181303, Tokens per sec: 177236.3133475658, Loss: 2.6210837364196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1712 , TFLOPS: 70.60179299324153, Tokens per sec: 174959.0204667949, Loss: 2.660778045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1713 , TFLOPS: 70.38534551606143, Tokens per sec: 174422.63977468465, Loss: 2.65342378616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1714 , TFLOPS: 71.52273375470403, Tokens per sec: 177241.21312369878, Loss: 2.6458098888397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1715 , TFLOPS: 71.47927925518691, Tokens per sec: 177133.52808698706, Loss: 2.6776936054229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1716 , TFLOPS: 69.55300248946035, Tokens per sec: 172360.0020646136, Loss: 2.642659902572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1717 , TFLOPS: 70.40620446379134, Tokens per sec: 174474.33054496217, Loss: 2.6406819820404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1718 , TFLOPS: 71.50257968015275, Tokens per sec: 177191.26910680614, Loss: 2.645332098007202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1719 , TFLOPS: 70.50159193830599, Tokens per sec: 174710.71121460723, Loss: 2.6676595211029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1720 , TFLOPS: 69.34467120498529, Tokens per sec: 171843.73419209884, Loss: 2.649880886077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1721 , TFLOPS: 71.52508702632441, Tokens per sec: 177247.04479000834, Loss: 2.6681036949157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1722 , TFLOPS: 71.5116512644165, Tokens per sec: 177213.74949192916, Loss: 2.643944025039673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1723 , TFLOPS: 69.65031367806084, Tokens per sec: 172601.14990967794, Loss: 2.6510958671569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1724 , TFLOPS: 70.61377470878078, Tokens per sec: 174988.7124777961, Loss: 2.6535580158233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1725 , TFLOPS: 71.46864745151743, Tokens per sec: 177107.1813063042, Loss: 2.6282949447631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1726 , TFLOPS: 70.4630228381504, Tokens per sec: 174615.13273568454, Loss: 2.659029006958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1727 , TFLOPS: 69.63272392479352, Tokens per sec: 172557.56056341025, Loss: 2.644351005554199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1728 , TFLOPS: 71.46612419961515, Tokens per sec: 177100.92841013125, Loss: 2.6726205348968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1729 , TFLOPS: 71.54307558019055, Tokens per sec: 177291.62240809144, Loss: 2.6299781799316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1730 , TFLOPS: 68.51812314774374, Tokens per sec: 169795.45705446924, Loss: 2.6448974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1731 , TFLOPS: 71.52139323975001, Tokens per sec: 177237.8911799728, Loss: 2.6471972465515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1732 , TFLOPS: 71.52467387581524, Tokens per sec: 177246.02095753435, Loss: 2.6326842308044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1733 , TFLOPS: 70.3472319334351, Tokens per sec: 174328.19011837887, Loss: 2.669045925140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1734 , TFLOPS: 70.46594492228857, Tokens per sec: 174622.37398207304, Loss: 2.6549699306488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1735 , TFLOPS: 71.5262671458055, Tokens per sec: 177249.96925608336, Loss: 2.614151954650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1736 , TFLOPS: 71.53428946898578, Tokens per sec: 177269.84945665594, Loss: 2.6458075046539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1737 , TFLOPS: 69.76825073779976, Tokens per sec: 172893.4109355506, Loss: 2.6385741233825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1738 , TFLOPS: 70.55038658256538, Tokens per sec: 174831.6297182558, Loss: 2.6214065551757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1739 , TFLOPS: 71.47572372753172, Tokens per sec: 177124.71709778375, Loss: 2.650651454925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1740 , TFLOPS: 69.60260364313532, Tokens per sec: 172482.9191874383, Loss: 2.655247211456299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1741 , TFLOPS: 70.4304606180318, Tokens per sec: 174534.43996720598, Loss: 2.656141519546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1742 , TFLOPS: 71.47430162516225, Tokens per sec: 177121.19297145473, Loss: 2.663872003555298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1743 , TFLOPS: 70.60176815659703, Tokens per sec: 174958.9589188538, Loss: 2.647508382797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1744 , TFLOPS: 70.30376520999752, Tokens per sec: 174220.4747894457, Loss: 2.632500410079956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1745 , TFLOPS: 70.46146859412717, Tokens per sec: 174611.28114778086, Loss: 2.645984649658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1746 , TFLOPS: 71.51473307257298, Tokens per sec: 177221.3865520301, Loss: 2.6274638175964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1747 , TFLOPS: 69.42356356941099, Tokens per sec: 172039.2381618571, Loss: 2.661741018295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1748 , TFLOPS: 71.51824102803512, Tokens per sec: 177230.0796520985, Loss: 2.6344053745269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1749 , TFLOPS: 70.54205487271959, Tokens per sec: 174810.9828234443, Loss: 2.6457791328430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1750 , TFLOPS: 69.42305590738584, Tokens per sec: 172037.9801194352, Loss: 2.651749849319458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1751 , TFLOPS: 71.54776473827752, Tokens per sec: 177303.24265838432, Loss: 2.656318426132202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1752 , TFLOPS: 71.4684704769589, Tokens per sec: 177106.7427438528, Loss: 2.636157512664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1753 , TFLOPS: 70.41734181454716, Tokens per sec: 174501.93012701554, Loss: 2.646155595779419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1754 , TFLOPS: 70.57551231886855, Tokens per sec: 174893.89406064336, Loss: 2.667630434036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1755 , TFLOPS: 71.50508644329004, Tokens per sec: 177197.48114200306, Loss: 2.6581833362579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1756 , TFLOPS: 70.50469614944984, Tokens per sec: 174718.40379177878, Loss: 2.644163131713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1757 , TFLOPS: 69.49992816254964, Tokens per sec: 172228.47803590904, Loss: 2.635085105895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1758 , TFLOPS: 71.48198628673961, Tokens per sec: 177140.2364093788, Loss: 2.6648807525634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1759 , TFLOPS: 71.52949089245203, Tokens per sec: 177257.95805539849, Loss: 2.668731689453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1760 , TFLOPS: 68.5465873441307, Tokens per sec: 169865.9944687062, Loss: 2.6404919624328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1761 , TFLOPS: 71.53761527912988, Tokens per sec: 177278.09118056126, Loss: 2.6516342163085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1762 , TFLOPS: 71.52358770809742, Tokens per sec: 177243.32931427832, Loss: 2.6427040100097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1763 , TFLOPS: 69.53650160214333, Tokens per sec: 172319.1110481192, Loss: 2.661947250366211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1764 , TFLOPS: 70.40086959076913, Tokens per sec: 174461.11014192834, Loss: 2.65114426612854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1765 , TFLOPS: 71.54445905105523, Tokens per sec: 177295.050801296, Loss: 2.6457784175872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1766 , TFLOPS: 71.54236567061582, Tokens per sec: 177289.86317396228, Loss: 2.649163246154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1767 , TFLOPS: 68.65730504978062, Tokens per sec: 170140.36513986884, Loss: 2.6423850059509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1768 , TFLOPS: 71.55099265682068, Tokens per sec: 177311.2417961189, Loss: 2.6363909244537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1769 , TFLOPS: 71.49663076855387, Tokens per sec: 177176.5270485374, Loss: 2.6649575233459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1770 , TFLOPS: 69.58799518263004, Tokens per sec: 172446.71781305145, Loss: 2.6605241298675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1771 , TFLOPS: 70.60220017355962, Tokens per sec: 174960.02950447774, Loss: 2.6479506492614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1772 , TFLOPS: 71.50306507909953, Tokens per sec: 177192.47197886734, Loss: 2.6385955810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1773 , TFLOPS: 71.50302400847364, Tokens per sec: 177192.3702013321, Loss: 2.63383150100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1774 , TFLOPS: 70.41571475549559, Tokens per sec: 174497.8980954504, Loss: 2.6419684886932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1775 , TFLOPS: 70.41836823960088, Tokens per sec: 174504.4737213687, Loss: 2.642275810241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1776 , TFLOPS: 71.48259826305429, Tokens per sec: 177141.75295410134, Loss: 2.6498613357543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1777 , TFLOPS: 69.0117931294725, Tokens per sec: 171018.82565143096, Loss: 2.631375551223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1778 , TFLOPS: 70.54635233293175, Tokens per sec: 174821.63240325436, Loss: 2.6371212005615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1779 , TFLOPS: 71.52641107842534, Tokens per sec: 177250.32593697082, Loss: 2.642812490463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1780 , TFLOPS: 69.51233915257852, Tokens per sec: 172259.23383638408, Loss: 2.6478958129882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1781 , TFLOPS: 71.46996226572526, Tokens per sec: 177110.43956074995, Loss: 2.6531829833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1782 , TFLOPS: 70.46517707297495, Tokens per sec: 174620.47116688834, Loss: 2.6444287300109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1783 , TFLOPS: 70.71122845434797, Tokens per sec: 175230.21359472792, Loss: 2.649585723876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1784 , TFLOPS: 70.2366058409592, Tokens per sec: 174054.0464747535, Loss: 2.6568541526794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1785 , TFLOPS: 71.39463554408219, Tokens per sec: 176923.77164659507, Loss: 2.650643825531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1786 , TFLOPS: 70.47270141775105, Tokens per sec: 174639.11732211852, Loss: 2.6387386322021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1787 , TFLOPS: 69.41914230603165, Tokens per sec: 172028.2817841607, Loss: 2.6756579875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1788 , TFLOPS: 71.51984458099217, Tokens per sec: 177234.05343297202, Loss: 2.6296029090881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1789 , TFLOPS: 71.49932104844686, Tokens per sec: 177183.19385847103, Loss: 2.6514358520507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1790 , TFLOPS: 69.6533830639851, Tokens per sec: 172608.75618611905, Loss: 2.6475915908813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1791 , TFLOPS: 70.67641280661111, Tokens per sec: 175143.9365843757, Loss: 2.63900089263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1792 , TFLOPS: 71.52074337266545, Tokens per sec: 177236.28073774852, Loss: 2.653958797454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1793 , TFLOPS: 70.49517930516568, Tokens per sec: 174694.82000327567, Loss: 2.669363498687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1794 , TFLOPS: 70.34040173054595, Tokens per sec: 174311.2641232106, Loss: 2.6471638679504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1795 , TFLOPS: 71.45772957640644, Tokens per sec: 177080.12560907594, Loss: 2.642820119857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1796 , TFLOPS: 71.45279844801841, Tokens per sec: 177067.9057297218, Loss: 2.6386380195617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1797 , TFLOPS: 68.15606911076456, Tokens per sec: 168898.2472672902, Loss: 2.63339900970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1798 , TFLOPS: 71.47756624667632, Tokens per sec: 177129.28306319437, Loss: 2.642282485961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1799 , TFLOPS: 71.50752994796356, Tokens per sec: 177203.5364157583, Loss: 2.648350238800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1800 , TFLOPS: 69.37672302703916, Tokens per sec: 171923.162137948, Loss: 2.637484073638916 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/optimizer.pt +[rank0]:[2024-08-26 05:52:01,750] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00635486000101082, 'preprocessing_with_comm': 0.0012308469995332416, 'state_converting': 0.9738579919994663, : 0.9827968249992409}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800, took 4.47s + eval ppl=11.066548347473145, eval loss=2.4039268493652344 +------------------------------------------------------------------ +iteration: 1801 , TFLOPS: 67.03259014355494, Tokens per sec: 166114.14262511977, Loss: 2.623095989227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1802 , TFLOPS: 71.51060925660738, Tokens per sec: 177211.16728179157, Loss: 2.6593565940856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1803 , TFLOPS: 71.51904009522997, Tokens per sec: 177232.05982863187, Loss: 2.6186840534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1804 , TFLOPS: 69.52253957925095, Tokens per sec: 172284.5116173481, Loss: 2.6546542644500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1805 , TFLOPS: 71.5444887609847, Tokens per sec: 177295.12442577406, Loss: 2.640963315963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1806 , TFLOPS: 71.47490596277878, Tokens per sec: 177122.69058664178, Loss: 2.6300208568573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1807 , TFLOPS: 69.44237716936541, Tokens per sec: 172085.86033503327, Loss: 2.656400203704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1808 , TFLOPS: 70.31238726098357, Tokens per sec: 174241.84118158647, Loss: 2.636434316635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1809 , TFLOPS: 71.54036784459359, Tokens per sec: 177284.91234100045, Loss: 2.666706085205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1810 , TFLOPS: 69.58418907058376, Tokens per sec: 172437.2858481229, Loss: 2.640324592590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1811 , TFLOPS: 70.43746755749085, Tokens per sec: 174551.8039350636, Loss: 2.6474881172180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1812 , TFLOPS: 71.50618110809509, Tokens per sec: 177200.19384197725, Loss: 2.6445913314819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1813 , TFLOPS: 71.50832548234719, Tokens per sec: 177205.5078375963, Loss: 2.6407663822174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1814 , TFLOPS: 69.69723303019505, Tokens per sec: 172717.42123285984, Loss: 2.639143228530884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1815 , TFLOPS: 70.28680000209715, Tokens per sec: 174178.43313539666, Loss: 2.6341769695281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1816 , TFLOPS: 71.52262195383904, Tokens per sec: 177240.93606883945, Loss: 2.6417582035064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1817 , TFLOPS: 69.18132244330972, Tokens per sec: 171438.93796631522, Loss: 2.6262669563293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1818 , TFLOPS: 71.50219740112273, Tokens per sec: 177190.3217772582, Loss: 2.6359269618988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1819 , TFLOPS: 70.39594777215046, Tokens per sec: 174448.91333320818, Loss: 2.651897430419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1820 , TFLOPS: 69.14714573710617, Tokens per sec: 171354.24432346784, Loss: 2.6473195552825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1821 , TFLOPS: 71.49601490236658, Tokens per sec: 177175.00086428202, Loss: 2.618288040161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1822 , TFLOPS: 71.51992785267515, Tokens per sec: 177234.2597893756, Loss: 2.6304850578308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1823 , TFLOPS: 70.42247168186006, Tokens per sec: 174514.64250331846, Loss: 2.666708469390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1824 , TFLOPS: 69.55262292750996, Tokens per sec: 172359.0614682883, Loss: 2.615349769592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1825 , TFLOPS: 71.45955704748059, Tokens per sec: 177084.65428371026, Loss: 2.6493942737579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1826 , TFLOPS: 70.49703415338534, Tokens per sec: 174699.41652149239, Loss: 2.655930995941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1827 , TFLOPS: 69.46348995448392, Tokens per sec: 172138.18014232224, Loss: 2.6538591384887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1828 , TFLOPS: 71.50942158885162, Tokens per sec: 177208.22411026037, Loss: 2.6325743198394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1829 , TFLOPS: 71.41354123710265, Tokens per sec: 176970.62203653375, Loss: 2.6386373043060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1830 , TFLOPS: 68.54311969245498, Tokens per sec: 169857.40124586044, Loss: 2.656658411026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1831 , TFLOPS: 71.49589139634806, Tokens per sec: 177174.6948027628, Loss: 2.6255905628204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1832 , TFLOPS: 71.5280082525759, Tokens per sec: 177254.28391045917, Loss: 2.641090154647827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1833 , TFLOPS: 71.50637650332611, Tokens per sec: 177200.6780528843, Loss: 2.644517660140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1834 , TFLOPS: 68.47050361916219, Tokens per sec: 169677.45061692005, Loss: 2.641204833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1835 , TFLOPS: 71.50582185293541, Tokens per sec: 177199.30356811636, Loss: 2.6385679244995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1836 , TFLOPS: 71.54795570038534, Tokens per sec: 177303.71588352375, Loss: 2.650216579437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1837 , TFLOPS: 68.60535961806944, Tokens per sec: 170011.63863782687, Loss: 2.6297473907470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1838 , TFLOPS: 71.49720001033212, Tokens per sec: 177177.93769237105, Loss: 2.6447792053222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1839 , TFLOPS: 71.49866563452292, Tokens per sec: 177181.56967056767, Loss: 2.6528565883636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1840 , TFLOPS: 70.41826261827995, Tokens per sec: 174504.2119801003, Loss: 2.6085398197174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1841 , TFLOPS: 69.80623746614808, Tokens per sec: 172987.54623298132, Loss: 2.658236265182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1842 , TFLOPS: 71.48247167637822, Tokens per sec: 177141.43925837334, Loss: 2.6351561546325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1843 , TFLOPS: 71.56659680442583, Tokens per sec: 177349.9105928232, Loss: 2.627462148666382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1844 , TFLOPS: 69.56693688801579, Tokens per sec: 172394.53303923467, Loss: 2.636423110961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1845 , TFLOPS: 70.40711684909758, Tokens per sec: 174476.59153625855, Loss: 2.6512229442596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1846 , TFLOPS: 71.50206374397321, Tokens per sec: 177189.99056011814, Loss: 2.6275272369384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1847 , TFLOPS: 69.60404124784336, Tokens per sec: 172486.48173027002, Loss: 2.6506011486053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1848 , TFLOPS: 70.45999346816382, Tokens per sec: 174607.62562314514, Loss: 2.642329454421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1849 , TFLOPS: 71.53706190322927, Tokens per sec: 177276.7198541202, Loss: 2.621123790740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1850 , TFLOPS: 70.60043974184612, Tokens per sec: 174955.66696076412, Loss: 2.639800548553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1851 , TFLOPS: 70.64709886619397, Tokens per sec: 175071.29341082496, Loss: 2.6345648765563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1852 , TFLOPS: 70.57018549931648, Tokens per sec: 174880.69361499572, Loss: 2.6263537406921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1853 , TFLOPS: 71.52506912114609, Tokens per sec: 177247.00041900406, Loss: 2.6291093826293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1854 , TFLOPS: 69.55433883813527, Tokens per sec: 172363.31368384097, Loss: 2.6505649089813232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1855 , TFLOPS: 71.50486423665555, Tokens per sec: 177196.93048948472, Loss: 2.6620938777923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1856 , TFLOPS: 70.46104406487774, Tokens per sec: 174610.22911753503, Loss: 2.6267967224121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1857 , TFLOPS: 69.49913905437732, Tokens per sec: 172226.5225389306, Loss: 2.616090774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1858 , TFLOPS: 71.5035107268066, Tokens per sec: 177193.57634297758, Loss: 2.6445164680480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1859 , TFLOPS: 71.49785552291007, Tokens per sec: 177179.56212474985, Loss: 2.6546359062194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1860 , TFLOPS: 69.42516296227342, Tokens per sec: 172043.20163355788, Loss: 2.6425087451934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1861 , TFLOPS: 70.75255371689845, Tokens per sec: 175332.62214768145, Loss: 2.630660057067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1862 , TFLOPS: 71.56388748160643, Tokens per sec: 177343.19659242016, Loss: 2.6204071044921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1863 , TFLOPS: 70.40529937688316, Tokens per sec: 174472.08763990016, Loss: 2.6474270820617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1864 , TFLOPS: 69.44962682538011, Tokens per sec: 172103.82578125325, Loss: 2.626049757003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1865 , TFLOPS: 71.52543141062469, Tokens per sec: 177247.89821224147, Loss: 2.6315114498138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1866 , TFLOPS: 71.49948178011873, Tokens per sec: 177183.59216925994, Loss: 2.640437126159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1867 , TFLOPS: 69.53179826270856, Tokens per sec: 172307.45565488422, Loss: 2.6550331115722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1868 , TFLOPS: 71.44405840459304, Tokens per sec: 177046.24693931846, Loss: 2.6166064739227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1869 , TFLOPS: 71.54719384020754, Tokens per sec: 177301.82791007732, Loss: 2.6521103382110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1870 , TFLOPS: 70.5979079188657, Tokens per sec: 174949.39282451532, Loss: 2.6327617168426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1871 , TFLOPS: 69.30435723625176, Tokens per sec: 171743.8317366269, Loss: 2.6287426948547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1872 , TFLOPS: 71.5246344906795, Tokens per sec: 177245.92335682933, Loss: 2.6451103687286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1873 , TFLOPS: 71.5382618376329, Tokens per sec: 177279.69342375605, Loss: 2.6313021183013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1874 , TFLOPS: 69.59854920330964, Tokens per sec: 172472.87183891685, Loss: 2.6488587856292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1875 , TFLOPS: 70.57143282678302, Tokens per sec: 174883.7846298623, Loss: 2.640902280807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1876 , TFLOPS: 71.48343150155873, Tokens per sec: 177143.81781090415, Loss: 2.6280553340911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1877 , TFLOPS: 69.46449861270553, Tokens per sec: 172140.67970850825, Loss: 2.636446475982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1878 , TFLOPS: 70.42906827169583, Tokens per sec: 174530.9895796058, Loss: 2.6264264583587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1879 , TFLOPS: 71.49160703836273, Tokens per sec: 177164.07769171416, Loss: 2.64180850982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1880 , TFLOPS: 70.29233126967107, Tokens per sec: 174192.14022576422, Loss: 2.640733480453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1881 , TFLOPS: 70.73462081966348, Tokens per sec: 175288.18245286148, Loss: 2.6337666511535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1882 , TFLOPS: 70.53969304162034, Tokens per sec: 174805.1299458032, Loss: 2.6362082958221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1883 , TFLOPS: 71.52886690290998, Tokens per sec: 177256.4117405752, Loss: 2.6204519271850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1884 , TFLOPS: 69.40191086645082, Tokens per sec: 171985.58037867965, Loss: 2.6644208431243896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1885 , TFLOPS: 70.3999495788464, Tokens per sec: 174458.83025103615, Loss: 2.6644225120544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1886 , TFLOPS: 71.50428100510604, Tokens per sec: 177195.48517745675, Loss: 2.647108793258667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1887 , TFLOPS: 69.57938043786243, Tokens per sec: 172425.3695265248, Loss: 2.6421313285827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1888 , TFLOPS: 71.51713598552713, Tokens per sec: 177227.34123503324, Loss: 2.6301393508911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1889 , TFLOPS: 70.53908084460168, Tokens per sec: 174803.6128541519, Loss: 2.6278536319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1890 , TFLOPS: 71.5342150559238, Tokens per sec: 177269.6650528911, Loss: 2.6457595825195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1891 , TFLOPS: 69.75180669078583, Tokens per sec: 172852.6607755888, Loss: 2.615535020828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1892 , TFLOPS: 71.51731797358826, Tokens per sec: 177227.79222149635, Loss: 2.626655340194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1893 , TFLOPS: 70.38712285198841, Tokens per sec: 174427.04420890144, Loss: 2.636139392852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1894 , TFLOPS: 69.27802793214767, Tokens per sec: 171678.58482064467, Loss: 2.6438517570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1895 , TFLOPS: 71.51523664306272, Tokens per sec: 177222.6344551766, Loss: 2.623208999633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1896 , TFLOPS: 71.5232561730946, Tokens per sec: 177242.50773401972, Loss: 2.629484176635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1897 , TFLOPS: 68.52464987745589, Tokens per sec: 169811.63100967457, Loss: 2.6222944259643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1898 , TFLOPS: 71.5179361293489, Tokens per sec: 177229.32407956666, Loss: 2.62772536277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1899 , TFLOPS: 71.51251800305573, Tokens per sec: 177215.89736575633, Loss: 2.653486967086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1900 , TFLOPS: 70.48213620938861, Tokens per sec: 174662.49777512447, Loss: 2.6102561950683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1901 , TFLOPS: 69.93920117112442, Tokens per sec: 173317.04494107395, Loss: 2.6408519744873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1902 , TFLOPS: 71.5177281022088, Tokens per sec: 177228.80856539746, Loss: 2.655306100845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1903 , TFLOPS: 71.46778500783027, Tokens per sec: 177105.04407584143, Loss: 2.6378777027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1904 , TFLOPS: 69.37682070147572, Tokens per sec: 171923.40418596152, Loss: 2.650951862335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1905 , TFLOPS: 71.50475381469222, Tokens per sec: 177196.6568516956, Loss: 2.6517176628112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1906 , TFLOPS: 71.35678414789045, Tokens per sec: 176829.97171715638, Loss: 2.622920274734497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1907 , TFLOPS: 70.42585009265838, Tokens per sec: 174523.01457743935, Loss: 2.6516525745391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1908 , TFLOPS: 70.42545090196117, Tokens per sec: 174522.02533891718, Loss: 2.6292824745178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1909 , TFLOPS: 71.49241004787537, Tokens per sec: 177166.06763774526, Loss: 2.650197982788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1910 , TFLOPS: 71.52551373866731, Tokens per sec: 177248.1022302003, Loss: 2.6374688148498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1911 , TFLOPS: 68.55074162232185, Tokens per sec: 169876.2892276954, Loss: 2.6423861980438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1912 , TFLOPS: 71.55606136161813, Tokens per sec: 177323.8026049709, Loss: 2.655449390411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1913 , TFLOPS: 71.51173321302926, Tokens per sec: 177213.952569619, Loss: 2.632556200027466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1914 , TFLOPS: 69.5979959870697, Tokens per sec: 172471.5009081324, Loss: 2.624704360961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1915 , TFLOPS: 70.61095483876956, Tokens per sec: 174981.72452927395, Loss: 2.643951654434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1916 , TFLOPS: 71.48309561916574, Tokens per sec: 177142.9854573338, Loss: 2.630260944366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1917 , TFLOPS: 70.64024666529265, Tokens per sec: 175054.31290215923, Loss: 2.635727882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1918 , TFLOPS: 70.66805809709925, Tokens per sec: 175123.2326938581, Loss: 2.666154146194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1919 , TFLOPS: 70.47197341903178, Tokens per sec: 174637.31326109648, Loss: 2.6252598762512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1920 , TFLOPS: 71.5092571132584, Tokens per sec: 177207.81652162186, Loss: 2.643986225128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1921 , TFLOPS: 69.61578550835662, Tokens per sec: 172515.585301559, Loss: 2.635972499847412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1922 , TFLOPS: 70.57156904779204, Tokens per sec: 174884.1222005286, Loss: 2.6458029747009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1923 , TFLOPS: 71.4794126534534, Tokens per sec: 177133.85866258643, Loss: 2.6292531490325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1924 , TFLOPS: 69.4183942607864, Tokens per sec: 172026.428045639, Loss: 2.6641950607299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1925 , TFLOPS: 71.49368526559333, Tokens per sec: 177169.22776772722, Loss: 2.6179230213165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1926 , TFLOPS: 70.50457692716743, Tokens per sec: 174718.10834582953, Loss: 2.657985210418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1927 , TFLOPS: 70.30674774569984, Tokens per sec: 174227.86584146947, Loss: 2.640193223953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1928 , TFLOPS: 70.57161145408178, Tokens per sec: 174884.22728798602, Loss: 2.634152412414551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1929 , TFLOPS: 71.4518667446944, Tokens per sec: 177065.59686624928, Loss: 2.646484136581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1930 , TFLOPS: 70.54771026124088, Tokens per sec: 174824.99749919254, Loss: 2.643521308898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1931 , TFLOPS: 69.60984193897413, Tokens per sec: 172500.85648189095, Loss: 2.627829074859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1932 , TFLOPS: 71.4952533948665, Tokens per sec: 177173.11376480982, Loss: 2.611445426940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1933 , TFLOPS: 71.48857425214881, Tokens per sec: 177156.56211338707, Loss: 2.6094307899475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1934 , TFLOPS: 68.50540032526574, Tokens per sec: 169763.92849883417, Loss: 2.6476125717163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1935 , TFLOPS: 71.51821806394328, Tokens per sec: 177230.02274454923, Loss: 2.625783920288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1936 , TFLOPS: 71.52262844680384, Tokens per sec: 177240.9521591215, Loss: 2.622102975845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1937 , TFLOPS: 69.58018875283354, Tokens per sec: 172427.37262006645, Loss: 2.6513054370880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1938 , TFLOPS: 70.36511274259348, Tokens per sec: 174372.5007332071, Loss: 2.6253747940063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1939 , TFLOPS: 71.52652798365581, Tokens per sec: 177250.61564101014, Loss: 2.658921957015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1940 , TFLOPS: 71.5257562210335, Tokens per sec: 177248.70312821536, Loss: 2.625739574432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1941 , TFLOPS: 69.26569246151018, Tokens per sec: 171648.0161655407, Loss: 2.5982329845428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1942 , TFLOPS: 71.53942831718247, Tokens per sec: 177282.5840885782, Loss: 2.6042914390563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1943 , TFLOPS: 71.45091147942945, Tokens per sec: 177063.2296136916, Loss: 2.6380276679992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1944 , TFLOPS: 69.61205728511112, Tokens per sec: 172506.34635365897, Loss: 2.628000259399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1945 , TFLOPS: 70.4939190812133, Tokens per sec: 174691.69702949657, Loss: 2.6412787437438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1946 , TFLOPS: 71.48721263299667, Tokens per sec: 177153.18787113336, Loss: 2.6540284156799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1947 , TFLOPS: 70.37732544219833, Tokens per sec: 174402.7651481678, Loss: 2.643657922744751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1948 , TFLOPS: 70.54436591588046, Tokens per sec: 174816.70984297936, Loss: 2.6425061225891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1949 , TFLOPS: 70.4913647459594, Tokens per sec: 174685.36710535412, Loss: 2.640996217727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1950 , TFLOPS: 71.52854524364453, Tokens per sec: 177255.6146334817, Loss: 2.621039390563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1951 , TFLOPS: 69.61218962623906, Tokens per sec: 172506.67430955256, Loss: 2.6331541538238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1952 , TFLOPS: 70.41933119064817, Tokens per sec: 174506.86002014167, Loss: 2.6623458862304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1953 , TFLOPS: 71.46224890759657, Tokens per sec: 177091.3250096105, Loss: 2.642284393310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1954 , TFLOPS: 69.09599668945461, Tokens per sec: 171227.49134885453, Loss: 2.626309394836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1955 , TFLOPS: 71.50392515867408, Tokens per sec: 177194.60335079863, Loss: 2.617027997970581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1956 , TFLOPS: 70.59559041057338, Tokens per sec: 174943.64978367233, Loss: 2.6478066444396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1957 , TFLOPS: 71.49471232726822, Tokens per sec: 177171.77293969778, Loss: 2.625121593475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1958 , TFLOPS: 69.87196967256051, Tokens per sec: 173150.43788147162, Loss: 2.65285062789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1959 , TFLOPS: 70.48925588139717, Tokens per sec: 174680.14110665827, Loss: 2.654848337173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1960 , TFLOPS: 71.536038428329, Tokens per sec: 177274.18357057223, Loss: 2.623715400695801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1961 , TFLOPS: 69.57111703295497, Tokens per sec: 172404.89189887495, Loss: 2.621173858642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1962 , TFLOPS: 71.52673253490107, Tokens per sec: 177251.12254152904, Loss: 2.6234140396118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1963 , TFLOPS: 70.49964387404505, Tokens per sec: 174705.8836967716, Loss: 2.6434144973754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1964 , TFLOPS: 69.58331639870806, Tokens per sec: 172435.12327108218, Loss: 2.6498630046844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1965 , TFLOPS: 71.49518732746715, Tokens per sec: 177172.9500425163, Loss: 2.6428534984588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1966 , TFLOPS: 71.54271295725127, Tokens per sec: 177290.7237885017, Loss: 2.6354074478149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1967 , TFLOPS: 70.4307048675407, Tokens per sec: 174535.04524439535, Loss: 2.614943027496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1968 , TFLOPS: 69.74645822572295, Tokens per sec: 172839.4066899809, Loss: 2.645552396774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1969 , TFLOPS: 71.52219604893617, Tokens per sec: 177239.8806295728, Loss: 2.629202365875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1970 , TFLOPS: 71.45782717046573, Tokens per sec: 177080.36745790578, Loss: 2.637237548828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1971 , TFLOPS: 68.34165879871131, Tokens per sec: 169358.15895841407, Loss: 2.618337631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1972 , TFLOPS: 71.46880865309797, Tokens per sec: 177107.5807815786, Loss: 2.6483254432678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1973 , TFLOPS: 71.49661137141943, Tokens per sec: 177176.47898030147, Loss: 2.650177478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1974 , TFLOPS: 69.39846724609068, Tokens per sec: 171977.04670808112, Loss: 2.6322367191314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1975 , TFLOPS: 70.87185613508042, Tokens per sec: 175628.26668218267, Loss: 2.636582374572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1976 , TFLOPS: 71.50666537770545, Tokens per sec: 177201.39391541842, Loss: 2.6415834426879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1977 , TFLOPS: 71.50762012547162, Tokens per sec: 177203.75988555714, Loss: 2.6425814628601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1978 , TFLOPS: 68.62319506962622, Tokens per sec: 170055.83685152174, Loss: 2.622610092163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1979 , TFLOPS: 71.51877660687246, Tokens per sec: 177231.40687545633, Loss: 2.611290454864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1980 , TFLOPS: 71.54653221123539, Tokens per sec: 177300.18832059987, Loss: 2.631737470626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1981 , TFLOPS: 69.53013144902532, Tokens per sec: 172303.32510696247, Loss: 2.6290509700775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1982 , TFLOPS: 70.49048906851999, Tokens per sec: 174683.19708019544, Loss: 2.6335325241088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1983 , TFLOPS: 71.49414274856879, Tokens per sec: 177170.3614609363, Loss: 2.6396186351776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1984 , TFLOPS: 70.67765780635825, Tokens per sec: 175147.0218308972, Loss: 2.6505184173583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1985 , TFLOPS: 67.19027443772286, Tokens per sec: 166504.90167642682, Loss: 2.6518046855926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1986 , TFLOPS: 71.52239630476431, Tokens per sec: 177240.37688557498, Loss: 2.6547534465789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1987 , TFLOPS: 71.47673088687398, Tokens per sec: 177127.21294958168, Loss: 2.6329002380371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1988 , TFLOPS: 69.43939065029615, Tokens per sec: 172078.45941178742, Loss: 2.6130483150482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1989 , TFLOPS: 70.05983913918595, Tokens per sec: 173615.9991152997, Loss: 2.6188747882843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1990 , TFLOPS: 71.46095022471398, Tokens per sec: 177088.10673036575, Loss: 2.608783721923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1991 , TFLOPS: 69.57109293446202, Tokens per sec: 172404.83218015474, Loss: 2.622371196746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1992 , TFLOPS: 71.51367394485362, Tokens per sec: 177218.761916869, Loss: 2.6234841346740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1993 , TFLOPS: 70.44188411503262, Tokens per sec: 174562.7486511761, Loss: 2.6343154907226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1994 , TFLOPS: 70.38693847598906, Tokens per sec: 174426.5873048643, Loss: 2.599613904953003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1995 , TFLOPS: 70.60917067298904, Tokens per sec: 174977.30316992773, Loss: 2.6277523040771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1996 , TFLOPS: 70.53153345864719, Tokens per sec: 174784.90960034344, Loss: 2.619051933288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1997 , TFLOPS: 71.5455167414126, Tokens per sec: 177297.6718745158, Loss: 2.6394333839416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1998 , TFLOPS: 69.61144180806518, Tokens per sec: 172504.82113373873, Loss: 2.6229918003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1999 , TFLOPS: 71.5067875012099, Tokens per sec: 177201.69655091548, Loss: 2.6461434364318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2000 , TFLOPS: 70.53527817540802, Tokens per sec: 174794.1894209914, Loss: 2.6497998237609863 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/optimizer.pt +[rank0]:[2024-08-26 06:17:07,998] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00641672399797244, 'preprocessing_with_comm': 0.0015408710023621097, 'state_converting': 0.9875604970002314, : 0.9970033719982894}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000, took 4.63s + eval ppl=12.214401245117188, eval loss=2.5026156902313232 +------------------------------------------------------------------ +iteration: 2001 , TFLOPS: 66.37796240724306, Tokens per sec: 164491.90298134062, Loss: 2.6397476196289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2002 , TFLOPS: 70.83973372766356, Tokens per sec: 175548.66381802838, Loss: 2.6337671279907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2003 , TFLOPS: 71.46642786563433, Tokens per sec: 177101.68092797833, Loss: 2.652313709259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2004 , TFLOPS: 70.07383546774777, Tokens per sec: 173650.68355929875, Loss: 2.632481098175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2005 , TFLOPS: 69.80789982567235, Tokens per sec: 172991.6657430065, Loss: 2.629749059677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2006 , TFLOPS: 71.52751727732814, Tokens per sec: 177253.06721970983, Loss: 2.623729705810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2007 , TFLOPS: 71.50900694841447, Tokens per sec: 177207.1965855807, Loss: 2.635507822036743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2008 , TFLOPS: 68.52494088331585, Tokens per sec: 169812.35215425253, Loss: 2.624664545059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2009 , TFLOPS: 71.45074181510385, Tokens per sec: 177062.8091668027, Loss: 2.614309310913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2010 , TFLOPS: 71.4836318202515, Tokens per sec: 177144.31422269173, Loss: 2.6314332485198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2011 , TFLOPS: 70.38911641672343, Tokens per sec: 174431.98448192407, Loss: 2.620551109313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2012 , TFLOPS: 69.66129139801707, Tokens per sec: 172628.35390902517, Loss: 2.611025333404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2013 , TFLOPS: 71.52233404234624, Tokens per sec: 177240.22259244422, Loss: 2.6379215717315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2014 , TFLOPS: 70.41143038685507, Tokens per sec: 174487.28095799702, Loss: 2.6459760665893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2015 , TFLOPS: 69.52455301596353, Tokens per sec: 172289.5011353202, Loss: 2.634732246398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2016 , TFLOPS: 71.54387694261773, Tokens per sec: 177293.60827246335, Loss: 2.630399703979492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2017 , TFLOPS: 71.50790015334046, Tokens per sec: 177204.45382546374, Loss: 2.6206254959106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2018 , TFLOPS: 70.06235603763925, Tokens per sec: 173622.2362669262, Loss: 2.6570069789886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2019 , TFLOPS: 69.79841897046066, Tokens per sec: 172968.1711393895, Loss: 2.648817777633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2020 , TFLOPS: 71.4878924514823, Tokens per sec: 177154.8725362289, Loss: 2.652641534805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2021 , TFLOPS: 70.42691378021277, Tokens per sec: 174525.65051237398, Loss: 2.6134040355682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2022 , TFLOPS: 70.63848139526125, Tokens per sec: 175049.9383685609, Loss: 2.6242148876190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2023 , TFLOPS: 70.35552064686694, Tokens per sec: 174348.73046334012, Loss: 2.6287598609924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2024 , TFLOPS: 71.47765750663177, Tokens per sec: 177129.50921541706, Loss: 2.61247181892395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2025 , TFLOPS: 69.66028049041354, Tokens per sec: 172625.84876862296, Loss: 2.6471080780029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2026 , TFLOPS: 71.512465040514, Tokens per sec: 177215.76611874363, Loss: 2.641843795776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2027 , TFLOPS: 70.41645077062869, Tokens per sec: 174499.722022029, Loss: 2.6431312561035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2028 , TFLOPS: 69.50613774815382, Tokens per sec: 172243.8660730785, Loss: 2.6190686225891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2029 , TFLOPS: 71.49254041554371, Tokens per sec: 177166.3907031886, Loss: 2.6293272972106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2030 , TFLOPS: 70.33260458526303, Tokens per sec: 174291.94193827306, Loss: 2.623532295227051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2031 , TFLOPS: 70.40100069800944, Tokens per sec: 174461.43504011247, Loss: 2.6354575157165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2032 , TFLOPS: 70.67705704406306, Tokens per sec: 175145.53307575028, Loss: 2.6322734355926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2033 , TFLOPS: 71.53435153961239, Tokens per sec: 177270.00327450637, Loss: 2.637523889541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2034 , TFLOPS: 69.42344725629741, Tokens per sec: 172038.94992514868, Loss: 2.6192522048950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2035 , TFLOPS: 70.64048519941096, Tokens per sec: 175054.90401598113, Loss: 2.621804714202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2036 , TFLOPS: 71.53089636566064, Tokens per sec: 177261.44097283506, Loss: 2.6082844734191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2037 , TFLOPS: 70.37899051709212, Tokens per sec: 174406.89138717778, Loss: 2.6288747787475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2038 , TFLOPS: 70.34987119117802, Tokens per sec: 174334.7304898041, Loss: 2.6247541904449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2039 , TFLOPS: 70.68443173972592, Tokens per sec: 175163.80838399264, Loss: 2.6339988708496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2040 , TFLOPS: 71.5073251588492, Tokens per sec: 177203.02892577354, Loss: 2.62851619720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2041 , TFLOPS: 69.3247360074857, Tokens per sec: 171794.33257665156, Loss: 2.6318111419677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2042 , TFLOPS: 70.60655919327885, Tokens per sec: 174970.83163552766, Loss: 2.6300110816955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2043 , TFLOPS: 71.48188741063812, Tokens per sec: 177139.9913835072, Loss: 2.6102914810180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2044 , TFLOPS: 71.56602054635336, Tokens per sec: 177348.48256183954, Loss: 2.620708465576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2045 , TFLOPS: 68.43481587889339, Tokens per sec: 169589.0124652048, Loss: 2.6270453929901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2046 , TFLOPS: 71.44656995560076, Tokens per sec: 177052.4708393858, Loss: 2.651317596435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2047 , TFLOPS: 71.49726002761379, Tokens per sec: 177178.08642180648, Loss: 2.6400668621063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2048 , TFLOPS: 70.39097003013724, Tokens per sec: 174436.5779401559, Loss: 2.6253018379211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2049 , TFLOPS: 69.73083930752189, Tokens per sec: 172800.7012901128, Loss: 2.640903949737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2050 , TFLOPS: 71.52438539705186, Tokens per sec: 177245.30607538013, Loss: 2.6378540992736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2051 , TFLOPS: 69.83755218738503, Tokens per sec: 173065.14756180783, Loss: 2.6397528648376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2052 , TFLOPS: 69.61387045433744, Tokens per sec: 172510.83958673888, Loss: 2.631518602371216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2053 , TFLOPS: 71.52986385726693, Tokens per sec: 177258.88230329528, Loss: 2.6167402267456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2054 , TFLOPS: 71.4903649577806, Tokens per sec: 177160.99967920515, Loss: 2.619072198867798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2055 , TFLOPS: 69.81276349821201, Tokens per sec: 173003.7184593375, Loss: 2.617819309234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2056 , TFLOPS: 70.45326044740294, Tokens per sec: 174590.94045599623, Loss: 2.6228675842285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2057 , TFLOPS: 71.51167209929235, Tokens per sec: 177213.80112304672, Loss: 2.6028192043304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2058 , TFLOPS: 70.40957892619612, Tokens per sec: 174482.69283452892, Loss: 2.6365458965301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2059 , TFLOPS: 70.66339456994004, Tokens per sec: 175111.67595982767, Loss: 2.621718406677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2060 , TFLOPS: 70.5107579483267, Tokens per sec: 174733.42559715788, Loss: 2.648587465286255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2061 , TFLOPS: 70.5083974706405, Tokens per sec: 174727.5760734234, Loss: 2.6117234230041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2062 , TFLOPS: 70.54126614322234, Tokens per sec: 174809.0282648644, Loss: 2.621095657348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2063 , TFLOPS: 71.57230899241955, Tokens per sec: 177364.06602392107, Loss: 2.6091015338897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2064 , TFLOPS: 70.41087329317911, Tokens per sec: 174485.90041849905, Loss: 2.6120424270629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2065 , TFLOPS: 69.57081965168658, Tokens per sec: 172404.1549553326, Loss: 2.6282567977905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2066 , TFLOPS: 71.45670389523436, Tokens per sec: 177077.58385814345, Loss: 2.6519761085510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2067 , TFLOPS: 70.51911760255958, Tokens per sec: 174754.1417412389, Loss: 2.6363465785980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2068 , TFLOPS: 69.8157010291996, Tokens per sec: 173010.99798472054, Loss: 2.647368907928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2069 , TFLOPS: 70.57509307501529, Tokens per sec: 174892.85512819147, Loss: 2.6339566707611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2070 , TFLOPS: 71.54830742138391, Tokens per sec: 177304.58748690345, Loss: 2.606503486633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2071 , TFLOPS: 70.45099478113741, Tokens per sec: 174585.3258854065, Loss: 2.6385669708251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2072 , TFLOPS: 69.79205576785984, Tokens per sec: 172952.40242238005, Loss: 2.6113498210906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2073 , TFLOPS: 71.51196356649714, Tokens per sec: 177214.52341088792, Loss: 2.631537675857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2074 , TFLOPS: 70.44699231543984, Tokens per sec: 174575.40733449993, Loss: 2.611013174057007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2075 , TFLOPS: 69.56695737295117, Tokens per sec: 172394.5838031611, Loss: 2.6190598011016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2076 , TFLOPS: 71.5656478428828, Tokens per sec: 177347.5589615828, Loss: 2.6562671661376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2077 , TFLOPS: 71.53009149594455, Tokens per sec: 177259.4464170149, Loss: 2.622771739959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2078 , TFLOPS: 69.28173455885161, Tokens per sec: 171687.77024993568, Loss: 2.6197612285614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2079 , TFLOPS: 70.68303143815312, Tokens per sec: 175160.3382824393, Loss: 2.6232481002807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2080 , TFLOPS: 71.53278772563061, Tokens per sec: 177266.12797119113, Loss: 2.642537832260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2081 , TFLOPS: 70.09531729414086, Tokens per sec: 173703.91789151935, Loss: 2.6031618118286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2082 , TFLOPS: 69.55909407505764, Tokens per sec: 172375.0976847672, Loss: 2.601475715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2083 , TFLOPS: 71.48529051774092, Tokens per sec: 177148.42465779113, Loss: 2.6403138637542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2084 , TFLOPS: 71.54844141585173, Tokens per sec: 177304.91953995542, Loss: 2.617360830307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2085 , TFLOPS: 70.42194046318951, Tokens per sec: 174513.32608493426, Loss: 2.6254777908325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2086 , TFLOPS: 69.64648682923489, Tokens per sec: 172591.66655672505, Loss: 2.6174561977386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2087 , TFLOPS: 71.53107084685627, Tokens per sec: 177261.87335645853, Loss: 2.642573833465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2088 , TFLOPS: 70.43298662906784, Tokens per sec: 174540.6997008164, Loss: 2.622751235961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2089 , TFLOPS: 69.65092057884605, Tokens per sec: 172602.65387667934, Loss: 2.640939712524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2090 , TFLOPS: 71.46575927900118, Tokens per sec: 177100.02409665048, Loss: 2.6395926475524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2091 , TFLOPS: 71.51832864346093, Tokens per sec: 177230.29677277533, Loss: 2.632568120956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2092 , TFLOPS: 69.14964919066828, Tokens per sec: 171360.44815717294, Loss: 2.5985498428344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2093 , TFLOPS: 70.3429515294122, Tokens per sec: 174317.5828056847, Loss: 2.6536784172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2094 , TFLOPS: 71.52752486314688, Tokens per sec: 177253.08601820434, Loss: 2.6127572059631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2095 , TFLOPS: 69.5433802649333, Tokens per sec: 172336.15713226, Loss: 2.6396942138671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2096 , TFLOPS: 71.56906819857093, Tokens per sec: 177356.03497976132, Loss: 2.612191677093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2097 , TFLOPS: 70.52519835963311, Tokens per sec: 174769.21052711777, Loss: 2.6350512504577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2098 , TFLOPS: 70.37273592969073, Tokens per sec: 174391.39183059745, Loss: 2.6341493129730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2099 , TFLOPS: 70.71634420884979, Tokens per sec: 175242.89099793026, Loss: 2.6135735511779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2100 , TFLOPS: 71.50847079150982, Tokens per sec: 177205.86792970842, Loss: 2.609767198562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2101 , TFLOPS: 69.37480671260877, Tokens per sec: 171918.41329969006, Loss: 2.630126714706421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2102 , TFLOPS: 70.59714415708424, Tokens per sec: 174947.50013868554, Loss: 2.632883310317993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2103 , TFLOPS: 71.50451442983966, Tokens per sec: 177196.06362966052, Loss: 2.629185199737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2104 , TFLOPS: 70.53968244021969, Tokens per sec: 174805.10367436454, Loss: 2.6207897663116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2105 , TFLOPS: 70.4602806640513, Tokens per sec: 174608.33732619154, Loss: 2.615616798400879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2106 , TFLOPS: 70.61835156731448, Tokens per sec: 175000.05443742516, Loss: 2.6224164962768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2107 , TFLOPS: 71.47063970114698, Tokens per sec: 177112.11832035065, Loss: 2.623669385910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2108 , TFLOPS: 69.33766982754395, Tokens per sec: 171826.3840075338, Loss: 2.618112564086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2109 , TFLOPS: 70.67692040609091, Tokens per sec: 175145.1944718034, Loss: 2.607649326324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2110 , TFLOPS: 71.4602800044108, Tokens per sec: 177086.44585062363, Loss: 2.63578724861145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2111 , TFLOPS: 70.44940565132201, Tokens per sec: 174581.38784666682, Loss: 2.6109421253204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2112 , TFLOPS: 69.41356562285458, Tokens per sec: 172014.46214892558, Loss: 2.6507816314697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2113 , TFLOPS: 71.51864558313987, Tokens per sec: 177231.08218421353, Loss: 2.643998622894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2114 , TFLOPS: 71.46265047586746, Tokens per sec: 177092.3201400222, Loss: 2.6425113677978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2115 , TFLOPS: 68.74835244110501, Tokens per sec: 170365.99060526883, Loss: 2.6223058700561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2116 , TFLOPS: 71.50587720070273, Tokens per sec: 177199.44072598082, Loss: 2.636784076690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2117 , TFLOPS: 71.54633772771852, Tokens per sec: 177299.70636902095, Loss: 2.6271564960479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2118 , TFLOPS: 70.40439615765277, Tokens per sec: 174469.8493631494, Loss: 2.621680498123169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2119 , TFLOPS: 69.49480990693685, Tokens per sec: 172215.7944346782, Loss: 2.631552219390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2120 , TFLOPS: 71.53146940045342, Tokens per sec: 177262.86101617658, Loss: 2.612750291824341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2121 , TFLOPS: 71.48602969005285, Tokens per sec: 177150.25640820665, Loss: 2.6183066368103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2122 , TFLOPS: 69.41112989306777, Tokens per sec: 172008.42614219608, Loss: 2.6434717178344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2123 , TFLOPS: 70.51201553450701, Tokens per sec: 174736.54203424725, Loss: 2.6085448265075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2124 , TFLOPS: 71.52083900369672, Tokens per sec: 177236.51772197863, Loss: 2.6359975337982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2125 , TFLOPS: 70.37789238958852, Tokens per sec: 174404.17010625516, Loss: 2.626246690750122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2126 , TFLOPS: 69.45769802789317, Tokens per sec: 172123.827110197, Loss: 2.6142024993896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2127 , TFLOPS: 71.49353878070806, Tokens per sec: 177168.86476204486, Loss: 2.6236376762390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2128 , TFLOPS: 70.39810329929618, Tokens per sec: 174454.2549669262, Loss: 2.648240089416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2129 , TFLOPS: 70.5603478979485, Tokens per sec: 174856.31495510464, Loss: 2.6157422065734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2130 , TFLOPS: 70.56955392254218, Tokens per sec: 174879.1284981751, Loss: 2.6158955097198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2131 , TFLOPS: 71.56386983752364, Tokens per sec: 177343.15286843939, Loss: 2.6352710723876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2132 , TFLOPS: 69.570877319482, Tokens per sec: 172404.2978624823, Loss: 2.6211259365081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2133 , TFLOPS: 71.55178280536647, Tokens per sec: 177313.19987125738, Loss: 2.6120994091033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2134 , TFLOPS: 70.40366807150819, Tokens per sec: 174468.04508547782, Loss: 2.6307249069213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2135 , TFLOPS: 70.43033136856216, Tokens per sec: 174534.11967278222, Loss: 2.614170551300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2136 , TFLOPS: 70.87550826673069, Tokens per sec: 175637.31706672756, Loss: 2.632143020629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2137 , TFLOPS: 71.53227609881304, Tokens per sec: 177264.8601035769, Loss: 2.611708641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2138 , TFLOPS: 70.42809364324759, Tokens per sec: 174528.5743429469, Loss: 2.644439220428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2139 , TFLOPS: 69.86721526598024, Tokens per sec: 173138.6559382246, Loss: 2.626899480819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2140 , TFLOPS: 71.51776008923939, Tokens per sec: 177228.88783278293, Loss: 2.6202285289764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2141 , TFLOPS: 70.43791829534692, Tokens per sec: 174552.92091312382, Loss: 2.6558456420898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2142 , TFLOPS: 69.60204430495936, Tokens per sec: 172481.53308582207, Loss: 2.6233749389648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2143 , TFLOPS: 71.51407621686114, Tokens per sec: 177219.75879121755, Loss: 2.6287589073181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2144 , TFLOPS: 71.5115086951591, Tokens per sec: 177213.39618960404, Loss: 2.6010334491729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2145 , TFLOPS: 69.4434085870333, Tokens per sec: 172088.41630163434, Loss: 2.6280739307403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2146 , TFLOPS: 70.5871879315307, Tokens per sec: 174922.82751499512, Loss: 2.6347851753234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2147 , TFLOPS: 71.52844204464361, Tokens per sec: 177255.3588949891, Loss: 2.6221063137054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2148 , TFLOPS: 70.36383055934697, Tokens per sec: 174369.32334187807, Loss: 2.6059956550598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2149 , TFLOPS: 70.67064633748262, Tokens per sec: 175129.64663864634, Loss: 2.5831298828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2150 , TFLOPS: 71.5590650961855, Tokens per sec: 177331.2461901175, Loss: 2.6341795921325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2151 , TFLOPS: 71.40185527793514, Tokens per sec: 176941.66294240198, Loss: 2.6423275470733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2152 , TFLOPS: 69.2668192193579, Tokens per sec: 171650.80839561, Loss: 2.600722551345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2153 , TFLOPS: 70.82625795686425, Tokens per sec: 175515.269373509, Loss: 2.6046299934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2154 , TFLOPS: 71.50909824037596, Tokens per sec: 177207.4228171179, Loss: 2.647634983062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2155 , TFLOPS: 70.42140252707222, Tokens per sec: 174511.99301997703, Loss: 2.6034040451049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2156 , TFLOPS: 69.6911110345864, Tokens per sec: 172702.25025334812, Loss: 2.6258208751678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2157 , TFLOPS: 71.54792614226092, Tokens per sec: 177303.64263523536, Loss: 2.603952407836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2158 , TFLOPS: 71.5306471897236, Tokens per sec: 177260.8234874141, Loss: 2.629871368408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2159 , TFLOPS: 69.59327691564323, Tokens per sec: 172459.80652928902, Loss: 2.5996954441070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2160 , TFLOPS: 70.38107887633562, Tokens per sec: 174412.06657143493, Loss: 2.6203391551971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2161 , TFLOPS: 71.56072281792662, Tokens per sec: 177335.35420720512, Loss: 2.588827610015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2162 , TFLOPS: 69.69440290203413, Tokens per sec: 172710.40786351272, Loss: 2.602799415588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2163 , TFLOPS: 70.56418364870538, Tokens per sec: 174865.820368023, Loss: 2.62107515335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2164 , TFLOPS: 71.48169107792916, Tokens per sec: 177139.50484942656, Loss: 2.6178030967712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2165 , TFLOPS: 70.39209508016987, Tokens per sec: 174439.36593807122, Loss: 2.6115808486938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2166 , TFLOPS: 70.4825032581604, Tokens per sec: 174663.40736241476, Loss: 2.620844602584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2167 , TFLOPS: 70.45427763478052, Tokens per sec: 174593.4611583703, Loss: 2.618283748626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2168 , TFLOPS: 71.52436397503398, Tokens per sec: 177245.25298925999, Loss: 2.621828079223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2169 , TFLOPS: 69.91815422341115, Tokens per sec: 173264.8882861278, Loss: 2.627126455307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2170 , TFLOPS: 71.5460597481521, Tokens per sec: 177299.01750503338, Loss: 2.620482921600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2171 , TFLOPS: 70.47777845632939, Tokens per sec: 174651.6987829976, Loss: 2.6404521465301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2172 , TFLOPS: 70.39101227583947, Tokens per sec: 174436.68262965974, Loss: 2.636547565460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2173 , TFLOPS: 70.60068823150944, Tokens per sec: 174956.2827455232, Loss: 2.633415937423706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2174 , TFLOPS: 71.51729990804718, Tokens per sec: 177227.74745309548, Loss: 2.604937791824341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2175 , TFLOPS: 69.12523721711868, Tokens per sec: 171299.95259751612, Loss: 2.6095006465911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2176 , TFLOPS: 70.74646838728349, Tokens per sec: 175317.5420305412, Loss: 2.624882221221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2177 , TFLOPS: 71.49189934605246, Tokens per sec: 177164.80206236965, Loss: 2.615044355392456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2178 , TFLOPS: 70.54513752235628, Tokens per sec: 174818.6219688259, Loss: 2.6230010986328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2179 , TFLOPS: 69.47106206838602, Tokens per sec: 172156.94467470882, Loss: 2.6090455055236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2180 , TFLOPS: 71.50343274675066, Tokens per sec: 177193.38309980877, Loss: 2.5975327491760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2181 , TFLOPS: 71.48767720516095, Tokens per sec: 177154.3391321333, Loss: 2.6140670776367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2182 , TFLOPS: 68.7215817246269, Tokens per sec: 170299.64982079822, Loss: 2.6386265754699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2183 , TFLOPS: 71.51349423572294, Tokens per sec: 177218.3165778466, Loss: 2.615370988845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2184 , TFLOPS: 71.4910459946622, Tokens per sec: 177162.6873636202, Loss: 2.634354591369629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2185 , TFLOPS: 70.21422494465821, Tokens per sec: 173998.5841482604, Loss: 2.602400302886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2186 , TFLOPS: 70.60006228453649, Tokens per sec: 174954.7315799705, Loss: 2.626345634460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2187 , TFLOPS: 71.55616017223491, Tokens per sec: 177324.04746856415, Loss: 2.59956955909729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2188 , TFLOPS: 71.49720940489543, Tokens per sec: 177177.9609731338, Loss: 2.606720447540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2189 , TFLOPS: 68.6645828038445, Tokens per sec: 170158.40021614992, Loss: 2.6304993629455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2190 , TFLOPS: 71.51158384337666, Tokens per sec: 177213.58241516538, Loss: 2.6256954669952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2191 , TFLOPS: 71.53223501396137, Tokens per sec: 177264.75829078857, Loss: 2.618786573410034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2192 , TFLOPS: 70.37907322093105, Tokens per sec: 174407.0963364012, Loss: 2.6263070106506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2193 , TFLOPS: 69.46000768753728, Tokens per sec: 172129.5507012251, Loss: 2.6107852458953857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2194 , TFLOPS: 71.50151990808685, Tokens per sec: 177188.64287488328, Loss: 2.6215081214904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2195 , TFLOPS: 70.5034405143268, Tokens per sec: 174715.29218962422, Loss: 2.631082773208618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2196 , TFLOPS: 70.71653115530847, Tokens per sec: 175243.35427184874, Loss: 2.6214771270751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2197 , TFLOPS: 70.44973723324598, Tokens per sec: 174582.20954320117, Loss: 2.6325275897979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2198 , TFLOPS: 71.5185808057563, Tokens per sec: 177230.9216587212, Loss: 2.611900806427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2199 , TFLOPS: 69.43574872980085, Tokens per sec: 172069.43433160943, Loss: 2.617325782775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2200 , TFLOPS: 70.47037760217972, Tokens per sec: 174633.35865114335, Loss: 2.6030454635620117 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/optimizer.pt +[rank0]:[2024-08-26 06:42:13,872] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006377596997481305, 'preprocessing_with_comm': 0.0012459979989216663, 'state_converting': 0.9540527139979531, : 0.9630277539981762}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200, took 4.29s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000200 + eval ppl=11.64908218383789, eval loss=2.4552273750305176 +------------------------------------------------------------------ +iteration: 2201 , TFLOPS: 66.77693337731031, Tokens per sec: 165480.59699544782, Loss: 2.625748634338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2202 , TFLOPS: 69.95638547406212, Tokens per sec: 173359.62953676083, Loss: 2.6285932064056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2203 , TFLOPS: 70.91169239836438, Tokens per sec: 175726.98533092695, Loss: 2.6183524131774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2204 , TFLOPS: 69.49561873940841, Tokens per sec: 172217.79881064294, Loss: 2.6407852172851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2205 , TFLOPS: 70.35810365008759, Tokens per sec: 174355.13142986217, Loss: 2.6257569789886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2206 , TFLOPS: 70.60073499413753, Tokens per sec: 174956.39862846694, Loss: 2.634307861328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2207 , TFLOPS: 70.34530004501856, Tokens per sec: 174323.40268606762, Loss: 2.6261134147644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2208 , TFLOPS: 71.48310925669642, Tokens per sec: 177143.0192526371, Loss: 2.625869035720825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2209 , TFLOPS: 69.86735987369546, Tokens per sec: 173139.01429207277, Loss: 2.5901670455932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2210 , TFLOPS: 71.51945362506217, Tokens per sec: 177233.08460111014, Loss: 2.6300389766693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2211 , TFLOPS: 70.45027119623619, Tokens per sec: 174583.5327623118, Loss: 2.5928537845611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2212 , TFLOPS: 70.35542416132571, Tokens per sec: 174348.4913615402, Loss: 2.6203384399414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2213 , TFLOPS: 70.68722842271428, Tokens per sec: 175170.73887251786, Loss: 2.6179494857788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2214 , TFLOPS: 71.48684174411629, Tokens per sec: 177152.26876763097, Loss: 2.6023802757263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2215 , TFLOPS: 69.39507793316557, Tokens per sec: 171968.64761728907, Loss: 2.617983102798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2216 , TFLOPS: 70.54950473758302, Tokens per sec: 174829.44440924624, Loss: 2.626372814178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2217 , TFLOPS: 71.53818198379479, Tokens per sec: 177279.49553714856, Loss: 2.6178550720214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2218 , TFLOPS: 71.51360467498377, Tokens per sec: 177218.5902585008, Loss: 2.6239755153656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2219 , TFLOPS: 68.64414459976481, Tokens per sec: 170107.75209498644, Loss: 2.634371042251587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2220 , TFLOPS: 71.47336221120531, Tokens per sec: 177118.86500018445, Loss: 2.587698221206665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2221 , TFLOPS: 71.46998694511711, Tokens per sec: 177110.50071900166, Loss: 2.653137683868408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2222 , TFLOPS: 70.39178806525688, Tokens per sec: 174438.60512129634, Loss: 2.6097095012664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2223 , TFLOPS: 69.53842373633968, Tokens per sec: 172323.8743083983, Loss: 2.596695899963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2224 , TFLOPS: 71.49779080244859, Tokens per sec: 177179.4017403168, Loss: 2.6087419986724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2225 , TFLOPS: 71.51996338737565, Tokens per sec: 177234.3478482779, Loss: 2.631394863128662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2226 , TFLOPS: 68.83671570711806, Tokens per sec: 170584.96451246645, Loss: 2.6363093852996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2227 , TFLOPS: 71.51692496822355, Tokens per sec: 177226.81831090947, Loss: 2.6165950298309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2228 , TFLOPS: 71.4496945136678, Tokens per sec: 177060.2138384744, Loss: 2.616981267929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2229 , TFLOPS: 69.45383447685428, Tokens per sec: 172114.2528051173, Loss: 2.609297037124634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2230 , TFLOPS: 69.79883341770855, Tokens per sec: 172969.1981853249, Loss: 2.6134021282196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2231 , TFLOPS: 71.55987117906108, Tokens per sec: 177333.24375228054, Loss: 2.6089866161346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2232 , TFLOPS: 70.45410014473694, Tokens per sec: 174593.02131849015, Loss: 2.6100785732269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2233 , TFLOPS: 69.6688931589227, Tokens per sec: 172647.1919099524, Loss: 2.601240873336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2234 , TFLOPS: 71.47804076816095, Tokens per sec: 177130.4589797061, Loss: 2.6120522022247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2235 , TFLOPS: 70.37881902885752, Tokens per sec: 174406.46642044146, Loss: 2.623218536376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2236 , TFLOPS: 70.57174763815301, Tokens per sec: 174884.56476711648, Loss: 2.6141486167907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2237 , TFLOPS: 70.39258361971838, Tokens per sec: 174440.57659288953, Loss: 2.5957117080688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2238 , TFLOPS: 71.49550132015075, Tokens per sec: 177173.72815097542, Loss: 2.6157000064849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2239 , TFLOPS: 70.401875585849, Tokens per sec: 174463.60310855578, Loss: 2.5964484214782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2240 , TFLOPS: 70.93408120190747, Tokens per sec: 175782.46725243708, Loss: 2.6015076637268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2241 , TFLOPS: 70.4833329087312, Tokens per sec: 174665.46332792274, Loss: 2.6260435581207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2242 , TFLOPS: 70.44179358756816, Tokens per sec: 174562.52431414672, Loss: 2.6177079677581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2243 , TFLOPS: 70.71648085762102, Tokens per sec: 175243.2296286385, Loss: 2.614182233810425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2244 , TFLOPS: 71.5070413881907, Tokens per sec: 177202.32571082187, Loss: 2.6246297359466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2245 , TFLOPS: 70.42314054300111, Tokens per sec: 174516.3000149071, Loss: 2.614849805831909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2246 , TFLOPS: 69.44807499678426, Tokens per sec: 172099.9801790447, Loss: 2.6323957443237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2247 , TFLOPS: 71.4903435947323, Tokens per sec: 177160.94673921814, Loss: 2.6242361068725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2248 , TFLOPS: 70.49966747478868, Tokens per sec: 174705.94218201417, Loss: 2.6160762310028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2249 , TFLOPS: 69.61269864858235, Tokens per sec: 172507.93572299267, Loss: 2.61575984954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2250 , TFLOPS: 71.54999269604238, Tokens per sec: 177308.76378315547, Loss: 2.6088931560516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2251 , TFLOPS: 71.46210600476138, Tokens per sec: 177090.9708806429, Loss: 2.6393189430236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2252 , TFLOPS: 69.33942636672417, Tokens per sec: 171830.73690512116, Loss: 2.625371217727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2253 , TFLOPS: 70.67836977481787, Tokens per sec: 175148.78616716992, Loss: 2.6269171237945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2254 , TFLOPS: 71.55531474435509, Tokens per sec: 177321.95240514638, Loss: 2.613130807876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2255 , TFLOPS: 70.40292764258683, Tokens per sec: 174466.2102210457, Loss: 2.6086370944976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2256 , TFLOPS: 69.58111668996699, Tokens per sec: 172429.67215050384, Loss: 2.6055378913879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2257 , TFLOPS: 71.53334281377374, Tokens per sec: 177267.50354075787, Loss: 2.6155736446380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2258 , TFLOPS: 71.53296267029636, Tokens per sec: 177266.56150334462, Loss: 2.6318840980529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2259 , TFLOPS: 69.44402390176863, Tokens per sec: 172089.94111933102, Loss: 2.614428758621216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2260 , TFLOPS: 70.71591314578302, Tokens per sec: 175241.8227761653, Loss: 2.6148316860198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2261 , TFLOPS: 71.51105245804996, Tokens per sec: 177212.26558378892, Loss: 2.613624095916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2262 , TFLOPS: 70.40236710720374, Tokens per sec: 174464.82115261836, Loss: 2.6305134296417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2263 , TFLOPS: 69.69216803708755, Tokens per sec: 172704.86962198443, Loss: 2.6282215118408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2264 , TFLOPS: 71.57227308637188, Tokens per sec: 177363.97704477957, Loss: 2.597916603088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2265 , TFLOPS: 71.54133758734095, Tokens per sec: 177287.3154703544, Loss: 2.599990129470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2266 , TFLOPS: 69.4623112763234, Tokens per sec: 172135.25924799836, Loss: 2.613858222961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2267 , TFLOPS: 70.49423545412097, Tokens per sec: 174692.48103641308, Loss: 2.602992296218872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2268 , TFLOPS: 71.48839436987606, Tokens per sec: 177156.11634529955, Loss: 2.623915672302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2269 , TFLOPS: 69.63872966827175, Tokens per sec: 172572.44345733654, Loss: 2.6331653594970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2270 , TFLOPS: 70.42264619301686, Tokens per sec: 174515.07496118898, Loss: 2.60697603225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2271 , TFLOPS: 71.49740112069559, Tokens per sec: 177178.43606600584, Loss: 2.620980739593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2272 , TFLOPS: 70.4240291988992, Tokens per sec: 174518.50220211613, Loss: 2.6140871047973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2273 , TFLOPS: 70.53303741295572, Tokens per sec: 174788.63656479426, Loss: 2.6341006755828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2274 , TFLOPS: 70.4817929315218, Tokens per sec: 174661.64709475532, Loss: 2.6194441318511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2275 , TFLOPS: 71.47163479178987, Tokens per sec: 177114.58426458202, Loss: 2.6232316493988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2276 , TFLOPS: 69.73106869953543, Tokens per sec: 172801.26974879196, Loss: 2.6403114795684814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2277 , TFLOPS: 71.51589562601252, Tokens per sec: 177224.26748751916, Loss: 2.613978385925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2278 , TFLOPS: 70.53807474013185, Tokens per sec: 174801.1196164441, Loss: 2.626070499420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2279 , TFLOPS: 70.36827075270237, Tokens per sec: 174380.32663014045, Loss: 2.629319906234741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2280 , TFLOPS: 70.62311836416825, Tokens per sec: 175011.86708513484, Loss: 2.6196703910827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2281 , TFLOPS: 71.48379496186374, Tokens per sec: 177144.7185055784, Loss: 2.6116085052490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2282 , TFLOPS: 69.30361835783152, Tokens per sec: 171742.00071450736, Loss: 2.6176958084106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2283 , TFLOPS: 70.64876722805325, Tokens per sec: 175075.42779529747, Loss: 2.615060329437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2284 , TFLOPS: 71.49470944054052, Tokens per sec: 177171.7657860685, Loss: 2.6085691452026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2285 , TFLOPS: 70.50992990572372, Tokens per sec: 174731.3736163713, Loss: 2.6154041290283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2286 , TFLOPS: 69.48672071142899, Tokens per sec: 172195.74851711123, Loss: 2.607754707336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2287 , TFLOPS: 71.52901317500141, Tokens per sec: 177256.77421893092, Loss: 2.591322898864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2288 , TFLOPS: 71.51461316300191, Tokens per sec: 177221.0894029039, Loss: 2.6353957653045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2289 , TFLOPS: 69.46732434597517, Tokens per sec: 172147.68218683064, Loss: 2.6129302978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2290 , TFLOPS: 70.96310037369753, Tokens per sec: 175854.37995685847, Loss: 2.589895725250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2291 , TFLOPS: 71.51471320190421, Tokens per sec: 177221.337310324, Loss: 2.6148195266723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2292 , TFLOPS: 71.56436383670075, Tokens per sec: 177344.37705281875, Loss: 2.590266466140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2293 , TFLOPS: 69.55200109998361, Tokens per sec: 172357.52051117807, Loss: 2.618772506713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2294 , TFLOPS: 71.52072195284671, Tokens per sec: 177236.22765707807, Loss: 2.609325408935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2295 , TFLOPS: 71.52488842709005, Tokens per sec: 177246.55263922806, Loss: 2.631938934326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2296 , TFLOPS: 69.47425842358912, Tokens per sec: 172164.8655950097, Loss: 2.6247854232788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2297 , TFLOPS: 70.48948451863673, Tokens per sec: 174680.70769492438, Loss: 2.6363089084625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2298 , TFLOPS: 71.4648403955439, Tokens per sec: 177097.74700221996, Loss: 2.6167068481445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2299 , TFLOPS: 70.48026090291236, Tokens per sec: 174657.85055909425, Loss: 2.601438522338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2300 , TFLOPS: 69.56182762261916, Tokens per sec: 172381.87171674977, Loss: 2.6239864826202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2301 , TFLOPS: 71.50304047339907, Tokens per sec: 177192.41100323104, Loss: 2.6088054180145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2302 , TFLOPS: 70.38876112063522, Tokens per sec: 174431.10401907843, Loss: 2.6208436489105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2303 , TFLOPS: 70.68058076325973, Tokens per sec: 175154.26524008936, Loss: 2.574397087097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2304 , TFLOPS: 70.47850547729152, Tokens per sec: 174653.50042102972, Loss: 2.616060733795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2305 , TFLOPS: 71.54833759862136, Tokens per sec: 177304.66226942214, Loss: 2.609950065612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2306 , TFLOPS: 70.37237443828234, Tokens per sec: 174390.49601506584, Loss: 2.626782178878784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2307 , TFLOPS: 69.84153478798216, Tokens per sec: 173075.01688480715, Loss: 2.6119587421417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2308 , TFLOPS: 71.49046493851245, Tokens per sec: 177161.24744247246, Loss: 2.5965988636016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2309 , TFLOPS: 70.42391488621979, Tokens per sec: 174518.21892270393, Loss: 2.626865863800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2310 , TFLOPS: 70.70476579897654, Tokens per sec: 175214.19842279723, Loss: 2.609137773513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2311 , TFLOPS: 70.56466951995151, Tokens per sec: 174867.02441049402, Loss: 2.6012566089630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2312 , TFLOPS: 71.46561031357713, Tokens per sec: 177099.6549439198, Loss: 2.6239075660705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2313 , TFLOPS: 69.11885986621675, Tokens per sec: 171284.14881945794, Loss: 2.62766695022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2314 , TFLOPS: 71.47266797412298, Tokens per sec: 177117.1446042179, Loss: 2.6306891441345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2315 , TFLOPS: 70.46214135653749, Tokens per sec: 174612.94832714452, Loss: 2.583778142929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2316 , TFLOPS: 69.62875913160813, Tokens per sec: 172547.73536913883, Loss: 2.6070945262908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2317 , TFLOPS: 71.51386547850814, Tokens per sec: 177219.23655836406, Loss: 2.6235926151275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2318 , TFLOPS: 71.44729331503683, Tokens per sec: 177054.263403753, Loss: 2.6221671104431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2319 , TFLOPS: 69.48113165214662, Tokens per sec: 172181.89821252384, Loss: 2.58799147605896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2320 , TFLOPS: 70.59029321906543, Tokens per sec: 174930.5227595818, Loss: 2.590085744857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2321 , TFLOPS: 71.46506718642948, Tokens per sec: 177098.3090150175, Loss: 2.614051103591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2322 , TFLOPS: 70.46288800182704, Tokens per sec: 174614.7985964218, Loss: 2.6122188568115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2323 , TFLOPS: 69.84400776627184, Tokens per sec: 173081.14519743004, Loss: 2.6144096851348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2324 , TFLOPS: 71.5464631001023, Tokens per sec: 177300.01705559887, Loss: 2.590993881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2325 , TFLOPS: 71.48204344101939, Tokens per sec: 177140.37804398013, Loss: 2.5872225761413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2326 , TFLOPS: 69.40557313106271, Tokens per sec: 171994.6558738197, Loss: 2.6148083209991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2327 , TFLOPS: 70.60945776784564, Tokens per sec: 174978.0146226085, Loss: 2.6222965717315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2328 , TFLOPS: 71.53814258378112, Tokens per sec: 177279.39789957445, Loss: 2.617063283920288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2329 , TFLOPS: 70.4136235723996, Tokens per sec: 174492.71591337566, Loss: 2.645689010620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2330 , TFLOPS: 69.61092599456789, Tokens per sec: 172503.5428910709, Loss: 2.6099050045013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2331 , TFLOPS: 71.54704381963016, Tokens per sec: 177301.45614256043, Loss: 2.619389057159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2332 , TFLOPS: 71.49718461015658, Tokens per sec: 177177.89952903957, Loss: 2.6130568981170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2333 , TFLOPS: 68.60651822094158, Tokens per sec: 170014.50978337467, Loss: 2.6096351146698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2334 , TFLOPS: 71.52360577679772, Tokens per sec: 177243.3740905081, Loss: 2.6143321990966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2335 , TFLOPS: 71.57313315779129, Tokens per sec: 177366.10839650157, Loss: 2.639629602432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2336 , TFLOPS: 69.64150448116104, Tokens per sec: 172579.31974934708, Loss: 2.6092424392700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2337 , TFLOPS: 70.56297871004985, Tokens per sec: 174862.83439729962, Loss: 2.6051132678985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2338 , TFLOPS: 71.52143179043442, Tokens per sec: 177237.98671281544, Loss: 2.599233627319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2339 , TFLOPS: 70.41839074637136, Tokens per sec: 174504.5294956253, Loss: 2.5951342582702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2340 , TFLOPS: 70.6307226687501, Tokens per sec: 175030.71138957195, Loss: 2.6034390926361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2341 , TFLOPS: 70.45320773906579, Tokens per sec: 174590.80983893044, Loss: 2.6177730560302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2342 , TFLOPS: 71.4722013199527, Tokens per sec: 177115.98818377775, Loss: 2.603586196899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2343 , TFLOPS: 69.56171041617672, Tokens per sec: 172381.58126627406, Loss: 2.6328787803649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2344 , TFLOPS: 70.37577432157099, Tokens per sec: 174398.9213003832, Loss: 2.6297905445098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2345 , TFLOPS: 71.5215014909011, Tokens per sec: 177238.15943825006, Loss: 2.5979349613189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2346 , TFLOPS: 70.42926775579333, Tokens per sec: 174531.48392317444, Loss: 2.6046271324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2347 , TFLOPS: 70.55972249684444, Tokens per sec: 174854.76514227517, Loss: 2.609389543533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2348 , TFLOPS: 70.49882747010238, Tokens per sec: 174703.86055786736, Loss: 2.6358094215393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2349 , TFLOPS: 70.46152419377637, Tokens per sec: 174611.41892983636, Loss: 2.5731959342956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2350 , TFLOPS: 70.65610479995318, Tokens per sec: 175093.6111067658, Loss: 2.597034454345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2351 , TFLOPS: 71.45632907941541, Tokens per sec: 177076.65502325472, Loss: 2.6083126068115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2352 , TFLOPS: 70.56901667305571, Tokens per sec: 174877.79713476473, Loss: 2.6088390350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2353 , TFLOPS: 69.5139028720435, Tokens per sec: 172263.1089054762, Loss: 2.6002378463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2354 , TFLOPS: 71.48524425542635, Tokens per sec: 177148.31001467942, Loss: 2.6024234294891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2355 , TFLOPS: 71.5259471042937, Tokens per sec: 177249.1761579617, Loss: 2.599565267562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2356 , TFLOPS: 69.32439510602568, Tokens per sec: 171793.48778528025, Loss: 2.6107163429260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2357 , TFLOPS: 70.9882399416278, Tokens per sec: 175916.67857554174, Loss: 2.596146821975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2358 , TFLOPS: 71.5301524092124, Tokens per sec: 177259.59736680277, Loss: 2.633002758026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2359 , TFLOPS: 70.58075745082331, Tokens per sec: 174906.89207542624, Loss: 2.6382641792297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2360 , TFLOPS: 69.83605073666848, Tokens per sec: 173061.42680153373, Loss: 2.6102123260498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2361 , TFLOPS: 71.54163907972874, Tokens per sec: 177288.06260170348, Loss: 2.613365650177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2362 , TFLOPS: 71.47937634840615, Tokens per sec: 177133.76869468004, Loss: 2.615382432937622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2363 , TFLOPS: 68.59781846309592, Tokens per sec: 169992.95082507568, Loss: 2.6067278385162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2364 , TFLOPS: 71.49295359069241, Tokens per sec: 177167.4145967221, Loss: 2.6120553016662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2365 , TFLOPS: 71.52961404106168, Tokens per sec: 177258.26323121914, Loss: 2.6101033687591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2366 , TFLOPS: 70.46559024757325, Tokens per sec: 174621.49505905784, Loss: 2.600306749343872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2367 , TFLOPS: 69.59489835443088, Tokens per sec: 172463.82463322108, Loss: 2.595109462738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2368 , TFLOPS: 71.42061967854518, Tokens per sec: 176988.16319418303, Loss: 2.6260714530944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2369 , TFLOPS: 70.39658616720357, Tokens per sec: 174450.4953464748, Loss: 2.62044095993042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2370 , TFLOPS: 70.66345987296688, Tokens per sec: 175111.8377879219, Loss: 2.6110305786132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2371 , TFLOPS: 70.40275989481216, Tokens per sec: 174465.79452358113, Loss: 2.6024811267852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2372 , TFLOPS: 71.51328534572903, Tokens per sec: 177217.79892543054, Loss: 2.616187810897827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2373 , TFLOPS: 70.3385657218518, Tokens per sec: 174306.7142914134, Loss: 2.596558094024658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2374 , TFLOPS: 69.43966515101219, Tokens per sec: 172079.1396548005, Loss: 2.6214981079101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2375 , TFLOPS: 71.51627760712768, Tokens per sec: 177225.2140788012, Loss: 2.598785877227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2376 , TFLOPS: 70.41574389125326, Tokens per sec: 174497.97029706743, Loss: 2.5969767570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2377 , TFLOPS: 70.62268955792247, Tokens per sec: 175010.80445602088, Loss: 2.628896474838257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2378 , TFLOPS: 70.43821098098284, Tokens per sec: 174553.64622037165, Loss: 2.598304271697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2379 , TFLOPS: 71.52274718512224, Tokens per sec: 177241.24640575456, Loss: 2.616405487060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2380 , TFLOPS: 69.58109829080959, Tokens per sec: 172429.626555365, Loss: 2.606747627258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2381 , TFLOPS: 70.52406999979749, Tokens per sec: 174766.41432714602, Loss: 2.621805429458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2382 , TFLOPS: 71.5562898827609, Tokens per sec: 177324.36890553634, Loss: 2.591618299484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2383 , TFLOPS: 69.52023733443636, Tokens per sec: 172278.80640108435, Loss: 2.602937698364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2384 , TFLOPS: 71.56120671421442, Tokens per sec: 177336.5533555119, Loss: 2.611600637435913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2385 , TFLOPS: 70.42655330584239, Tokens per sec: 174524.75721717466, Loss: 2.595902442932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2386 , TFLOPS: 70.3304123516702, Tokens per sec: 174286.5093419356, Loss: 2.6066484451293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2387 , TFLOPS: 70.6121926362313, Tokens per sec: 174984.7919277388, Loss: 2.6081254482269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2388 , TFLOPS: 71.5625632508199, Tokens per sec: 177339.91500265512, Loss: 2.6214852333068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2389 , TFLOPS: 69.46425206875449, Tokens per sec: 172140.06874543853, Loss: 2.611175775527954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2390 , TFLOPS: 70.66129998602922, Tokens per sec: 175106.4853501592, Loss: 2.59653639793396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2391 , TFLOPS: 71.53307274449469, Tokens per sec: 177266.83427933377, Loss: 2.6351513862609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2392 , TFLOPS: 71.46724985480772, Tokens per sec: 177103.71790769958, Loss: 2.611534595489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2393 , TFLOPS: 69.52406438119372, Tokens per sec: 172288.29024453307, Loss: 2.6077115535736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2394 , TFLOPS: 70.60072691639203, Tokens per sec: 174956.3786109238, Loss: 2.601754665374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2395 , TFLOPS: 71.5262115828056, Tokens per sec: 177249.83156484886, Loss: 2.6005570888519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2396 , TFLOPS: 70.223766048743, Tokens per sec: 174022.2280552219, Loss: 2.6219522953033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2397 , TFLOPS: 70.6918748686253, Tokens per sec: 175182.25327733366, Loss: 2.617525100708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2398 , TFLOPS: 71.50648265188086, Tokens per sec: 177200.9411006962, Loss: 2.6024303436279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2399 , TFLOPS: 71.51686201011098, Tokens per sec: 177226.66229377114, Loss: 2.614048719406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2400 , TFLOPS: 68.64343241408535, Tokens per sec: 170105.987220419, Loss: 2.609595775604248 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/optimizer.pt +[rank0]:[2024-08-26 07:07:19,588] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006386797002051026, 'preprocessing_with_comm': 0.0012175400006526615, 'state_converting': 0.9650442719976127, : 0.973997937002423}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400, took 4.21s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000400 + eval ppl=10.713427543640137, eval loss=2.371497869491577 +------------------------------------------------------------------ +iteration: 2401 , TFLOPS: 67.01949131116767, Tokens per sec: 166081.68227550856, Loss: 2.6187007427215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2402 , TFLOPS: 71.46928031838114, Tokens per sec: 177108.74962010855, Loss: 2.621474504470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2403 , TFLOPS: 70.37502171958752, Tokens per sec: 174397.05626976222, Loss: 2.629476547241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2404 , TFLOPS: 69.58259532827627, Tokens per sec: 172433.33637911972, Loss: 2.620123863220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2405 , TFLOPS: 71.52691579904595, Tokens per sec: 177251.57669026827, Loss: 2.6438729763031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2406 , TFLOPS: 70.4051435364172, Tokens per sec: 174471.70145005826, Loss: 2.610138177871704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2407 , TFLOPS: 70.2921377773891, Tokens per sec: 174191.66073057457, Loss: 2.6252646446228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2408 , TFLOPS: 71.47964084933182, Tokens per sec: 177134.424157111, Loss: 2.6211466789245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2409 , TFLOPS: 71.51192105604075, Tokens per sec: 177214.4180652941, Loss: 2.587890863418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2410 , TFLOPS: 69.12439329279768, Tokens per sec: 171297.86126008272, Loss: 2.5821878910064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2411 , TFLOPS: 70.4424012665927, Tokens per sec: 174564.03020971088, Loss: 2.601710796356201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2412 , TFLOPS: 71.4870400741989, Tokens per sec: 177152.76025142355, Loss: 2.6227047443389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2413 , TFLOPS: 70.04010822685923, Tokens per sec: 173567.10374101464, Loss: 2.5790634155273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2414 , TFLOPS: 70.60943196529365, Tokens per sec: 174977.95068104228, Loss: 2.5960538387298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2415 , TFLOPS: 70.48467489940636, Tokens per sec: 174668.78892864825, Loss: 2.6206612586975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2416 , TFLOPS: 71.5184228961416, Tokens per sec: 177230.53034130082, Loss: 2.603459358215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2417 , TFLOPS: 69.43246498770299, Tokens per sec: 172061.29685695685, Loss: 2.6022183895111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2418 , TFLOPS: 71.4933657859383, Tokens per sec: 177168.43606194842, Loss: 2.6224253177642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2419 , TFLOPS: 70.42665932676361, Tokens per sec: 174525.0199486966, Loss: 2.616347551345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2420 , TFLOPS: 70.40121385951913, Tokens per sec: 174461.9632778152, Loss: 2.6101605892181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2421 , TFLOPS: 70.79039537614982, Tokens per sec: 175426.39794790925, Loss: 2.6148746013641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2422 , TFLOPS: 70.33776800508973, Tokens per sec: 174304.73746140127, Loss: 2.601921319961548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2423 , TFLOPS: 70.30288251793685, Tokens per sec: 174218.28738128277, Loss: 2.605698347091675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2424 , TFLOPS: 70.7448479200867, Tokens per sec: 175313.52633431833, Loss: 2.6094653606414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2425 , TFLOPS: 71.508591936048, Tokens per sec: 177206.1681392192, Loss: 2.593474864959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2426 , TFLOPS: 70.45448685310642, Tokens per sec: 174593.97962442916, Loss: 2.6002795696258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2427 , TFLOPS: 69.69863641275245, Tokens per sec: 172720.89896943243, Loss: 2.5945334434509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2428 , TFLOPS: 71.50429494042542, Tokens per sec: 177195.5197107133, Loss: 2.623070001602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2429 , TFLOPS: 71.508550173377, Tokens per sec: 177206.06464671993, Loss: 2.579193592071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2430 , TFLOPS: 68.57410166505439, Tokens per sec: 169934.17798690777, Loss: 2.5870466232299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2431 , TFLOPS: 71.50249642955788, Tokens per sec: 177191.0628026612, Loss: 2.6227118968963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2432 , TFLOPS: 71.54168487370198, Tokens per sec: 177288.1760842138, Loss: 2.6034998893737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2433 , TFLOPS: 70.37052668650453, Tokens per sec: 174385.91708261383, Loss: 2.6108503341674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2434 , TFLOPS: 69.47403861304919, Tokens per sec: 172164.3208802776, Loss: 2.590907335281372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2435 , TFLOPS: 71.5294644517313, Tokens per sec: 177257.89253237983, Loss: 2.601172685623169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2436 , TFLOPS: 71.54737365108937, Tokens per sec: 177302.2735012503, Loss: 2.6294784545898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2437 , TFLOPS: 68.62369693590595, Tokens per sec: 170057.08053144807, Loss: 2.593691825866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2438 , TFLOPS: 71.50064521860699, Tokens per sec: 177186.47529799715, Loss: 2.605945110321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2439 , TFLOPS: 71.46347391194043, Tokens per sec: 177094.36070531994, Loss: 2.602503776550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2440 , TFLOPS: 70.43010665648258, Tokens per sec: 174533.56281149495, Loss: 2.6188278198242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2441 , TFLOPS: 69.55259571689412, Tokens per sec: 172358.99403738475, Loss: 2.600808620452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2442 , TFLOPS: 71.55579206851752, Tokens per sec: 177323.13526700248, Loss: 2.6108503341674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2443 , TFLOPS: 71.55531976360788, Tokens per sec: 177321.96484340774, Loss: 2.5849828720092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2444 , TFLOPS: 68.98294734197171, Tokens per sec: 170947.3426123782, Loss: 2.5996928215026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2445 , TFLOPS: 71.54812711269722, Tokens per sec: 177304.14066211518, Loss: 2.601083517074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2446 , TFLOPS: 71.52048110844272, Tokens per sec: 177235.63081811377, Loss: 2.611626625061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2447 , TFLOPS: 69.53025789956565, Tokens per sec: 172303.6384653311, Loss: 2.5987353324890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2448 , TFLOPS: 70.43893136610653, Tokens per sec: 174555.4314140651, Loss: 2.616286516189575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2449 , TFLOPS: 71.55230583193341, Tokens per sec: 177314.49598870642, Loss: 2.5952656269073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2450 , TFLOPS: 70.36181246808182, Tokens per sec: 174364.32228941197, Loss: 2.596930980682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2451 , TFLOPS: 70.9374981677946, Tokens per sec: 175790.93487031525, Loss: 2.605001211166382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2452 , TFLOPS: 70.45434696782954, Tokens per sec: 174593.63297330344, Loss: 2.5977022647857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2453 , TFLOPS: 70.41290009947636, Tokens per sec: 174490.9230677748, Loss: 2.5888776779174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2454 , TFLOPS: 70.64714814138254, Tokens per sec: 175071.41552017035, Loss: 2.6109585762023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2455 , TFLOPS: 71.54586120648975, Tokens per sec: 177298.52549692293, Loss: 2.5826549530029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2456 , TFLOPS: 70.49556390203603, Tokens per sec: 174695.77307668724, Loss: 2.6020641326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2457 , TFLOPS: 69.55106202485504, Tokens per sec: 172355.19337956177, Loss: 2.5920379161834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2458 , TFLOPS: 71.55974595314154, Tokens per sec: 177332.93342865718, Loss: 2.6226422786712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2459 , TFLOPS: 70.50525879432847, Tokens per sec: 174719.79808776846, Loss: 2.610323905944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2460 , TFLOPS: 70.05296349132944, Tokens per sec: 173598.96050249608, Loss: 2.6056861877441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2461 , TFLOPS: 70.57695497233745, Tokens per sec: 174897.46911486072, Loss: 2.6051111221313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2462 , TFLOPS: 71.50332788191734, Tokens per sec: 177193.12323320002, Loss: 2.618777275085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2463 , TFLOPS: 70.57305316728352, Tokens per sec: 174887.80001211775, Loss: 2.605862617492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2464 , TFLOPS: 69.48387789736476, Tokens per sec: 172188.7037106983, Loss: 2.6215710639953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2465 , TFLOPS: 71.48897212000298, Tokens per sec: 177157.5480737585, Loss: 2.59702205657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2466 , TFLOPS: 71.51087909834892, Tokens per sec: 177211.83597935262, Loss: 2.5936994552612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2467 , TFLOPS: 68.43755903391371, Tokens per sec: 169595.81030552866, Loss: 2.60190749168396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2468 , TFLOPS: 71.48790164092449, Tokens per sec: 177154.895308679, Loss: 2.6022260189056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2469 , TFLOPS: 71.46842243231649, Tokens per sec: 177106.62368393637, Loss: 2.6141974925994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2470 , TFLOPS: 69.18259310849442, Tokens per sec: 171442.0868146182, Loss: 2.6026997566223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2471 , TFLOPS: 70.68190303292864, Tokens per sec: 175157.54196998896, Loss: 2.6300933361053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2472 , TFLOPS: 71.47785581471987, Tokens per sec: 177130.00064470485, Loss: 2.6091811656951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2473 , TFLOPS: 71.55105241633709, Tokens per sec: 177311.38988678358, Loss: 2.588992118835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2474 , TFLOPS: 68.71620489818511, Tokens per sec: 170286.32545256912, Loss: 2.5844061374664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2475 , TFLOPS: 71.49384951507598, Tokens per sec: 177169.6347960388, Loss: 2.608379602432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2476 , TFLOPS: 71.4778214508972, Tokens per sec: 177129.91548736673, Loss: 2.614854097366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2477 , TFLOPS: 69.61327952146017, Tokens per sec: 172509.37518997237, Loss: 2.620060920715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2478 , TFLOPS: 70.55408865033665, Tokens per sec: 174840.80385000943, Loss: 2.5905041694641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2479 , TFLOPS: 71.52122965595117, Tokens per sec: 177237.48580129904, Loss: 2.618001699447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2480 , TFLOPS: 70.41648046892522, Tokens per sec: 174499.7956176793, Loss: 2.598342180252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2481 , TFLOPS: 69.5565793378227, Tokens per sec: 172368.86588887783, Loss: 2.580003023147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2482 , TFLOPS: 71.52258223277867, Tokens per sec: 177240.83763567623, Loss: 2.593108892440796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2483 , TFLOPS: 71.47777852751548, Tokens per sec: 177129.80911849847, Loss: 2.629171371459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2484 , TFLOPS: 69.60814230621592, Tokens per sec: 172496.64460468615, Loss: 2.6159110069274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2485 , TFLOPS: 70.5621617414183, Tokens per sec: 174860.80985903414, Loss: 2.591761350631714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2486 , TFLOPS: 71.51652132135831, Tokens per sec: 177225.81802951207, Loss: 2.610541343688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2487 , TFLOPS: 70.41192612448103, Tokens per sec: 174488.50945044417, Loss: 2.6012868881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2488 , TFLOPS: 70.90174792794168, Tokens per sec: 175702.3418377457, Loss: 2.5896835327148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2489 , TFLOPS: 70.49084902447571, Tokens per sec: 174684.08909070614, Loss: 2.6109166145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2490 , TFLOPS: 70.39131467943656, Tokens per sec: 174437.43201908583, Loss: 2.602247953414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2491 , TFLOPS: 70.65885739059898, Tokens per sec: 175100.43232960897, Loss: 2.6103427410125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2492 , TFLOPS: 71.53468379606458, Tokens per sec: 177270.82664259596, Loss: 2.5864434242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2493 , TFLOPS: 70.49884117588307, Tokens per sec: 174703.8945223017, Loss: 2.6017134189605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2494 , TFLOPS: 69.51181061503125, Tokens per sec: 172257.9240621187, Loss: 2.5715975761413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2495 , TFLOPS: 71.51440586036423, Tokens per sec: 177220.57568413147, Loss: 2.590369939804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2496 , TFLOPS: 70.37019469001316, Tokens per sec: 174385.09435873578, Loss: 2.6228301525115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2497 , TFLOPS: 69.6285541750847, Tokens per sec: 172547.22746429595, Loss: 2.6258187294006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2498 , TFLOPS: 71.51911612881986, Tokens per sec: 177232.2482482434, Loss: 2.5972681045532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2499 , TFLOPS: 71.51779161253037, Tokens per sec: 177228.96595097054, Loss: 2.5957841873168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2500 , TFLOPS: 69.43763439773554, Tokens per sec: 172074.10722447504, Loss: 2.6046695709228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2501 , TFLOPS: 70.68071745311171, Tokens per sec: 175154.6039726002, Loss: 2.613966226577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2502 , TFLOPS: 71.49345651661623, Tokens per sec: 177168.6609025632, Loss: 2.6114978790283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2503 , TFLOPS: 71.4822923266575, Tokens per sec: 177140.9948100084, Loss: 2.591714859008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2504 , TFLOPS: 67.0180378433593, Tokens per sec: 166078.0804221667, Loss: 2.5867013931274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2505 , TFLOPS: 71.51040629470599, Tokens per sec: 177210.6643198418, Loss: 2.6014325618743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2506 , TFLOPS: 71.4887734992438, Tokens per sec: 177157.05586963735, Loss: 2.6203978061676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2507 , TFLOPS: 70.29460325266429, Tokens per sec: 174197.77044990103, Loss: 2.604987144470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2508 , TFLOPS: 69.2226083627742, Tokens per sec: 171541.24902276765, Loss: 2.597486972808838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2509 , TFLOPS: 71.49926489005142, Tokens per sec: 177183.05469178074, Loss: 2.618450164794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2510 , TFLOPS: 71.54213778944214, Tokens per sec: 177289.29845931067, Loss: 2.609802484512329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2511 , TFLOPS: 68.86942311864307, Tokens per sec: 170666.01708124427, Loss: 2.5933494567871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2512 , TFLOPS: 71.51135005371603, Tokens per sec: 177213.0030586323, Loss: 2.620121955871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2513 , TFLOPS: 71.54809218811015, Tokens per sec: 177304.05411514116, Loss: 2.6081132888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2514 , TFLOPS: 69.61523225607353, Tokens per sec: 172514.2142814557, Loss: 2.6143288612365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2515 , TFLOPS: 70.50924149708659, Tokens per sec: 174729.667663934, Loss: 2.6014461517333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2516 , TFLOPS: 71.50069219609811, Tokens per sec: 177186.59171339517, Loss: 2.5866782665252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2517 , TFLOPS: 69.52288401161844, Tokens per sec: 172285.3651586971, Loss: 2.5975565910339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2518 , TFLOPS: 70.58012840408472, Tokens per sec: 174905.3332283128, Loss: 2.619403839111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2519 , TFLOPS: 71.52690794276913, Tokens per sec: 177251.55722154886, Loss: 2.5803709030151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2520 , TFLOPS: 70.39248266177478, Tokens per sec: 174440.32640798384, Loss: 2.5953526496887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2521 , TFLOPS: 70.67563241021115, Tokens per sec: 175142.00267612888, Loss: 2.6015708446502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2522 , TFLOPS: 70.45196724036323, Tokens per sec: 174587.73574649327, Loss: 2.6066088676452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2523 , TFLOPS: 71.48119105841587, Tokens per sec: 177138.26574599094, Loss: 2.586925983428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2524 , TFLOPS: 69.57390391380522, Tokens per sec: 172411.79809662214, Loss: 2.5781731605529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2525 , TFLOPS: 71.52885922982861, Tokens per sec: 177256.39272583427, Loss: 2.6151633262634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2526 , TFLOPS: 70.52650898776494, Tokens per sec: 174772.45840800606, Loss: 2.6111738681793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2527 , TFLOPS: 70.4550341737142, Tokens per sec: 174595.33594518757, Loss: 2.596792221069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2528 , TFLOPS: 70.66552944873222, Tokens per sec: 175116.96642465098, Loss: 2.601062297821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2529 , TFLOPS: 71.51025354034324, Tokens per sec: 177210.28577770363, Loss: 2.6047275066375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2530 , TFLOPS: 70.43616742407221, Tokens per sec: 174548.5820612307, Loss: 2.59781551361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2531 , TFLOPS: 69.59479141981991, Tokens per sec: 172463.55963747526, Loss: 2.5969181060791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2532 , TFLOPS: 71.50292122108279, Tokens per sec: 177192.11548285457, Loss: 2.605360984802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2533 , TFLOPS: 70.53537272100981, Tokens per sec: 174794.42371540767, Loss: 2.584218978881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2534 , TFLOPS: 69.5273068765805, Tokens per sec: 172296.32550529184, Loss: 2.610684633255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2535 , TFLOPS: 71.52947388383305, Tokens per sec: 177257.915906167, Loss: 2.591063976287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2536 , TFLOPS: 71.5080247052942, Tokens per sec: 177204.76247892564, Loss: 2.620039224624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2537 , TFLOPS: 69.47009894308529, Tokens per sec: 172154.55794411662, Loss: 2.5937209129333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2538 , TFLOPS: 70.93010027018903, Tokens per sec: 175772.60206510295, Loss: 2.622335433959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2539 , TFLOPS: 71.50109484353376, Tokens per sec: 177187.58951809598, Loss: 2.6461188793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2540 , TFLOPS: 70.13188860923012, Tokens per sec: 173794.54563897345, Loss: 2.5977020263671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2541 , TFLOPS: 69.58713898830813, Tokens per sec: 172444.59606920416, Loss: 2.626580238342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2542 , TFLOPS: 71.54340872975516, Tokens per sec: 177292.44798941197, Loss: 2.583425998687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2543 , TFLOPS: 71.52856071528846, Tokens per sec: 177255.65297391967, Loss: 2.6023147106170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2544 , TFLOPS: 68.59781481009422, Tokens per sec: 169992.94177253506, Loss: 2.5865049362182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2545 , TFLOPS: 71.50260470074952, Tokens per sec: 177191.33111060114, Loss: 2.5940358638763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2546 , TFLOPS: 71.44930482569272, Tokens per sec: 177059.24814874446, Loss: 2.6166679859161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2547 , TFLOPS: 70.34097868100723, Tokens per sec: 174312.69387001, Loss: 2.5705018043518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2548 , TFLOPS: 69.65177429321973, Tokens per sec: 172604.7694749417, Loss: 2.5749287605285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2549 , TFLOPS: 71.54213717073378, Tokens per sec: 177289.29692608322, Loss: 2.585524797439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2550 , TFLOPS: 71.49205095037554, Tokens per sec: 177165.17775458287, Loss: 2.6128745079040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2551 , TFLOPS: 69.41449301625039, Tokens per sec: 172016.76033192122, Loss: 2.6140551567077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2552 , TFLOPS: 70.50824014987359, Tokens per sec: 174727.18621523268, Loss: 2.604818344116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2553 , TFLOPS: 71.53322837018692, Tokens per sec: 177267.21993694268, Loss: 2.597771167755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2554 , TFLOPS: 70.4012644773809, Tokens per sec: 174462.08871445266, Loss: 2.6036083698272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2555 , TFLOPS: 69.93414408088636, Tokens per sec: 173304.51291437956, Loss: 2.6118345260620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2556 , TFLOPS: 71.5029510800149, Tokens per sec: 177192.18947657757, Loss: 2.59299898147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2557 , TFLOPS: 70.35925811327704, Tokens per sec: 174357.99231682025, Loss: 2.5956151485443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2558 , TFLOPS: 70.65080989362524, Tokens per sec: 175080.48974560303, Loss: 2.5770938396453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2559 , TFLOPS: 70.25530467214291, Tokens per sec: 174100.38423827302, Loss: 2.605747699737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2560 , TFLOPS: 71.49325880742637, Tokens per sec: 177168.17095741126, Loss: 2.618089199066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2561 , TFLOPS: 69.88632309461147, Tokens per sec: 173186.0072424739, Loss: 2.595827341079712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2562 , TFLOPS: 71.50805997710054, Tokens per sec: 177204.8498863473, Loss: 2.6177425384521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2563 , TFLOPS: 70.48242918753026, Tokens per sec: 174663.2238072337, Loss: 2.5987327098846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2564 , TFLOPS: 69.48409162380837, Tokens per sec: 172189.23334836945, Loss: 2.593794345855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2565 , TFLOPS: 71.50835529749766, Tokens per sec: 177205.58172282358, Loss: 2.6019346714019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2566 , TFLOPS: 71.51626431437485, Tokens per sec: 177225.18113789536, Loss: 2.589927911758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2567 , TFLOPS: 69.44416087874482, Tokens per sec: 172090.28056336724, Loss: 2.5813164710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2568 , TFLOPS: 70.67567117726708, Tokens per sec: 175142.098745164, Loss: 2.602126121520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2569 , TFLOPS: 71.5091604797518, Tokens per sec: 177207.57705314748, Loss: 2.59852933883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2570 , TFLOPS: 70.41755048574872, Tokens per sec: 174502.44723724003, Loss: 2.589872360229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2571 , TFLOPS: 69.50027986128065, Tokens per sec: 172229.34958410732, Loss: 2.6029016971588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2572 , TFLOPS: 71.50845399571284, Tokens per sec: 177205.82630787327, Loss: 2.5879454612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2573 , TFLOPS: 71.51456630244563, Tokens per sec: 177220.9732772833, Loss: 2.5957467555999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2574 , TFLOPS: 70.32520456608239, Tokens per sec: 174273.60387556403, Loss: 2.6208415031433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2575 , TFLOPS: 70.56394756703213, Tokens per sec: 174865.23533163013, Loss: 2.6058359146118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2576 , TFLOPS: 71.52666192556225, Tokens per sec: 177250.94756380917, Loss: 2.6055352687835693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2577 , TFLOPS: 71.51772061506631, Tokens per sec: 177228.7900114336, Loss: 2.5989279747009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2578 , TFLOPS: 69.61699614918314, Tokens per sec: 172518.5854028902, Loss: 2.585498571395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2579 , TFLOPS: 71.46764216001323, Tokens per sec: 177104.6900832148, Loss: 2.604520320892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2580 , TFLOPS: 71.48024106956346, Tokens per sec: 177135.91156896474, Loss: 2.6041815280914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2581 , TFLOPS: 69.59762246630882, Tokens per sec: 172470.5752825403, Loss: 2.6159017086029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2582 , TFLOPS: 70.44254628746444, Tokens per sec: 174564.38958740648, Loss: 2.5969460010528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2583 , TFLOPS: 71.49446860716759, Tokens per sec: 177171.16897444043, Loss: 2.612123966217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2584 , TFLOPS: 69.54108211098173, Tokens per sec: 172330.46205360547, Loss: 2.5771334171295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2585 , TFLOPS: 70.54686406330666, Tokens per sec: 174822.90052749513, Loss: 2.5730764865875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2586 , TFLOPS: 71.5196056457989, Tokens per sec: 177233.46132524224, Loss: 2.573878526687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2587 , TFLOPS: 70.38499924094643, Tokens per sec: 174421.7816668043, Loss: 2.6046268939971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2588 , TFLOPS: 70.72805617912427, Tokens per sec: 175271.91455045043, Loss: 2.5859663486480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2589 , TFLOPS: 70.43669793904937, Tokens per sec: 174549.89673578515, Loss: 2.588289976119995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2590 , TFLOPS: 71.47284060828568, Tokens per sec: 177117.5724106903, Loss: 2.5958080291748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2591 , TFLOPS: 69.4215336121285, Tokens per sec: 172034.20770409284, Loss: 2.5869407653808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2592 , TFLOPS: 70.50790453337687, Tokens per sec: 174726.35452058262, Loss: 2.6252760887145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2593 , TFLOPS: 71.49410879389265, Tokens per sec: 177170.27731750844, Loss: 2.616347312927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2594 , TFLOPS: 70.34856098790071, Tokens per sec: 174331.48366175208, Loss: 2.6097705364227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2595 , TFLOPS: 70.67212952348596, Tokens per sec: 175133.32213694902, Loss: 2.605203866958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2596 , TFLOPS: 70.21901319354814, Tokens per sec: 174010.44995647875, Loss: 2.581803798675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2597 , TFLOPS: 71.49832916482656, Tokens per sec: 177180.73586159488, Loss: 2.585801362991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2598 , TFLOPS: 69.57456670960656, Tokens per sec: 172413.4405776308, Loss: 2.6032161712646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2599 , TFLOPS: 71.49942162397, Tokens per sec: 177183.44309569665, Loss: 2.600145101547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2600 , TFLOPS: 70.54506841470234, Tokens per sec: 174818.45071244662, Loss: 2.6045830249786377 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/model.pt +[rank0]:[2024-08-26 07:32:26,261] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0064538589977019, 'preprocessing_with_comm': 0.0013226340015535243, 'state_converting': 0.9785605219985882, : 0.9876916050016007}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600, took 4.29s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000600 + eval ppl=11.030041694641113, eval loss=2.400622606277466 +------------------------------------------------------------------ +iteration: 2601 , TFLOPS: 63.57223553705994, Tokens per sec: 157539.00272069717, Loss: 2.586963176727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2602 , TFLOPS: 71.46089333564882, Tokens per sec: 177087.96575299543, Loss: 2.6120078563690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2603 , TFLOPS: 69.43698035544128, Tokens per sec: 172072.48643562113, Loss: 2.5915894508361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2604 , TFLOPS: 69.45155233195418, Tokens per sec: 172108.59739865584, Loss: 2.592799663543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2605 , TFLOPS: 71.50478152156292, Tokens per sec: 177196.72551237338, Loss: 2.5928597450256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2606 , TFLOPS: 71.52930642644445, Tokens per sec: 177257.5009283111, Loss: 2.595649480819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2607 , TFLOPS: 69.74036775288667, Tokens per sec: 172824.31382737163, Loss: 2.607421398162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2608 , TFLOPS: 70.65750935321057, Tokens per sec: 175097.0917444619, Loss: 2.5973265171051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2609 , TFLOPS: 71.51433569072769, Tokens per sec: 177220.4017960423, Loss: 2.5943305492401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2610 , TFLOPS: 71.48274133474146, Tokens per sec: 177142.1075015028, Loss: 2.599555730819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2611 , TFLOPS: 68.50871942230377, Tokens per sec: 169772.15358692314, Loss: 2.5812430381774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2612 , TFLOPS: 71.52281020192942, Tokens per sec: 177241.40256834464, Loss: 2.619706869125366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2613 , TFLOPS: 71.50273611957336, Tokens per sec: 177191.65678092357, Loss: 2.5879580974578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2614 , TFLOPS: 70.42401548470465, Tokens per sec: 174518.46821683127, Loss: 2.589198350906372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2615 , TFLOPS: 69.75326193472145, Tokens per sec: 172856.2670303696, Loss: 2.6109957695007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2616 , TFLOPS: 71.48523670757717, Tokens per sec: 177148.29131027768, Loss: 2.601076126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2617 , TFLOPS: 70.32945450284097, Tokens per sec: 174284.135687021, Loss: 2.5866470336914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2618 , TFLOPS: 69.55327144732458, Tokens per sec: 172360.66857182927, Loss: 2.6117441654205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2619 , TFLOPS: 71.39942195558338, Tokens per sec: 176935.63290156174, Loss: 2.585188388824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2620 , TFLOPS: 71.47822973701663, Tokens per sec: 177130.92726534713, Loss: 2.5844833850860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2621 , TFLOPS: 69.77031410652894, Tokens per sec: 172898.5241905604, Loss: 2.5825486183166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2622 , TFLOPS: 70.4966835399515, Tokens per sec: 174698.5476627801, Loss: 2.6024169921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2623 , TFLOPS: 71.52762457097553, Tokens per sec: 177253.33310518743, Loss: 2.582982063293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2624 , TFLOPS: 69.58569880749965, Tokens per sec: 172441.02714251174, Loss: 2.6050362586975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2625 , TFLOPS: 71.55587611056885, Tokens per sec: 177323.3435324637, Loss: 2.613278388977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2626 , TFLOPS: 70.38944622414029, Tokens per sec: 174432.80178103433, Loss: 2.6114959716796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2627 , TFLOPS: 70.35275205640058, Tokens per sec: 174341.8695911791, Loss: 2.5822811126708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2628 , TFLOPS: 70.5347466066771, Tokens per sec: 174792.87213511905, Loss: 2.57076358795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2629 , TFLOPS: 71.47343358376565, Tokens per sec: 177119.04186925132, Loss: 2.5889382362365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2630 , TFLOPS: 70.39558401112262, Tokens per sec: 174448.0118933096, Loss: 2.592311382293701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2631 , TFLOPS: 69.50246796547705, Tokens per sec: 172234.7719473456, Loss: 2.575493574142456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2632 , TFLOPS: 71.55134665112648, Tokens per sec: 177312.11903300436, Loss: 2.579216480255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2633 , TFLOPS: 70.37780426245074, Tokens per sec: 174403.95171749967, Loss: 2.5834476947784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2634 , TFLOPS: 70.35042735737487, Tokens per sec: 174336.10873090665, Loss: 2.580703020095825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2635 , TFLOPS: 70.53902117524339, Tokens per sec: 174803.4649869089, Loss: 2.5893807411193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2636 , TFLOPS: 71.47968398725129, Tokens per sec: 177134.53105762752, Loss: 2.599522829055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2637 , TFLOPS: 70.46646572856747, Tokens per sec: 174623.66459740372, Loss: 2.616666793823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2638 , TFLOPS: 69.44965758134266, Tokens per sec: 172103.90199791652, Loss: 2.5948667526245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2639 , TFLOPS: 71.54976678723553, Tokens per sec: 177308.2039562462, Loss: 2.620413064956665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2640 , TFLOPS: 70.6002212863038, Tokens per sec: 174955.1256038654, Loss: 2.575432300567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2641 , TFLOPS: 69.57680160009244, Tokens per sec: 172418.97888244834, Loss: 2.6224467754364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2642 , TFLOPS: 71.51989710038802, Tokens per sec: 177234.18358182043, Loss: 2.5746378898620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2643 , TFLOPS: 71.51553770291451, Tokens per sec: 177223.3805146538, Loss: 2.5946571826934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2644 , TFLOPS: 69.25437172965914, Tokens per sec: 171619.96214493227, Loss: 2.5952343940734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2645 , TFLOPS: 71.48265571753215, Tokens per sec: 177141.8953326266, Loss: 2.583118438720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2646 , TFLOPS: 71.4419087246674, Tokens per sec: 177040.9197956559, Loss: 2.586711883544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2647 , TFLOPS: 70.34555063951647, Tokens per sec: 174324.02368683871, Loss: 2.578845739364624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2648 , TFLOPS: 69.76359567021692, Tokens per sec: 172881.8751652823, Loss: 2.5899035930633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2649 , TFLOPS: 71.55459343176294, Tokens per sec: 177320.1649130839, Loss: 2.5831565856933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2650 , TFLOPS: 71.45281411836574, Tokens per sec: 177067.9445625687, Loss: 2.599295139312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2651 , TFLOPS: 69.46658038446407, Tokens per sec: 172145.838568252, Loss: 2.5951128005981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2652 , TFLOPS: 70.50624167220303, Tokens per sec: 174722.23376741455, Loss: 2.594447612762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2653 , TFLOPS: 71.5189670567908, Tokens per sec: 177231.87883133377, Loss: 2.6127843856811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2654 , TFLOPS: 70.38128405796516, Tokens per sec: 174412.57503411555, Loss: 2.5964505672454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2655 , TFLOPS: 69.74847703577956, Tokens per sec: 172844.40952369143, Loss: 2.6249494552612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2656 , TFLOPS: 71.46696177152909, Tokens per sec: 177103.00400560015, Loss: 2.6101131439208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2657 , TFLOPS: 71.5018455152704, Tokens per sec: 177189.44976535274, Loss: 2.5911731719970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2658 , TFLOPS: 69.47676153172921, Tokens per sec: 172171.06857272098, Loss: 2.605236053466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2659 , TFLOPS: 70.46609715864624, Tokens per sec: 174622.7512405376, Loss: 2.5802581310272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2660 , TFLOPS: 71.49423267026879, Tokens per sec: 177170.58429681457, Loss: 2.607139825820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2661 , TFLOPS: 69.66449152329169, Tokens per sec: 172636.28417197854, Loss: 2.593590021133423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2662 , TFLOPS: 71.51725222283251, Tokens per sec: 177227.62928388055, Loss: 2.5908396244049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2663 , TFLOPS: 70.54854446523454, Tokens per sec: 174827.064748584, Loss: 2.5945518016815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2664 , TFLOPS: 70.35872774495589, Tokens per sec: 174356.67800569555, Loss: 2.5903968811035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2665 , TFLOPS: 70.85201068288568, Tokens per sec: 175579.08746548704, Loss: 2.59366512298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2666 , TFLOPS: 71.55102633680589, Tokens per sec: 177311.3252588324, Loss: 2.5823729038238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2667 , TFLOPS: 69.36616205504406, Tokens per sec: 171896.99088595703, Loss: 2.5910449028015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2668 , TFLOPS: 70.31034785160197, Tokens per sec: 174236.78730047937, Loss: 2.6061251163482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2669 , TFLOPS: 71.45149527747759, Tokens per sec: 177064.67632956555, Loss: 2.6037425994873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2670 , TFLOPS: 70.34774184471895, Tokens per sec: 174329.45373471163, Loss: 2.5949690341949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2671 , TFLOPS: 69.4405005792767, Tokens per sec: 172081.20993807, Loss: 2.5914080142974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2672 , TFLOPS: 71.52374923494918, Tokens per sec: 177243.7295956106, Loss: 2.5984225273132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2673 , TFLOPS: 71.46235921763937, Tokens per sec: 177091.59837004828, Loss: 2.5992467403411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2674 , TFLOPS: 69.43526967488843, Tokens per sec: 172068.24718076419, Loss: 2.602841377258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2675 , TFLOPS: 70.7482191202182, Tokens per sec: 175321.88053961252, Loss: 2.5967888832092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2676 , TFLOPS: 71.54558307593419, Tokens per sec: 177297.8362587676, Loss: 2.5877745151519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2677 , TFLOPS: 70.53725780299249, Tokens per sec: 174799.0951562197, Loss: 2.608980178833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2678 , TFLOPS: 69.44200317666579, Tokens per sec: 172084.93353992494, Loss: 2.5931248664855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2679 , TFLOPS: 71.46666510468846, Tokens per sec: 177102.2688324885, Loss: 2.610750198364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2680 , TFLOPS: 71.51609109366768, Tokens per sec: 177224.75187790114, Loss: 2.5871315002441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2681 , TFLOPS: 69.39874478011731, Tokens per sec: 171977.73446797204, Loss: 2.5894339084625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2682 , TFLOPS: 70.86185204184982, Tokens per sec: 175603.47543711535, Loss: 2.587153434753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2683 , TFLOPS: 71.55483463170957, Tokens per sec: 177320.762633122, Loss: 2.6129097938537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2684 , TFLOPS: 70.35418524316741, Tokens per sec: 174345.42118586585, Loss: 2.5873029232025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2685 , TFLOPS: 69.75117547967886, Tokens per sec: 172851.0965649321, Loss: 2.604897975921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2686 , TFLOPS: 71.4935353437996, Tokens per sec: 177168.8562450071, Loss: 2.573338508605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2687 , TFLOPS: 71.53025894734053, Tokens per sec: 177259.86138002042, Loss: 2.6013340950012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2688 , TFLOPS: 69.76625395957136, Tokens per sec: 172888.46269913737, Loss: 2.5886270999908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2689 , TFLOPS: 70.43875705536358, Tokens per sec: 174554.99945284214, Loss: 2.5902957916259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2690 , TFLOPS: 71.5421767492035, Tokens per sec: 177289.39500589113, Loss: 2.607988119125366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2691 , TFLOPS: 69.53485075498959, Tokens per sec: 172315.02006701642, Loss: 2.593698501586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2692 , TFLOPS: 70.53163439302838, Tokens per sec: 174785.1597268589, Loss: 2.593787670135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2693 , TFLOPS: 71.49767754428959, Tokens per sec: 177179.12107412235, Loss: 2.62715744972229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2694 , TFLOPS: 70.39103897661454, Tokens per sec: 174436.7487971217, Loss: 2.6092658042907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2695 , TFLOPS: 70.64075472563745, Tokens per sec: 175055.5719316611, Loss: 2.6146907806396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2696 , TFLOPS: 70.42292796995378, Tokens per sec: 174515.77323547893, Loss: 2.5896613597869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2697 , TFLOPS: 71.49128219053972, Tokens per sec: 177163.2726830238, Loss: 2.614570140838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2698 , TFLOPS: 69.51635372474806, Tokens per sec: 172269.1823884618, Loss: 2.589165687561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2699 , TFLOPS: 71.497899764236, Tokens per sec: 177179.67175962913, Loss: 2.5889158248901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2700 , TFLOPS: 70.49791190844739, Tokens per sec: 174701.59169522885, Loss: 2.5964694023132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2701 , TFLOPS: 70.37268800938887, Tokens per sec: 174391.2730788106, Loss: 2.5808520317077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2702 , TFLOPS: 70.68968689225981, Tokens per sec: 175176.8312308745, Loss: 2.6015849113464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2703 , TFLOPS: 71.50668703057083, Tokens per sec: 177201.44757360412, Loss: 2.576838254928589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2704 , TFLOPS: 70.438899259083, Tokens per sec: 174555.35184932392, Loss: 2.5636072158813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2705 , TFLOPS: 69.74452326915362, Tokens per sec: 172834.61165444905, Loss: 2.5782055854797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2706 , TFLOPS: 71.53793786094651, Tokens per sec: 177278.89057383823, Loss: 2.58640456199646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2707 , TFLOPS: 70.46774671219849, Tokens per sec: 174626.83901595336, Loss: 2.593268871307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2708 , TFLOPS: 69.61440030712157, Tokens per sec: 172512.152620306, Loss: 2.6249048709869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2709 , TFLOPS: 71.51137880888915, Tokens per sec: 177213.07431711894, Loss: 2.5925869941711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2710 , TFLOPS: 71.51602840749305, Tokens per sec: 177224.59653465502, Loss: 2.6010851860046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2711 , TFLOPS: 69.01585360790097, Tokens per sec: 171028.88796427535, Loss: 2.5789835453033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2712 , TFLOPS: 70.84691140200208, Tokens per sec: 175566.45088572532, Loss: 2.622771739959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2713 , TFLOPS: 71.4998665309846, Tokens per sec: 177184.5456242894, Loss: 2.582824468612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2714 , TFLOPS: 70.30051952700431, Tokens per sec: 174212.43162944703, Loss: 2.5927586555480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2715 , TFLOPS: 70.70679386623335, Tokens per sec: 175219.22419686968, Loss: 2.6056156158447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2716 , TFLOPS: 71.54717195008384, Tokens per sec: 177301.7736639394, Loss: 2.5935680866241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2717 , TFLOPS: 71.50845846749831, Tokens per sec: 177205.83738945032, Loss: 2.613469362258911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2718 , TFLOPS: 68.5490286215904, Tokens per sec: 169872.04422318024, Loss: 2.598951816558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2719 , TFLOPS: 71.53400173420975, Tokens per sec: 177269.13641818438, Loss: 2.60025691986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2720 , TFLOPS: 71.50903931396171, Tokens per sec: 177207.27679097213, Loss: 2.6174285411834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2721 , TFLOPS: 70.42652390823667, Tokens per sec: 174524.68436666927, Loss: 2.5964207649230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2722 , TFLOPS: 69.61328991028198, Tokens per sec: 172509.4009346172, Loss: 2.6149070262908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2723 , TFLOPS: 71.49812916923672, Tokens per sec: 177180.24025049194, Loss: 2.587468147277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2724 , TFLOPS: 71.51279604078839, Tokens per sec: 177216.58637388624, Loss: 2.5906925201416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2725 , TFLOPS: 69.53490040866691, Tokens per sec: 172315.1431142986, Loss: 2.5677695274353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2726 , TFLOPS: 70.53005193052954, Tokens per sec: 174781.2382104641, Loss: 2.604982852935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2727 , TFLOPS: 71.52851275412463, Tokens per sec: 177255.53412087235, Loss: 2.5858447551727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2728 , TFLOPS: 69.55635614580018, Tokens per sec: 172368.31279445943, Loss: 2.5847437381744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2729 , TFLOPS: 70.3289622426927, Tokens per sec: 174282.91581214662, Loss: 2.5727717876434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2730 , TFLOPS: 71.48954961878613, Tokens per sec: 177158.97917935989, Loss: 2.5855000019073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2731 , TFLOPS: 70.34577832678224, Tokens per sec: 174324.5879209651, Loss: 2.588653326034546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2732 , TFLOPS: 70.60890543691754, Tokens per sec: 174976.6458857244, Loss: 2.5879359245300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2733 , TFLOPS: 70.51264893670312, Tokens per sec: 174738.11167466422, Loss: 2.596606969833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2734 , TFLOPS: 71.49895031393302, Tokens per sec: 177182.2751375059, Loss: 2.602444648742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2735 , TFLOPS: 69.501696591146, Tokens per sec: 172232.86039677932, Loss: 2.596284866333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2736 , TFLOPS: 71.47003560934247, Tokens per sec: 177110.62131431288, Loss: 2.582156181335449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2737 , TFLOPS: 70.36848869179684, Tokens per sec: 174380.86670722452, Loss: 2.5931191444396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2738 , TFLOPS: 69.43485481680614, Tokens per sec: 172067.21911673585, Loss: 2.563598871231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2739 , TFLOPS: 71.4748344714706, Tokens per sec: 177122.5134233047, Loss: 2.583347797393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2740 , TFLOPS: 71.52528071792504, Tokens per sec: 177247.5247791316, Loss: 2.591263771057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2741 , TFLOPS: 69.4599547234376, Tokens per sec: 172129.41945035162, Loss: 2.5881001949310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2742 , TFLOPS: 70.65369488561221, Tokens per sec: 175087.6390735549, Loss: 2.602954626083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2743 , TFLOPS: 71.46144434737137, Tokens per sec: 177089.33122074287, Loss: 2.5682215690612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2744 , TFLOPS: 70.47402719678237, Tokens per sec: 174642.40274860474, Loss: 2.567030191421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2745 , TFLOPS: 69.62251092732548, Tokens per sec: 172532.2516306299, Loss: 2.5788865089416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2746 , TFLOPS: 71.51899219516007, Tokens per sec: 177231.941126982, Loss: 2.584533214569092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2747 , TFLOPS: 71.53542877462787, Tokens per sec: 177272.67278154194, Loss: 2.577047109603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2748 , TFLOPS: 68.74330622146086, Tokens per sec: 170353.4855171036, Loss: 2.585195541381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2749 , TFLOPS: 70.81637642984799, Tokens per sec: 175490.78186102264, Loss: 2.5718719959259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2750 , TFLOPS: 71.4621818690578, Tokens per sec: 177091.15888072655, Loss: 2.56636118888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2751 , TFLOPS: 70.40023498422879, Tokens per sec: 174459.5375170137, Loss: 2.5885260105133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2752 , TFLOPS: 70.65256301371625, Tokens per sec: 175084.8341703107, Loss: 2.5915799140930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2753 , TFLOPS: 71.52611081599645, Tokens per sec: 177249.5818535954, Loss: 2.5981028079986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2754 , TFLOPS: 71.56732077758167, Tokens per sec: 177351.7046780556, Loss: 2.586660861968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2755 , TFLOPS: 68.46104262486533, Tokens per sec: 169654.00523083785, Loss: 2.6212527751922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2756 , TFLOPS: 71.47769405122023, Tokens per sec: 177129.59977693297, Loss: 2.6047439575195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2757 , TFLOPS: 71.48798345791067, Tokens per sec: 177155.0980601836, Loss: 2.6143760681152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2758 , TFLOPS: 69.60470408719002, Tokens per sec: 172488.12431918885, Loss: 2.5724449157714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2759 , TFLOPS: 70.54482475313695, Tokens per sec: 174817.84689224605, Loss: 2.588172674179077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2760 , TFLOPS: 71.47954428619722, Tokens per sec: 177134.1848630262, Loss: 2.5837252140045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2761 , TFLOPS: 70.37031150340702, Tokens per sec: 174385.38383519382, Loss: 2.6059412956237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2762 , TFLOPS: 70.55367830154991, Tokens per sec: 174839.7869605122, Loss: 2.605381727218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2763 , TFLOPS: 70.36909123455948, Tokens per sec: 174382.35987456588, Loss: 2.5860447883605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2764 , TFLOPS: 71.54746075773107, Tokens per sec: 177302.48936110403, Loss: 2.5901432037353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2765 , TFLOPS: 69.62138598040295, Tokens per sec: 172529.4638882327, Loss: 2.5886447429656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2766 , TFLOPS: 70.58458413789683, Tokens per sec: 174916.3750275388, Loss: 2.5758979320526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2767 , TFLOPS: 71.51290529946992, Tokens per sec: 177216.85712893493, Loss: 2.5836610794067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2768 , TFLOPS: 70.39922944791417, Tokens per sec: 174457.04568725708, Loss: 2.578333616256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2769 , TFLOPS: 70.53348972245179, Tokens per sec: 174789.7574375515, Loss: 2.610534191131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2770 , TFLOPS: 70.50315698485112, Tokens per sec: 174714.58957235012, Loss: 2.593322277069092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2771 , TFLOPS: 71.47393718069475, Tokens per sec: 177120.2898379174, Loss: 2.571566104888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2772 , TFLOPS: 69.90920962311736, Tokens per sec: 173242.7225812707, Loss: 2.5729994773864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2773 , TFLOPS: 71.45056758097138, Tokens per sec: 177062.37739542895, Loss: 2.58223819732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2774 , TFLOPS: 70.46174685577058, Tokens per sec: 174611.9707107863, Loss: 2.5882883071899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2775 , TFLOPS: 69.48659658972787, Tokens per sec: 172195.44092986273, Loss: 2.5904476642608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2776 , TFLOPS: 71.40252143681795, Tokens per sec: 176943.31375749706, Loss: 2.572208881378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2777 , TFLOPS: 71.52648928255307, Tokens per sec: 177250.51973541421, Loss: 2.581232786178589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2778 , TFLOPS: 68.66496686820142, Tokens per sec: 170159.35196993456, Loss: 2.600599527359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2779 , TFLOPS: 71.44196973852995, Tokens per sec: 177041.07099472856, Loss: 2.5698418617248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2780 , TFLOPS: 71.51987993206713, Tokens per sec: 177234.14103683, Loss: 2.5918118953704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2781 , TFLOPS: 69.51980686378333, Tokens per sec: 172277.73964738584, Loss: 2.6192517280578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2782 , TFLOPS: 70.55638315260714, Tokens per sec: 174846.489879396, Loss: 2.603010654449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2783 , TFLOPS: 71.5189157754271, Tokens per sec: 177231.7517504654, Loss: 2.5788705348968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2784 , TFLOPS: 71.3368931376717, Tokens per sec: 176780.67960265867, Loss: 2.6133530139923096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2785 , TFLOPS: 69.53176084996913, Tokens per sec: 172307.36294199456, Loss: 2.572864055633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2786 , TFLOPS: 71.4969715768054, Tokens per sec: 177177.3716089278, Loss: 2.5904526710510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2787 , TFLOPS: 71.49202449871163, Tokens per sec: 177165.1122044458, Loss: 2.575679063796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2788 , TFLOPS: 69.1347556803463, Tokens per sec: 171323.54039793945, Loss: 2.5854368209838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2789 , TFLOPS: 70.60441498601351, Tokens per sec: 174965.5180537202, Loss: 2.5787508487701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2790 , TFLOPS: 71.46664393910675, Tokens per sec: 177102.2163818454, Loss: 2.58655047416687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2791 , TFLOPS: 71.51677266911258, Tokens per sec: 177226.44089693538, Loss: 2.587584972381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2792 , TFLOPS: 68.56689420001015, Tokens per sec: 169916.31709456033, Loss: 2.5889389514923096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2793 , TFLOPS: 71.54846531792762, Tokens per sec: 177304.9787719325, Loss: 2.5736896991729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2794 , TFLOPS: 71.50255089342909, Tokens per sec: 177191.19777013376, Loss: 2.594216823577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2795 , TFLOPS: 69.46249417566064, Tokens per sec: 172135.71249270404, Loss: 2.5802927017211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2796 , TFLOPS: 70.50331598226073, Tokens per sec: 174714.9835854462, Loss: 2.5796093940734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2797 , TFLOPS: 71.52977373959766, Tokens per sec: 177258.65898178352, Loss: 2.5644755363464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2798 , TFLOPS: 70.41410881471295, Tokens per sec: 174493.91839728205, Loss: 2.594783067703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2799 , TFLOPS: 70.95901974790213, Tokens per sec: 175844.2677166195, Loss: 2.610579490661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2800 , TFLOPS: 70.41965809662686, Tokens per sec: 174507.67012916857, Loss: 2.5806708335876465 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/optimizer.pt +[rank0]:[2024-08-26 07:57:32,935] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006340168998576701, 'preprocessing_with_comm': 0.0012224630008859094, 'state_converting': 0.9887664560010307, : 0.9976871910002956}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800, took 4.33s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0000800 + eval ppl=11.967317581176758, eval loss=2.4821794033050537 +------------------------------------------------------------------ +iteration: 2801 , TFLOPS: 66.26697461402482, Tokens per sec: 164216.86300343112, Loss: 2.600888967514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2802 , TFLOPS: 70.55546000248668, Tokens per sec: 174844.20221170413, Loss: 2.5802059173583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2803 , TFLOPS: 70.11428888314823, Tokens per sec: 173750.9315789727, Loss: 2.5924794673919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2804 , TFLOPS: 71.47515807954092, Tokens per sec: 177123.31535975132, Loss: 2.582775115966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2805 , TFLOPS: 68.87395035306035, Tokens per sec: 170677.23606684632, Loss: 2.6049816608428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2806 , TFLOPS: 71.53615996595904, Tokens per sec: 177274.48475420795, Loss: 2.553579330444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2807 , TFLOPS: 70.44143325142596, Tokens per sec: 174561.63136149207, Loss: 2.590916872024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2808 , TFLOPS: 69.56461099892097, Tokens per sec: 172388.76922983988, Loss: 2.58439040184021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2809 , TFLOPS: 70.91392381240985, Tokens per sec: 175732.5150207422, Loss: 2.5924055576324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2810 , TFLOPS: 71.50229818749506, Tokens per sec: 177190.57153699137, Loss: 2.5926246643066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2811 , TFLOPS: 69.54852403215929, Tokens per sec: 172348.90395407876, Loss: 2.596137762069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2812 , TFLOPS: 69.53569537040391, Tokens per sec: 172317.11311705518, Loss: 2.6112585067749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2813 , TFLOPS: 71.53857347318902, Tokens per sec: 177280.46569099362, Loss: 2.6053779125213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2814 , TFLOPS: 70.36070111225123, Tokens per sec: 174361.5682272378, Loss: 2.60508131980896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2815 , TFLOPS: 69.64067070417875, Tokens per sec: 172577.25355813684, Loss: 2.581569194793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2816 , TFLOPS: 71.53022899791762, Tokens per sec: 177259.78716205125, Loss: 2.595442056655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2817 , TFLOPS: 71.45345539359191, Tokens per sec: 177069.53371322146, Loss: 2.598809003829956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2818 , TFLOPS: 68.53205325094166, Tokens per sec: 169829.97738472125, Loss: 2.5991580486297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2819 , TFLOPS: 71.57471147021226, Tokens per sec: 177370.01962854623, Loss: 2.605325222015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2820 , TFLOPS: 71.46727864201603, Tokens per sec: 177103.78924557293, Loss: 2.577564239501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2821 , TFLOPS: 70.43898334083481, Tokens per sec: 174555.56021316728, Loss: 2.589750289916992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2822 , TFLOPS: 69.5247042612145, Tokens per sec: 172289.87593771316, Loss: 2.5994608402252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2823 , TFLOPS: 71.52597104934675, Tokens per sec: 177249.23549644087, Loss: 2.584468364715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2824 , TFLOPS: 71.51737685569225, Tokens per sec: 177227.93813783643, Loss: 2.5867037773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2825 , TFLOPS: 69.48292742940895, Tokens per sec: 172186.3483464011, Loss: 2.5663681030273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2826 , TFLOPS: 70.51937849178113, Tokens per sec: 174754.78825346922, Loss: 2.6002233028411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2827 , TFLOPS: 71.53370096339434, Tokens per sec: 177268.39107497112, Loss: 2.596309185028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2828 , TFLOPS: 69.62771754901082, Tokens per sec: 172545.15421272293, Loss: 2.569847583770752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2829 , TFLOPS: 69.47270924081683, Tokens per sec: 172161.0265494434, Loss: 2.5893588066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2830 , TFLOPS: 71.48007766428432, Tokens per sec: 177135.5066326825, Loss: 2.587369918823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2831 , TFLOPS: 70.59873122271789, Tokens per sec: 174951.43306215544, Loss: 2.587869167327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2832 , TFLOPS: 69.66013075361244, Tokens per sec: 172625.47770433495, Loss: 2.5793070793151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2833 , TFLOPS: 70.49706009914308, Tokens per sec: 174699.48081793822, Loss: 2.587437152862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2834 , TFLOPS: 71.50323608322985, Tokens per sec: 177192.89574593995, Loss: 2.582277774810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2835 , TFLOPS: 69.46896082220594, Tokens per sec: 172151.7375552032, Loss: 2.6055076122283936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2836 , TFLOPS: 71.55076093020045, Tokens per sec: 177310.66755202718, Loss: 2.580638885498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2837 , TFLOPS: 70.470907854065, Tokens per sec: 174634.67267372715, Loss: 2.5818300247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2838 , TFLOPS: 69.67883696235045, Tokens per sec: 172671.83375024673, Loss: 2.595771074295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2839 , TFLOPS: 71.49244574833207, Tokens per sec: 177166.15610740974, Loss: 2.61045503616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2840 , TFLOPS: 71.44974711325256, Tokens per sec: 177060.34418603976, Loss: 2.59065580368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2841 , TFLOPS: 70.42256714299091, Tokens per sec: 174514.87906651662, Loss: 2.582843780517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2842 , TFLOPS: 69.77157960533626, Tokens per sec: 172901.66023601108, Loss: 2.6190221309661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2843 , TFLOPS: 71.51800645097623, Tokens per sec: 177229.4983443058, Loss: 2.5990376472473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2844 , TFLOPS: 69.88557016838378, Tokens per sec: 173184.14140834, Loss: 2.594940662384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2845 , TFLOPS: 69.6550188516214, Tokens per sec: 172612.8098480787, Loss: 2.5649373531341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2846 , TFLOPS: 71.5455276086293, Tokens per sec: 177297.69880467586, Loss: 2.587444543838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2847 , TFLOPS: 71.53023534730653, Tokens per sec: 177259.8028965364, Loss: 2.5894179344177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2848 , TFLOPS: 68.52341414783577, Tokens per sec: 169808.56873554905, Loss: 2.569099187850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2849 , TFLOPS: 70.59959847316526, Tokens per sec: 174953.58220429954, Loss: 2.5818967819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2850 , TFLOPS: 71.42318654569858, Tokens per sec: 176994.52417375293, Loss: 2.589481830596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2851 , TFLOPS: 70.4226819665179, Tokens per sec: 174515.16361186534, Loss: 2.575958728790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2852 , TFLOPS: 69.62456620643341, Tokens per sec: 172537.34483866705, Loss: 2.5800812244415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2853 , TFLOPS: 71.52346183613454, Tokens per sec: 177243.01738968835, Loss: 2.5964884757995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2854 , TFLOPS: 71.47732301983416, Tokens per sec: 177128.6803202858, Loss: 2.5600802898406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2855 , TFLOPS: 68.35286503426055, Tokens per sec: 169385.92924457387, Loss: 2.595188856124878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2856 , TFLOPS: 71.55491836204718, Tokens per sec: 177320.9701261223, Loss: 2.6101176738739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2857 , TFLOPS: 71.49501634249721, Tokens per sec: 177172.52632292526, Loss: 2.573495864868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2858 , TFLOPS: 70.71329844911051, Tokens per sec: 175235.3432697782, Loss: 2.6046435832977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2859 , TFLOPS: 69.67048807899691, Tokens per sec: 172651.14429759124, Loss: 2.585644006729126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2860 , TFLOPS: 71.51509328670815, Tokens per sec: 177222.27920233796, Loss: 2.585491895675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2861 , TFLOPS: 71.47217551981481, Tokens per sec: 177115.92424819394, Loss: 2.5884480476379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2862 , TFLOPS: 69.46703978262023, Tokens per sec: 172146.97700748986, Loss: 2.5825815200805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2863 , TFLOPS: 70.52540080310288, Tokens per sec: 174769.7122043369, Loss: 2.5692853927612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2864 , TFLOPS: 71.47132136567386, Tokens per sec: 177113.8075601398, Loss: 2.5835676193237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2865 , TFLOPS: 69.58505506968932, Tokens per sec: 172439.43188930416, Loss: 2.5802483558654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2866 , TFLOPS: 70.49135909585621, Tokens per sec: 174685.353103776, Loss: 2.591715097427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2867 , TFLOPS: 71.5489424250273, Tokens per sec: 177306.16109588323, Loss: 2.5805611610412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2868 , TFLOPS: 69.43823319899907, Tokens per sec: 172075.59111996947, Loss: 2.5966410636901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2869 , TFLOPS: 70.56782396634226, Tokens per sec: 174874.84147614127, Loss: 2.572556972503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2870 , TFLOPS: 70.5209609356677, Tokens per sec: 174758.70972374073, Loss: 2.597506284713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2871 , TFLOPS: 70.58909177439068, Tokens per sec: 174927.54544732775, Loss: 2.5839407444000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2872 , TFLOPS: 69.68416204624658, Tokens per sec: 172685.0298947478, Loss: 2.5881905555725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2873 , TFLOPS: 71.50808438339577, Tokens per sec: 177204.91036783546, Loss: 2.5928051471710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2874 , TFLOPS: 70.4248966028232, Tokens per sec: 174520.6517245922, Loss: 2.59138822555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2875 , TFLOPS: 70.43643032378965, Tokens per sec: 174549.2335556913, Loss: 2.5997915267944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2876 , TFLOPS: 70.88112014277762, Tokens per sec: 175651.22391377098, Loss: 2.5665717124938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2877 , TFLOPS: 70.52629994277096, Tokens per sec: 174771.94037148278, Loss: 2.5962154865264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2878 , TFLOPS: 70.73287187482116, Tokens per sec: 175283.84837488004, Loss: 2.572115421295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2879 , TFLOPS: 70.50174065786466, Tokens per sec: 174711.0797580564, Loss: 2.5857298374176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2880 , TFLOPS: 71.51401411381939, Tokens per sec: 177219.6048930389, Loss: 2.5954291820526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2881 , TFLOPS: 70.43836774573808, Tokens per sec: 174554.0347007042, Loss: 2.595407724380493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2882 , TFLOPS: 69.52561981762916, Tokens per sec: 172292.14478736557, Loss: 2.5624938011169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2883 , TFLOPS: 71.55942790424493, Tokens per sec: 177332.14526845538, Loss: 2.612795829772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2884 , TFLOPS: 71.56927576984022, Tokens per sec: 177356.54936423243, Loss: 2.6133859157562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2885 , TFLOPS: 69.27194574737484, Tokens per sec: 171663.51249676966, Loss: 2.6021201610565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2886 , TFLOPS: 71.49402255082158, Tokens per sec: 177170.06359767786, Loss: 2.569072961807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2887 , TFLOPS: 71.54757486978399, Tokens per sec: 177302.77214334163, Loss: 2.6001105308532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2888 , TFLOPS: 69.44940961221158, Tokens per sec: 172103.28750309366, Loss: 2.575317859649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2889 , TFLOPS: 69.942396047782, Tokens per sec: 173324.96219737615, Loss: 2.5783936977386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2890 , TFLOPS: 71.51805727701732, Tokens per sec: 177229.62429683455, Loss: 2.568289279937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2891 , TFLOPS: 71.52250231731219, Tokens per sec: 177240.63959634685, Loss: 2.593269109725952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2892 , TFLOPS: 67.53557183659436, Tokens per sec: 167360.58666847894, Loss: 2.5695900917053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2893 , TFLOPS: 71.5446171916451, Tokens per sec: 177295.44269109832, Loss: 2.578416585922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2894 , TFLOPS: 71.5300665430356, Tokens per sec: 177259.38458095773, Loss: 2.5805180072784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2895 , TFLOPS: 70.10210425099278, Tokens per sec: 173720.73671824345, Loss: 2.605252742767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2896 , TFLOPS: 69.79246339567878, Tokens per sec: 172953.41256901927, Loss: 2.5664889812469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2897 , TFLOPS: 71.52712790925852, Tokens per sec: 177252.1023227409, Loss: 2.5829224586486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2898 , TFLOPS: 70.72152184400733, Tokens per sec: 175255.72174821427, Loss: 2.5661211013793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2899 , TFLOPS: 70.46040072116331, Tokens per sec: 174608.63484094047, Loss: 2.6052567958831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2900 , TFLOPS: 70.46044678242868, Tokens per sec: 174608.74898583014, Loss: 2.5839569568634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2901 , TFLOPS: 71.5565386086787, Tokens per sec: 177324.98527576006, Loss: 2.56345272064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2902 , TFLOPS: 69.60362444378583, Tokens per sec: 172485.44884390075, Loss: 2.5655534267425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2903 , TFLOPS: 70.57722882169409, Tokens per sec: 174898.14774373345, Loss: 2.5742738246917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2904 , TFLOPS: 71.49391714400667, Tokens per sec: 177169.802387979, Loss: 2.5666770935058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2905 , TFLOPS: 69.6308536085913, Tokens per sec: 172552.9257138288, Loss: 2.5905745029449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2906 , TFLOPS: 71.48448175159575, Tokens per sec: 177146.42044619043, Loss: 2.5941483974456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2907 , TFLOPS: 70.48210864193679, Tokens per sec: 174662.42945994207, Loss: 2.62154221534729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2908 , TFLOPS: 71.5110549114418, Tokens per sec: 177212.27166356417, Loss: 2.586623191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2909 , TFLOPS: 69.02400717935309, Tokens per sec: 171049.09341252377, Loss: 2.6105751991271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2910 , TFLOPS: 71.4997667653556, Tokens per sec: 177184.29839407062, Loss: 2.5849356651306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2911 , TFLOPS: 70.47281319663706, Tokens per sec: 174639.3943225115, Loss: 2.5890307426452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2912 , TFLOPS: 68.74336215059618, Tokens per sec: 170353.62411566207, Loss: 2.60083270072937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2913 , TFLOPS: 71.5013104893948, Tokens per sec: 177188.123912295, Loss: 2.580104351043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2914 , TFLOPS: 71.4902678347417, Tokens per sec: 177160.75899761578, Loss: 2.5752551555633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2915 , TFLOPS: 69.33272509034323, Tokens per sec: 171814.13040404266, Loss: 2.5904171466827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2916 , TFLOPS: 70.70661239149912, Tokens per sec: 175218.77448248712, Loss: 2.607661485671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2917 , TFLOPS: 71.5490501600551, Tokens per sec: 177306.4280751501, Loss: 2.6070375442504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2918 , TFLOPS: 69.76739462550098, Tokens per sec: 172891.28939496572, Loss: 2.588880777359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2919 , TFLOPS: 70.35253921631394, Tokens per sec: 174341.34214999815, Loss: 2.5651330947875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2920 , TFLOPS: 71.55715087571753, Tokens per sec: 177326.50254092907, Loss: 2.5764379501342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2921 , TFLOPS: 71.47881009250003, Tokens per sec: 177132.36545016643, Loss: 2.5874297618865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2922 , TFLOPS: 68.57407390906961, Tokens per sec: 169934.10920451992, Loss: 2.603865623474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2923 , TFLOPS: 71.46332249627527, Tokens per sec: 177093.9854806219, Loss: 2.5799241065979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2924 , TFLOPS: 71.49011377633286, Tokens per sec: 177160.37722390777, Loss: 2.6074612140655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2925 , TFLOPS: 69.71874632323053, Tokens per sec: 172770.73354288662, Loss: 2.592191219329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2926 , TFLOPS: 70.6483532789458, Tokens per sec: 175074.401983809, Loss: 2.5724828243255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2927 , TFLOPS: 71.54525576505046, Tokens per sec: 177297.02514634148, Loss: 2.5979228019714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2928 , TFLOPS: 71.56027360343106, Tokens per sec: 177334.24100420004, Loss: 2.6096038818359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2929 , TFLOPS: 67.85281461781587, Tokens per sec: 168146.7492275232, Loss: 2.5822887420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2930 , TFLOPS: 71.50764591259113, Tokens per sec: 177203.82378887996, Loss: 2.5802950859069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2931 , TFLOPS: 71.50550407071884, Tokens per sec: 177198.5160687771, Loss: 2.5819363594055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2932 , TFLOPS: 68.59584380447765, Tokens per sec: 169988.057403493, Loss: 2.585087537765503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2933 , TFLOPS: 70.44000915982546, Tokens per sec: 174558.10230562938, Loss: 2.5806312561035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2934 , TFLOPS: 71.5295776849659, Tokens per sec: 177258.17313680885, Loss: 2.6026339530944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2935 , TFLOPS: 70.38889537395107, Tokens per sec: 174431.43671358432, Loss: 2.578016757965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2936 , TFLOPS: 70.68580361472522, Tokens per sec: 175167.20804136543, Loss: 2.571103096008301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2937 , TFLOPS: 70.52116503623589, Tokens per sec: 174759.21550743224, Loss: 2.5596513748168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2938 , TFLOPS: 70.73818331372738, Tokens per sec: 175297.01070559293, Loss: 2.6041362285614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2939 , TFLOPS: 70.3603446068373, Tokens per sec: 174360.68476754974, Loss: 2.574991464614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2940 , TFLOPS: 70.43377866419969, Tokens per sec: 174542.66245112297, Loss: 2.5895321369171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2941 , TFLOPS: 71.48834859584933, Tokens per sec: 177156.0029122189, Loss: 2.594508409500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2942 , TFLOPS: 70.34337493588683, Tokens per sec: 174318.6320535709, Loss: 2.5836315155029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2943 , TFLOPS: 70.875909160089, Tokens per sec: 175638.3105246315, Loss: 2.568722724914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2944 , TFLOPS: 70.52175640667174, Tokens per sec: 174760.6809885171, Loss: 2.60662579536438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2945 , TFLOPS: 70.66553486268117, Tokens per sec: 175116.97984101292, Loss: 2.59158992767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2946 , TFLOPS: 70.28613017318153, Tokens per sec: 174176.77322555598, Loss: 2.6028478145599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2947 , TFLOPS: 71.52584117813646, Tokens per sec: 177248.91366127532, Loss: 2.5887842178344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2948 , TFLOPS: 70.37020057325851, Tokens per sec: 174385.10893806585, Loss: 2.5694363117218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2949 , TFLOPS: 68.66856516350022, Tokens per sec: 170168.26894206993, Loss: 2.5701122283935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2950 , TFLOPS: 71.53821638637396, Tokens per sec: 177279.58079052952, Loss: 2.5905301570892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2951 , TFLOPS: 70.59197082440795, Tokens per sec: 174934.68005042407, Loss: 2.593229293823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2952 , TFLOPS: 68.76099793398977, Tokens per sec: 170397.3275296533, Loss: 2.573392629623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2953 , TFLOPS: 71.54109628509038, Tokens per sec: 177286.71749679584, Loss: 2.592095375061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2954 , TFLOPS: 71.54095043726377, Tokens per sec: 177286.35606981502, Loss: 2.5670969486236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2955 , TFLOPS: 69.35202555908471, Tokens per sec: 171861.9590917062, Loss: 2.584017515182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2956 , TFLOPS: 70.67650442872579, Tokens per sec: 175144.16363406891, Loss: 2.5765695571899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2957 , TFLOPS: 71.4889724192402, Tokens per sec: 177157.54881530127, Loss: 2.583712100982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2958 , TFLOPS: 71.435574283607, Tokens per sec: 177025.22235291195, Loss: 2.5726451873779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2959 , TFLOPS: 68.87612725120825, Tokens per sec: 170682.6306602619, Loss: 2.5606775283813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2960 , TFLOPS: 71.46773414712096, Tokens per sec: 177104.918037401, Loss: 2.590777635574341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2961 , TFLOPS: 71.52081645337901, Tokens per sec: 177236.4618398072, Loss: 2.5909066200256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2962 , TFLOPS: 69.33361563108546, Tokens per sec: 171816.33726210307, Loss: 2.619591474533081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2963 , TFLOPS: 70.5981804716991, Tokens per sec: 174950.06824046065, Loss: 2.600038766860962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2964 , TFLOPS: 71.53273942051761, Tokens per sec: 177266.00826579993, Loss: 2.587172031402588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2965 , TFLOPS: 70.6617152845808, Tokens per sec: 175107.51450571892, Loss: 2.567495822906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2966 , TFLOPS: 69.42027559101054, Tokens per sec: 172031.09018918007, Loss: 2.589585781097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2967 , TFLOPS: 71.53628294421866, Tokens per sec: 177274.78950788247, Loss: 2.586848020553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2968 , TFLOPS: 71.53369772557596, Tokens per sec: 177268.3830513005, Loss: 2.5942797660827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2969 , TFLOPS: 68.60036618027787, Tokens per sec: 169999.26434890679, Loss: 2.5704047679901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2970 , TFLOPS: 70.59467198564096, Tokens per sec: 174941.37382551693, Loss: 2.573477029800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2971 , TFLOPS: 71.55219724577694, Tokens per sec: 177314.22690024893, Loss: 2.5807647705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2972 , TFLOPS: 69.62108350006515, Tokens per sec: 172528.7143086347, Loss: 2.5986974239349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2973 , TFLOPS: 70.85484169388774, Tokens per sec: 175586.1030226118, Loss: 2.574551820755005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2974 , TFLOPS: 70.36588852231647, Tokens per sec: 174374.42320081982, Loss: 2.577106237411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2975 , TFLOPS: 71.52262214706235, Tokens per sec: 177240.9365476681, Loss: 2.5542922019958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2976 , TFLOPS: 69.76156017426021, Tokens per sec: 172876.8309820732, Loss: 2.5724198818206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2977 , TFLOPS: 70.55214658732389, Tokens per sec: 174835.99120392787, Loss: 2.592965602874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2978 , TFLOPS: 71.50053255976816, Tokens per sec: 177186.19611698418, Loss: 2.6011974811553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2979 , TFLOPS: 69.60394339921982, Tokens per sec: 172486.23925060197, Loss: 2.5932557582855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2980 , TFLOPS: 71.48466331292917, Tokens per sec: 177146.87037517532, Loss: 2.580127000808716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2981 , TFLOPS: 70.47371172758896, Tokens per sec: 174641.62098119163, Loss: 2.5627057552337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2982 , TFLOPS: 70.01550134587939, Tokens per sec: 173506.12517927488, Loss: 2.5652239322662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2983 , TFLOPS: 70.46302881195071, Tokens per sec: 174615.14753941976, Loss: 2.598114252090454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2984 , TFLOPS: 71.48536374802883, Tokens per sec: 177148.60613051156, Loss: 2.5713136196136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2985 , TFLOPS: 69.73846511107433, Tokens per sec: 172819.59887136737, Loss: 2.572117328643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2986 , TFLOPS: 70.37767892023263, Tokens per sec: 174403.64110567566, Loss: 2.592280864715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2987 , TFLOPS: 71.4767577421353, Tokens per sec: 177127.27949987756, Loss: 2.592512607574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2988 , TFLOPS: 71.51813765590087, Tokens per sec: 177229.82348456248, Loss: 2.581291913986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2989 , TFLOPS: 67.4873656743311, Tokens per sec: 167241.12648804343, Loss: 2.572063446044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2990 , TFLOPS: 71.50852000070495, Tokens per sec: 177205.98987551476, Loss: 2.5655486583709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2991 , TFLOPS: 71.53420485904483, Tokens per sec: 177269.63978390174, Loss: 2.580230712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2992 , TFLOPS: 69.73663796858662, Tokens per sec: 172815.07101100645, Loss: 2.591397762298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2993 , TFLOPS: 69.85262026262265, Tokens per sec: 173102.48791213214, Loss: 2.5927281379699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2994 , TFLOPS: 71.54407795797759, Tokens per sec: 177294.10641066876, Loss: 2.589937686920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2995 , TFLOPS: 71.46852390495985, Tokens per sec: 177106.87514432476, Loss: 2.5787978172302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2996 , TFLOPS: 68.4247307548593, Tokens per sec: 169564.02041688023, Loss: 2.590939521789551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2997 , TFLOPS: 71.4481160693674, Tokens per sec: 177056.302279618, Loss: 2.5864920616149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2998 , TFLOPS: 71.52025987854753, Tokens per sec: 177235.082586063, Loss: 2.579293727874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2999 , TFLOPS: 68.73016758534477, Tokens per sec: 170320.92652946772, Loss: 2.5816433429718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3000 , TFLOPS: 71.55581599485573, Tokens per sec: 177323.1945591043, Loss: 2.577594757080078 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/optimizer.pt +[rank0]:[2024-08-26 08:22:40,816] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006325946000288241, 'preprocessing_with_comm': 0.0015313209987652954, 'state_converting': 0.9874547990002611, : 0.9966794259999006}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000, took 4.27s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001000 + eval ppl=11.349608421325684, eval loss=2.4291832447052 +------------------------------------------------------------------ +iteration: 3001 , TFLOPS: 67.9707258862871, Tokens per sec: 168438.9463397934, Loss: 2.585448980331421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3002 , TFLOPS: 69.47014722814816, Tokens per sec: 172154.67759982144, Loss: 2.6016504764556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3003 , TFLOPS: 69.004574141823, Tokens per sec: 171000.9362047984, Loss: 2.5689876079559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3004 , TFLOPS: 71.4537915857347, Tokens per sec: 177070.3668343859, Loss: 2.5918400287628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3005 , TFLOPS: 71.39946377388004, Tokens per sec: 176935.73653190758, Loss: 2.591684103012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3006 , TFLOPS: 68.89996491612112, Tokens per sec: 170741.70301985161, Loss: 2.5813887119293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3007 , TFLOPS: 70.34132912116313, Tokens per sec: 174313.56229932047, Loss: 2.5717365741729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3008 , TFLOPS: 71.50779148945212, Tokens per sec: 177204.1845443781, Loss: 2.587785243988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3009 , TFLOPS: 68.82410489877549, Tokens per sec: 170553.71354019837, Loss: 2.58060884475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3010 , TFLOPS: 70.40657230449168, Tokens per sec: 174475.24209473858, Loss: 2.582970142364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3011 , TFLOPS: 71.53088203334693, Tokens per sec: 177261.40545578286, Loss: 2.580028533935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3012 , TFLOPS: 70.55780183570879, Tokens per sec: 174850.00553240272, Loss: 2.605785608291626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3013 , TFLOPS: 70.56780143924593, Tokens per sec: 174874.785651515, Loss: 2.5876104831695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3014 , TFLOPS: 70.1802656628446, Tokens per sec: 173914.42930700257, Loss: 2.5526809692382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3015 , TFLOPS: 71.36508081498988, Tokens per sec: 176850.53177218212, Loss: 2.5867903232574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3016 , TFLOPS: 68.6637897535011, Tokens per sec: 170156.43495003728, Loss: 2.5720882415771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3017 , TFLOPS: 71.46451335033862, Tokens per sec: 177096.93654817427, Loss: 2.5818378925323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3018 , TFLOPS: 70.31210988829828, Tokens per sec: 174241.15382151716, Loss: 2.5829927921295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3019 , TFLOPS: 69.5331557685404, Tokens per sec: 172310.81970387712, Loss: 2.5712080001831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3020 , TFLOPS: 71.53889220668874, Tokens per sec: 177281.2555477173, Loss: 2.567136287689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3021 , TFLOPS: 69.88403959678224, Tokens per sec: 173180.348483305, Loss: 2.5548839569091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3022 , TFLOPS: 70.41525459935512, Tokens per sec: 174496.75777784392, Loss: 2.589268445968628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3023 , TFLOPS: 69.90897007568981, Tokens per sec: 173242.12895635693, Loss: 2.582228422164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3024 , TFLOPS: 71.52882424677475, Tokens per sec: 177256.30603397312, Loss: 2.5904529094696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3025 , TFLOPS: 69.28185144929631, Tokens per sec: 171688.0599173342, Loss: 2.5846128463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3026 , TFLOPS: 68.75049067780921, Tokens per sec: 170371.28939136805, Loss: 2.5841124057769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3027 , TFLOPS: 71.47249481013078, Tokens per sec: 177116.71548476967, Loss: 2.572861671447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3028 , TFLOPS: 71.54226448124558, Tokens per sec: 177289.61241555586, Loss: 2.5885086059570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3029 , TFLOPS: 68.30061157126423, Tokens per sec: 169256.43940707482, Loss: 2.57346248626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3030 , TFLOPS: 70.84489371788003, Tokens per sec: 175561.45084220462, Loss: 2.5772862434387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3031 , TFLOPS: 71.52660342436032, Tokens per sec: 177250.80259138637, Loss: 2.578084707260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3032 , TFLOPS: 69.59510667156808, Tokens per sec: 172464.34086603508, Loss: 2.5836968421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3033 , TFLOPS: 70.36657306840367, Tokens per sec: 174376.1195814329, Loss: 2.581373453140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3034 , TFLOPS: 71.49679900951166, Tokens per sec: 177176.94396816404, Loss: 2.589794635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3035 , TFLOPS: 71.44195218341248, Tokens per sec: 177041.02749121364, Loss: 2.5880825519561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3036 , TFLOPS: 68.46696005954165, Tokens per sec: 169668.66928582505, Loss: 2.5980312824249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3037 , TFLOPS: 71.49435557199203, Tokens per sec: 177170.88886082353, Loss: 2.5997390747070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3038 , TFLOPS: 71.49242165197697, Tokens per sec: 177166.09639398733, Loss: 2.5842108726501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3039 , TFLOPS: 69.48283068697168, Tokens per sec: 172186.10860798444, Loss: 2.577582359313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3040 , TFLOPS: 70.45268316299926, Tokens per sec: 174589.50988165077, Loss: 2.5778510570526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3041 , TFLOPS: 71.52041225050635, Tokens per sec: 177235.4601805621, Loss: 2.573450803756714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3042 , TFLOPS: 71.56173983177723, Tokens per sec: 177337.87447956024, Loss: 2.566526174545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3043 , TFLOPS: 69.07804062215737, Tokens per sec: 171182.99423606784, Loss: 2.574247121810913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3044 , TFLOPS: 70.5206729537171, Tokens per sec: 174757.99607274326, Loss: 2.5902252197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3045 , TFLOPS: 71.54483301309466, Tokens per sec: 177295.97752042505, Loss: 2.5640504360198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3046 , TFLOPS: 68.65977665853512, Tokens per sec: 170146.49005863268, Loss: 2.5769951343536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3047 , TFLOPS: 70.41990740409261, Tokens per sec: 174508.28794053217, Loss: 2.58565354347229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3048 , TFLOPS: 71.53539183267371, Tokens per sec: 177272.58123531003, Loss: 2.5839524269104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3049 , TFLOPS: 69.6477876040853, Tokens per sec: 172594.8900200965, Loss: 2.5864415168762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3050 , TFLOPS: 70.69403618146082, Tokens per sec: 175187.60924862832, Loss: 2.5946547985076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3051 , TFLOPS: 70.53492796142886, Tokens per sec: 174793.3215521718, Loss: 2.5663034915924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3052 , TFLOPS: 71.51340605258603, Tokens per sec: 177218.09805031918, Loss: 2.5705270767211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3053 , TFLOPS: 69.73821814342224, Tokens per sec: 172818.98685831975, Loss: 2.565458297729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3054 , TFLOPS: 71.54977606023805, Tokens per sec: 177308.22693576795, Loss: 2.5545637607574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3055 , TFLOPS: 70.462819749228, Tokens per sec: 174614.62945896274, Loss: 2.588538646697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3056 , TFLOPS: 68.73471656936863, Tokens per sec: 170332.19941299083, Loss: 2.5799901485443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3057 , TFLOPS: 71.5286586865258, Tokens per sec: 177255.89575743824, Loss: 2.574252128601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3058 , TFLOPS: 71.50557427136962, Tokens per sec: 177198.69003372296, Loss: 2.5717196464538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3059 , TFLOPS: 68.67892399094669, Tokens per sec: 170193.93925760133, Loss: 2.5651447772979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3060 , TFLOPS: 71.50076350739567, Tokens per sec: 177186.76843064613, Loss: 2.574749231338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3061 , TFLOPS: 71.47384758984478, Tokens per sec: 177120.06782192193, Loss: 2.578948497772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3062 , TFLOPS: 69.96237454104603, Tokens per sec: 173374.47110449715, Loss: 2.583012342453003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3063 , TFLOPS: 68.30958649159972, Tokens per sec: 169278.68024833527, Loss: 2.570314884185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3064 , TFLOPS: 71.49490017746069, Tokens per sec: 177172.23845316813, Loss: 2.581575870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3065 , TFLOPS: 71.50851940186804, Tokens per sec: 177205.98839153096, Loss: 2.5789904594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3066 , TFLOPS: 68.4890518299374, Tokens per sec: 169723.4151264751, Loss: 2.574826240539551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3067 , TFLOPS: 71.50433089420234, Tokens per sec: 177195.60880813314, Loss: 2.5874767303466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3068 , TFLOPS: 71.48467357285087, Tokens per sec: 177146.8958003915, Loss: 2.592207670211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3069 , TFLOPS: 68.12802496649388, Tokens per sec: 168828.75078846992, Loss: 2.5769872665405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3070 , TFLOPS: 70.66231081595582, Tokens per sec: 175108.99029806943, Loss: 2.5911850929260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3071 , TFLOPS: 71.50391994382761, Tokens per sec: 177194.59042783466, Loss: 2.5848498344421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3072 , TFLOPS: 71.55453043633719, Tokens per sec: 177320.00880347937, Loss: 2.5695414543151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3073 , TFLOPS: 68.56075047071525, Tokens per sec: 169901.0922565799, Loss: 2.589615821838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3074 , TFLOPS: 71.41342502845188, Tokens per sec: 176970.33405869568, Loss: 2.567627191543579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3075 , TFLOPS: 71.55717087632186, Tokens per sec: 177326.55210462987, Loss: 2.576570510864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3076 , TFLOPS: 68.80409040411047, Tokens per sec: 170504.11541763984, Loss: 2.5554521083831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3077 , TFLOPS: 70.19893027783664, Tokens per sec: 173960.68227902986, Loss: 2.5720250606536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3078 , TFLOPS: 71.49208537617255, Tokens per sec: 177165.26306550016, Loss: 2.589005470275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3079 , TFLOPS: 70.57737053136476, Tokens per sec: 174898.49891590799, Loss: 2.5837819576263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3080 , TFLOPS: 70.65372908385466, Tokens per sec: 175087.7238205669, Loss: 2.5707249641418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3081 , TFLOPS: 70.60651441916842, Tokens per sec: 174970.72068034974, Loss: 2.5601425170898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3082 , TFLOPS: 71.5249804263181, Tokens per sec: 177246.78062344974, Loss: 2.586101770401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3083 , TFLOPS: 68.87339512967986, Tokens per sec: 170675.86016214656, Loss: 2.571472406387329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3084 , TFLOPS: 70.4474063173246, Tokens per sec: 174576.43327677643, Loss: 2.584554672241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3085 , TFLOPS: 71.52069247519772, Tokens per sec: 177236.15460821663, Loss: 2.594886302947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3086 , TFLOPS: 68.54126138856243, Tokens per sec: 169852.79616410562, Loss: 2.5802154541015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3087 , TFLOPS: 71.48543926601292, Tokens per sec: 177148.7932723951, Loss: 2.591536521911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3088 , TFLOPS: 70.50037795527024, Tokens per sec: 174707.70283091336, Loss: 2.57242488861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3089 , TFLOPS: 70.38814802196133, Tokens per sec: 174429.58469302612, Loss: 2.5923495292663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3090 , TFLOPS: 70.13545671634412, Tokens per sec: 173803.38780146142, Loss: 2.5821285247802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3091 , TFLOPS: 71.51278877685763, Tokens per sec: 177216.56837306562, Loss: 2.558212995529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3092 , TFLOPS: 70.46664825175083, Tokens per sec: 174624.11690995866, Loss: 2.5694544315338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3093 , TFLOPS: 69.46352183459213, Tokens per sec: 172138.25914474225, Loss: 2.592653751373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3094 , TFLOPS: 71.51899981779736, Tokens per sec: 177231.96001671694, Loss: 2.573455810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3095 , TFLOPS: 70.51499173252503, Tokens per sec: 174743.91738079116, Loss: 2.5733790397644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3096 , TFLOPS: 69.64755433434868, Tokens per sec: 172594.31195199245, Loss: 2.550361156463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3097 , TFLOPS: 70.97352655099839, Tokens per sec: 175880.21717274826, Loss: 2.5730226039886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3098 , TFLOPS: 71.51408450502959, Tokens per sec: 177219.77933021198, Loss: 2.580186605453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3099 , TFLOPS: 69.57973691895312, Tokens per sec: 172426.25292593718, Loss: 2.572242021560669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3100 , TFLOPS: 70.36916807884646, Tokens per sec: 174382.55030317412, Loss: 2.5936005115509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3101 , TFLOPS: 71.44901209487121, Tokens per sec: 177058.5227295218, Loss: 2.593658447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3102 , TFLOPS: 71.43126188774662, Tokens per sec: 177014.53576092032, Loss: 2.588365077972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3103 , TFLOPS: 68.48563119134049, Tokens per sec: 169714.9384072175, Loss: 2.6041197776794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3104 , TFLOPS: 71.52740989699303, Tokens per sec: 177252.80111941052, Loss: 2.5871529579162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3105 , TFLOPS: 71.56518751922678, Tokens per sec: 177346.4182288544, Loss: 2.5670413970947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3106 , TFLOPS: 67.67237846780135, Tokens per sec: 167699.6085711044, Loss: 2.5710997581481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3107 , TFLOPS: 71.52932246863345, Tokens per sec: 177257.5406826226, Loss: 2.585946798324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3108 , TFLOPS: 71.48165616034356, Tokens per sec: 177139.41831980294, Loss: 2.5645840167999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3109 , TFLOPS: 70.45541584804933, Tokens per sec: 174596.28177623494, Loss: 2.5852854251861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3110 , TFLOPS: 68.93347573497792, Tokens per sec: 170824.7465205002, Loss: 2.588942766189575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3111 , TFLOPS: 71.48395728388282, Tokens per sec: 177145.12075742293, Loss: 2.601863384246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3112 , TFLOPS: 71.51390448469027, Tokens per sec: 177219.3332199803, Loss: 2.588575839996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3113 , TFLOPS: 69.38953480878439, Tokens per sec: 171954.91114444696, Loss: 2.5900392532348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3114 , TFLOPS: 70.39692723029013, Tokens per sec: 174451.34053837453, Loss: 2.559699296951294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3115 , TFLOPS: 71.47895654591166, Tokens per sec: 177132.7283778536, Loss: 2.56821346282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3116 , TFLOPS: 69.56100380415988, Tokens per sec: 172379.83020385657, Loss: 2.575857162475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3117 , TFLOPS: 70.63758702681206, Tokens per sec: 175047.72202502098, Loss: 2.5638866424560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3118 , TFLOPS: 70.41790609127321, Tokens per sec: 174503.32846690287, Loss: 2.580653429031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3119 , TFLOPS: 70.5261361724459, Tokens per sec: 174771.5345305765, Loss: 2.6062045097351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3120 , TFLOPS: 70.33348906097441, Tokens per sec: 174294.13376651893, Loss: 2.594200611114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3121 , TFLOPS: 70.26034820578016, Tokens per sec: 174112.88267021722, Loss: 2.5835559368133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3122 , TFLOPS: 71.50927156241902, Tokens per sec: 177207.8523282336, Loss: 2.574868679046631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3123 , TFLOPS: 68.8713066330831, Tokens per sec: 170670.6846375129, Loss: 2.5718910694122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3124 , TFLOPS: 71.54719148101341, Tokens per sec: 177301.8220637344, Loss: 2.580003023147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3125 , TFLOPS: 69.90205498130234, Tokens per sec: 173224.99259070423, Loss: 2.5930228233337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3126 , TFLOPS: 69.15661709133285, Tokens per sec: 171377.71538259732, Loss: 2.5653560161590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3127 , TFLOPS: 70.84824372314162, Tokens per sec: 175569.7525242764, Loss: 2.609985589981079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3128 , TFLOPS: 71.5358183490466, Tokens per sec: 177273.6381898667, Loss: 2.5730948448181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3129 , TFLOPS: 69.69684889218397, Tokens per sec: 172716.4692965521, Loss: 2.593756675720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3130 , TFLOPS: 69.05949684041124, Tokens per sec: 171137.04070213475, Loss: 2.5682194232940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3131 , TFLOPS: 71.44263057256246, Tokens per sec: 177042.708614258, Loss: 2.577138900756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3132 , TFLOPS: 71.48193400640804, Tokens per sec: 177140.106852958, Loss: 2.5579257011413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3133 , TFLOPS: 68.48164198537026, Tokens per sec: 169705.052715376, Loss: 2.5828933715820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3134 , TFLOPS: 71.4856652996848, Tokens per sec: 177149.3534087337, Loss: 2.586430788040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3135 , TFLOPS: 71.50090366799121, Tokens per sec: 177187.1157640418, Loss: 2.5479061603546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3136 , TFLOPS: 68.71516505631709, Tokens per sec: 170283.7486098721, Loss: 2.5507559776306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3137 , TFLOPS: 70.68147092703092, Tokens per sec: 175156.47116397397, Loss: 2.5549027919769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3138 , TFLOPS: 71.39211648538637, Tokens per sec: 176917.52914164957, Loss: 2.5696985721588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3139 , TFLOPS: 71.48459790205617, Tokens per sec: 177146.70827982642, Loss: 2.574629783630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3140 , TFLOPS: 69.62892016185617, Tokens per sec: 172548.1344198324, Loss: 2.6035404205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3141 , TFLOPS: 71.56918568830778, Tokens per sec: 177356.3261322718, Loss: 2.5749948024749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3142 , TFLOPS: 71.45261340338061, Tokens per sec: 177067.44716872496, Loss: 2.582752227783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3143 , TFLOPS: 67.95799651597468, Tokens per sec: 168407.40155790333, Loss: 2.5663132667541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3144 , TFLOPS: 71.54108513888903, Tokens per sec: 177286.68987528107, Loss: 2.595801830291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3145 , TFLOPS: 71.53112372095346, Tokens per sec: 177262.00438429593, Loss: 2.592412233352661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3146 , TFLOPS: 70.55863569543244, Tokens per sec: 174852.07192865518, Loss: 2.5824153423309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3147 , TFLOPS: 68.80975077802535, Tokens per sec: 170518.14244774258, Loss: 2.5889782905578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3148 , TFLOPS: 71.51576014983615, Tokens per sec: 177223.93176263018, Loss: 2.567099094390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3149 , TFLOPS: 71.52887005331307, Tokens per sec: 177256.41954762107, Loss: 2.5364885330200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3150 , TFLOPS: 68.1921171859205, Tokens per sec: 168987.5783685499, Loss: 2.554452419281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3151 , TFLOPS: 70.45780768859511, Tokens per sec: 174602.20902059038, Loss: 2.5975561141967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3152 , TFLOPS: 71.45477655383246, Tokens per sec: 177072.80769383538, Loss: 2.5937485694885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3153 , TFLOPS: 69.36383986605571, Tokens per sec: 171891.2362458337, Loss: 2.5875494480133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3154 , TFLOPS: 71.4731459041156, Tokens per sec: 177118.32896738793, Loss: 2.5821924209594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3155 , TFLOPS: 70.3566517258195, Tokens per sec: 174351.53340158323, Loss: 2.5847504138946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3156 , TFLOPS: 69.97535443160051, Tokens per sec: 173406.63670314412, Loss: 2.5812828540802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3157 , TFLOPS: 70.1266336242666, Tokens per sec: 173781.5232073495, Loss: 2.555271625518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3158 , TFLOPS: 70.46159824861981, Tokens per sec: 174611.60244589625, Loss: 2.5649032592773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3159 , TFLOPS: 71.56064027338937, Tokens per sec: 177335.1496527488, Loss: 2.5674214363098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3160 , TFLOPS: 69.42782066874874, Tokens per sec: 172049.78772297426, Loss: 2.5692527294158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3161 , TFLOPS: 71.58149187604731, Tokens per sec: 177386.82222112882, Loss: 2.591034412384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3162 , TFLOPS: 70.58663583600374, Tokens per sec: 174921.45936146093, Loss: 2.5523972511291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3163 , TFLOPS: 69.6509864063298, Tokens per sec: 172602.81700443558, Loss: 2.5651698112487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3164 , TFLOPS: 70.96977057590757, Tokens per sec: 175870.90945271842, Loss: 2.578974962234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3165 , TFLOPS: 71.54184420076568, Tokens per sec: 177288.57091422894, Loss: 2.577324390411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3166 , TFLOPS: 69.5163409825178, Tokens per sec: 172269.15081181153, Loss: 2.5674173831939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3167 , TFLOPS: 69.45821033378148, Tokens per sec: 172125.09666062336, Loss: 2.571598529815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3168 , TFLOPS: 71.52308204889404, Tokens per sec: 177242.0762350694, Loss: 2.5825319290161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3169 , TFLOPS: 70.5357302881096, Tokens per sec: 174795.30980607023, Loss: 2.5617284774780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3170 , TFLOPS: 68.83426698892507, Tokens per sec: 170578.89631903567, Loss: 2.5585806369781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3171 , TFLOPS: 71.52657544932609, Tokens per sec: 177250.73326616985, Loss: 2.569716453552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3172 , TFLOPS: 71.52863369601455, Tokens per sec: 177255.83382819843, Loss: 2.5882339477539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3173 , TFLOPS: 68.40113264017793, Tokens per sec: 169505.54168915292, Loss: 2.5815954208374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3174 , TFLOPS: 71.51520638746743, Tokens per sec: 177222.55947847854, Loss: 2.536747694015503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3175 , TFLOPS: 71.55314566689215, Tokens per sec: 177316.5771922496, Loss: 2.5911567211151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3176 , TFLOPS: 70.4487891240814, Tokens per sec: 174579.8600242485, Loss: 2.561497688293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3177 , TFLOPS: 69.61618319822175, Tokens per sec: 172516.57082085393, Loss: 2.5891289710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3178 , TFLOPS: 71.53458273470208, Tokens per sec: 177270.57620140686, Loss: 2.584287166595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3179 , TFLOPS: 71.49873246153092, Tokens per sec: 177181.73527525508, Loss: 2.571430206298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3180 , TFLOPS: 68.45179477428113, Tokens per sec: 169631.08803835537, Loss: 2.579582452774048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3181 , TFLOPS: 71.54074340727111, Tokens per sec: 177285.84302668698, Loss: 2.5946662425994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3182 , TFLOPS: 71.50326194447716, Tokens per sec: 177192.9598329596, Loss: 2.5883569717407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3183 , TFLOPS: 69.5875945677676, Tokens per sec: 172445.72504529092, Loss: 2.5714845657348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3184 , TFLOPS: 69.70355722212608, Tokens per sec: 172733.09327713217, Loss: 2.5967111587524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3185 , TFLOPS: 71.5085573109027, Tokens per sec: 177206.0823342949, Loss: 2.5707948207855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3186 , TFLOPS: 70.62079083550395, Tokens per sec: 175006.09921270626, Loss: 2.598700761795044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3187 , TFLOPS: 69.44473769436361, Tokens per sec: 172091.70997601212, Loss: 2.5833089351654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3188 , TFLOPS: 70.53276926248458, Tokens per sec: 174787.9720583875, Loss: 2.5702333450317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3189 , TFLOPS: 71.53905816229005, Tokens per sec: 177281.66680397888, Loss: 2.589077949523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3190 , TFLOPS: 68.68119040662758, Tokens per sec: 170199.55568532489, Loss: 2.566241502761841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3191 , TFLOPS: 71.48717406128625, Tokens per sec: 177153.0922861859, Loss: 2.5734565258026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3192 , TFLOPS: 70.43005079720227, Tokens per sec: 174533.42438604494, Loss: 2.5682504177093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3193 , TFLOPS: 69.45199535970967, Tokens per sec: 172109.6952702375, Loss: 2.5735671520233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3194 , TFLOPS: 71.51489308142013, Tokens per sec: 177221.78307157982, Loss: 2.573485851287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3195 , TFLOPS: 70.59526670749625, Tokens per sec: 174942.84761178837, Loss: 2.584907293319702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3196 , TFLOPS: 71.49355326218672, Tokens per sec: 177168.90064874428, Loss: 2.553851366043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3197 , TFLOPS: 69.020786640483, Tokens per sec: 171041.112562432, Loss: 2.582148551940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3198 , TFLOPS: 71.49084770995631, Tokens per sec: 177162.19599227613, Loss: 2.5827629566192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3199 , TFLOPS: 70.47135861666099, Tokens per sec: 174635.7897130956, Loss: 2.595820665359497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3200 , TFLOPS: 69.51889533997198, Tokens per sec: 172275.48079096866, Loss: 2.5968778133392334 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/optimizer.pt +[rank0]:[2024-08-26 08:47:50,925] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063581719987269025, 'preprocessing_with_comm': 0.0014828469975327607, 'state_converting': 0.9781950309989043, : 0.9874202059982053}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200, took 4.74s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001200 + eval ppl=10.406428337097168, eval loss=2.342423677444458 +------------------------------------------------------------------ +iteration: 3201 , TFLOPS: 66.78243559325294, Tokens per sec: 165494.23209267278, Loss: 2.598687171936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3202 , TFLOPS: 71.53093127342704, Tokens per sec: 177261.52747812564, Loss: 2.575568675994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3203 , TFLOPS: 69.28512347183467, Tokens per sec: 171696.16834962726, Loss: 2.584639310836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3204 , TFLOPS: 69.78703784400207, Tokens per sec: 172939.96745429054, Loss: 2.557512044906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3205 , TFLOPS: 71.54983107967328, Tokens per sec: 177308.36327998934, Loss: 2.5521693229675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3206 , TFLOPS: 70.28862657192757, Tokens per sec: 174182.95957664968, Loss: 2.575507402420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3207 , TFLOPS: 69.07330659904382, Tokens per sec: 171171.26280529678, Loss: 2.5774166584014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3208 , TFLOPS: 71.50048768751773, Tokens per sec: 177186.08491860452, Loss: 2.5685019493103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3209 , TFLOPS: 71.48977919225979, Tokens per sec: 177159.5480877172, Loss: 2.579210042953491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3210 , TFLOPS: 68.54407799825773, Tokens per sec: 169859.77603320577, Loss: 2.570146083831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3211 , TFLOPS: 70.90757744504235, Tokens per sec: 175716.78802329546, Loss: 2.5438785552978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3212 , TFLOPS: 71.46755140801528, Tokens per sec: 177104.46518976675, Loss: 2.5721147060394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3213 , TFLOPS: 70.60713107359528, Tokens per sec: 174972.24881794958, Loss: 2.585451364517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3214 , TFLOPS: 68.62815066115809, Tokens per sec: 170068.1173532412, Loss: 2.5595860481262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3215 , TFLOPS: 71.4890684190311, Tokens per sec: 177157.78671335836, Loss: 2.578779697418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3216 , TFLOPS: 71.51636764250502, Tokens per sec: 177225.43719638445, Loss: 2.5631513595581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3217 , TFLOPS: 69.59389863986266, Tokens per sec: 172461.34723039335, Loss: 2.569201946258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3218 , TFLOPS: 70.42563049598918, Tokens per sec: 174522.47039270253, Loss: 2.573334217071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3219 , TFLOPS: 71.4972810182124, Tokens per sec: 177178.13843882212, Loss: 2.589857816696167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3220 , TFLOPS: 69.49596299632876, Tokens per sec: 172218.6519172146, Loss: 2.5688304901123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3221 , TFLOPS: 70.36699469243726, Tokens per sec: 174377.16441223383, Loss: 2.5809895992279053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3222 , TFLOPS: 71.55553212932345, Tokens per sec: 177322.49110904493, Loss: 2.565990686416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3223 , TFLOPS: 70.34603144830609, Tokens per sec: 174325.21518398484, Loss: 2.5767102241516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3224 , TFLOPS: 70.6882974155922, Tokens per sec: 175173.38795462865, Loss: 2.574272871017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3225 , TFLOPS: 70.39417535357859, Tokens per sec: 174444.52108473907, Loss: 2.572895050048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3226 , TFLOPS: 71.54207230239498, Tokens per sec: 177289.13617519385, Loss: 2.575192451477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3227 , TFLOPS: 69.46016799289015, Tokens per sec: 172129.9479555486, Loss: 2.5718865394592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3228 , TFLOPS: 71.56639830351266, Tokens per sec: 177349.4186856937, Loss: 2.563890218734741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3229 , TFLOPS: 70.51338871701275, Tokens per sec: 174739.94493176503, Loss: 2.605618476867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3230 , TFLOPS: 69.58621679891358, Tokens per sec: 172442.3107822969, Loss: 2.5891811847686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3231 , TFLOPS: 70.75229941058815, Tokens per sec: 175331.9919486304, Loss: 2.5864431858062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3232 , TFLOPS: 71.5195718214767, Tokens per sec: 177233.37750484573, Loss: 2.5844194889068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3233 , TFLOPS: 69.48114834799922, Tokens per sec: 172181.93958668577, Loss: 2.5834708213806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3234 , TFLOPS: 69.50656059433119, Tokens per sec: 172244.9139324864, Loss: 2.5815834999084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3235 , TFLOPS: 71.5422126153886, Tokens per sec: 177289.48388624878, Loss: 2.5374879837036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3236 , TFLOPS: 70.43117343333607, Tokens per sec: 174536.20640205304, Loss: 2.590348958969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3237 , TFLOPS: 69.59049642228878, Tokens per sec: 172452.9161604601, Loss: 2.5670831203460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3238 , TFLOPS: 71.46981039806167, Tokens per sec: 177110.06321594998, Loss: 2.579258441925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3239 , TFLOPS: 71.45774556717288, Tokens per sec: 177080.16523595672, Loss: 2.5699381828308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3240 , TFLOPS: 68.40868889544187, Tokens per sec: 169524.26692208715, Loss: 2.5788676738739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3241 , TFLOPS: 71.42210697488939, Tokens per sec: 176991.8488783632, Loss: 2.566775321960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3242 , TFLOPS: 71.509307733713, Tokens per sec: 177207.94196468467, Loss: 2.5857179164886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3243 , TFLOPS: 71.5090931792916, Tokens per sec: 177207.4102751933, Loss: 2.5898125171661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3244 , TFLOPS: 69.50667703096386, Tokens per sec: 172245.20247528874, Loss: 2.5672054290771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3245 , TFLOPS: 71.5281473120944, Tokens per sec: 177254.6285152647, Loss: 2.577482223510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3246 , TFLOPS: 71.51142866921911, Tokens per sec: 177213.19787650916, Loss: 2.57033109664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3247 , TFLOPS: 69.45060198282992, Tokens per sec: 172106.24232883635, Loss: 2.554945230484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3248 , TFLOPS: 70.39616986076138, Tokens per sec: 174449.4636932511, Loss: 2.5721583366394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3249 , TFLOPS: 71.5387949710585, Tokens per sec: 177281.0145871142, Loss: 2.5607166290283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3250 , TFLOPS: 69.60937471038807, Tokens per sec: 172499.69863798522, Loss: 2.5819647312164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3251 , TFLOPS: 69.633896203336, Tokens per sec: 172560.46559877563, Loss: 2.554252862930298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3252 , TFLOPS: 71.54487235280624, Tokens per sec: 177296.07500856396, Loss: 2.5782177448272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3253 , TFLOPS: 70.62164692806884, Tokens per sec: 175008.22070438843, Loss: 2.5679805278778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3254 , TFLOPS: 69.5451773977537, Tokens per sec: 172340.6106253595, Loss: 2.5782783031463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3255 , TFLOPS: 70.54227362640023, Tokens per sec: 174811.52491916265, Loss: 2.57566237449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3256 , TFLOPS: 71.46986805384458, Tokens per sec: 177110.20609333136, Loss: 2.556041955947876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3257 , TFLOPS: 69.57774976444918, Tokens per sec: 172421.32853817326, Loss: 2.549316167831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3258 , TFLOPS: 70.4420432017145, Tokens per sec: 174563.14288549864, Loss: 2.588648796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3259 , TFLOPS: 71.52146701743638, Tokens per sec: 177238.0740092069, Loss: 2.5618064403533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3260 , TFLOPS: 70.60067921226747, Tokens per sec: 174956.26039484807, Loss: 2.5729212760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3261 , TFLOPS: 70.70783850188451, Tokens per sec: 175221.8129190893, Loss: 2.5962061882019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3262 , TFLOPS: 70.49614003078814, Tokens per sec: 174697.2007872008, Loss: 2.590003728866577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3263 , TFLOPS: 71.5386733278021, Tokens per sec: 177280.71314172473, Loss: 2.5642313957214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3264 , TFLOPS: 69.57263656109673, Tokens per sec: 172408.65745700005, Loss: 2.5548343658447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3265 , TFLOPS: 71.51832124312746, Tokens per sec: 177230.27843393377, Loss: 2.5524744987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3266 , TFLOPS: 70.4365259633598, Tokens per sec: 174549.4705610817, Loss: 2.564098834991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3267 , TFLOPS: 69.5478532451596, Tokens per sec: 172347.24167000022, Loss: 2.5822324752807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3268 , TFLOPS: 71.54727849051568, Tokens per sec: 177302.0376828659, Loss: 2.57307767868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3269 , TFLOPS: 71.55854782226527, Tokens per sec: 177329.96432836074, Loss: 2.5773770809173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3270 , TFLOPS: 68.65888546455173, Tokens per sec: 170144.28158176874, Loss: 2.550281047821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3271 , TFLOPS: 70.69824535811114, Tokens per sec: 175198.04005204688, Loss: 2.5538012981414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3272 , TFLOPS: 71.5615904563135, Tokens per sec: 177337.50431070605, Loss: 2.589813232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3273 , TFLOPS: 69.59127092165289, Tokens per sec: 172454.8354552025, Loss: 2.574669361114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3274 , TFLOPS: 69.61543937680294, Tokens per sec: 172514.72754943944, Loss: 2.5597145557403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3275 , TFLOPS: 71.43457165719632, Tokens per sec: 177022.7377342179, Loss: 2.558311700820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3276 , TFLOPS: 71.51186546711503, Tokens per sec: 177214.28030981252, Loss: 2.5705580711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3277 , TFLOPS: 69.39034594904214, Tokens per sec: 171956.92123936012, Loss: 2.578213691711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3278 , TFLOPS: 70.87643848254075, Tokens per sec: 175639.62224397663, Loss: 2.5915067195892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3279 , TFLOPS: 71.52975296185122, Tokens per sec: 177258.60749223898, Loss: 2.5826737880706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3280 , TFLOPS: 70.63270539593852, Tokens per sec: 175035.6248059603, Loss: 2.5731639862060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3281 , TFLOPS: 69.47063779441382, Tokens per sec: 172155.8932770681, Loss: 2.590613603591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3282 , TFLOPS: 71.51338842372294, Tokens per sec: 177218.05436405443, Loss: 2.5610296726226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3283 , TFLOPS: 71.53968673766686, Tokens per sec: 177283.2244830059, Loss: 2.578397035598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3284 , TFLOPS: 68.47573598816545, Tokens per sec: 169690.41700370316, Loss: 2.561553955078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3285 , TFLOPS: 71.4417278046368, Tokens per sec: 177040.4714558901, Loss: 2.569206953048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3286 , TFLOPS: 71.51704850833254, Tokens per sec: 177227.1244569086, Loss: 2.567105770111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3287 , TFLOPS: 69.5740724801345, Tokens per sec: 172412.21582255518, Loss: 2.5848147869110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3288 , TFLOPS: 70.58007257005248, Tokens per sec: 174905.19486543033, Loss: 2.5731382369995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3289 , TFLOPS: 71.52483862529022, Tokens per sec: 177246.42922488195, Loss: 2.5700151920318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3290 , TFLOPS: 70.61491344748636, Tokens per sec: 174991.53439775092, Loss: 2.5722851753234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3291 , TFLOPS: 69.83537988205079, Tokens per sec: 173059.76434989035, Loss: 2.576242685317993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3292 , TFLOPS: 70.44098856983037, Tokens per sec: 174560.52939151245, Loss: 2.5569424629211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3293 , TFLOPS: 71.4784537178393, Tokens per sec: 177131.48231449924, Loss: 2.576033353805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3294 , TFLOPS: 69.02294763476165, Tokens per sec: 171046.46774430768, Loss: 2.5701663494110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3295 , TFLOPS: 70.48991380167631, Tokens per sec: 174681.77150558593, Loss: 2.592639446258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3296 , TFLOPS: 71.55229629717024, Tokens per sec: 177314.47236051294, Loss: 2.5696561336517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3297 , TFLOPS: 70.49621777233679, Tokens per sec: 174697.3934393223, Loss: 2.589390277862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3298 , TFLOPS: 70.57423037768103, Tokens per sec: 174890.71726916323, Loss: 2.5528628826141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3299 , TFLOPS: 70.44728816985132, Tokens per sec: 174576.14049432264, Loss: 2.552380323410034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3300 , TFLOPS: 70.53592263349752, Tokens per sec: 174795.78645913006, Loss: 2.5494437217712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3301 , TFLOPS: 70.3568395522196, Tokens per sec: 174351.99885609344, Loss: 2.568920850753784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3302 , TFLOPS: 71.53891077610683, Tokens per sec: 177281.3015647809, Loss: 2.5962796211242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3303 , TFLOPS: 70.54076894922486, Tokens per sec: 174807.79616336818, Loss: 2.5910868644714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3304 , TFLOPS: 69.54294590856621, Tokens per sec: 172335.08074933404, Loss: 2.550463914871216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3305 , TFLOPS: 71.5137312947074, Tokens per sec: 177218.90403612427, Loss: 2.5327444076538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3306 , TFLOPS: 71.48081912214741, Tokens per sec: 177137.3440469455, Loss: 2.570674180984497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3307 , TFLOPS: 68.46670459051072, Tokens per sec: 169668.03620542414, Loss: 2.548184633255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3308 , TFLOPS: 71.52399125115222, Tokens per sec: 177244.32933842205, Loss: 2.552428722381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3309 , TFLOPS: 71.50001870990253, Tokens per sec: 177184.92274041192, Loss: 2.5597681999206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3310 , TFLOPS: 70.54118277583402, Tokens per sec: 174808.82167129245, Loss: 2.5759077072143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3311 , TFLOPS: 69.04703138271161, Tokens per sec: 171106.1499247715, Loss: 2.5630946159362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3312 , TFLOPS: 71.52273730071639, Tokens per sec: 177241.22191110803, Loss: 2.561518430709839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3313 , TFLOPS: 71.51987404825897, Tokens per sec: 177234.12645610524, Loss: 2.542100429534912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3314 , TFLOPS: 67.88947368617315, Tokens per sec: 168237.59443724228, Loss: 2.5679192543029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3315 , TFLOPS: 71.54536848748778, Tokens per sec: 177297.30448495856, Loss: 2.5847327709198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3316 , TFLOPS: 71.52812290173475, Tokens per sec: 177254.56802370443, Loss: 2.595949649810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3317 , TFLOPS: 70.39824854035189, Tokens per sec: 174454.61489026184, Loss: 2.548705577850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3318 , TFLOPS: 69.47405165886568, Tokens per sec: 172164.353209248, Loss: 2.5919885635375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3319 , TFLOPS: 71.56556790857341, Tokens per sec: 177347.36087555866, Loss: 2.5783560276031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3320 , TFLOPS: 70.71882148057826, Tokens per sec: 175249.02995016743, Loss: 2.568310260772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3321 , TFLOPS: 70.360251642579, Tokens per sec: 174360.45439187682, Loss: 2.56719970703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3322 , TFLOPS: 70.10922787886588, Tokens per sec: 173738.38985284534, Loss: 2.564919948577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3323 , TFLOPS: 71.51498981740018, Tokens per sec: 177222.0227939947, Loss: 2.5538973808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3324 , TFLOPS: 69.4965436741638, Tokens per sec: 172220.0909008568, Loss: 2.5664384365081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3325 , TFLOPS: 70.45399480063247, Tokens per sec: 174592.7602641947, Loss: 2.5917139053344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3326 , TFLOPS: 71.50059649340248, Tokens per sec: 177186.35455157288, Loss: 2.57065486907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3327 , TFLOPS: 70.66362762038617, Tokens per sec: 175112.25348450578, Loss: 2.571279764175415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3328 , TFLOPS: 70.65800606306784, Tokens per sec: 175098.32264620534, Loss: 2.5682239532470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3329 , TFLOPS: 70.50024707843166, Tokens per sec: 174707.37850369004, Loss: 2.5968570709228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3330 , TFLOPS: 71.50308500122766, Tokens per sec: 177192.52134809547, Loss: 2.5696797370910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3331 , TFLOPS: 68.19063241904864, Tokens per sec: 168983.89895268087, Loss: 2.5429980754852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3332 , TFLOPS: 70.44639089374265, Tokens per sec: 174573.9169452822, Loss: 2.5472464561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3333 , TFLOPS: 71.54818349290552, Tokens per sec: 177304.28037848216, Loss: 2.574179172515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3334 , TFLOPS: 68.8320882973386, Tokens per sec: 170573.4972812817, Loss: 2.5789761543273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3335 , TFLOPS: 71.50243690562743, Tokens per sec: 177190.91529580444, Loss: 2.5665907859802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3336 , TFLOPS: 70.47319628310987, Tokens per sec: 174640.34365299158, Loss: 2.5660929679870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3337 , TFLOPS: 70.50031883955512, Tokens per sec: 174707.55633565917, Loss: 2.5969865322113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3338 , TFLOPS: 70.6048461785303, Tokens per sec: 174966.58659627658, Loss: 2.593648910522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3339 , TFLOPS: 71.53682764731268, Tokens per sec: 177276.13934215344, Loss: 2.5771381855010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3340 , TFLOPS: 69.62666641963108, Tokens per sec: 172542.54939832844, Loss: 2.5725955963134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3341 , TFLOPS: 70.41448319989657, Tokens per sec: 174494.84616500896, Loss: 2.5642623901367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3342 , TFLOPS: 71.5390707578595, Tokens per sec: 177281.69801718753, Loss: 2.545989513397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3343 , TFLOPS: 71.48165269557626, Tokens per sec: 177139.4097337279, Loss: 2.5855469703674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3344 , TFLOPS: 69.4457622410066, Tokens per sec: 172094.24891545658, Loss: 2.5755271911621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3345 , TFLOPS: 70.90491979267173, Tokens per sec: 175710.2020679565, Loss: 2.5571436882019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3346 , TFLOPS: 71.5252769397542, Tokens per sec: 177247.51541640804, Loss: 2.5708587169647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3347 , TFLOPS: 69.68375615939915, Tokens per sec: 172684.0240624277, Loss: 2.589776039123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3348 , TFLOPS: 70.41442679418081, Tokens per sec: 174494.70638543172, Loss: 2.573119878768921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3349 , TFLOPS: 71.4789215537605, Tokens per sec: 177132.64166344836, Loss: 2.5693435668945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3350 , TFLOPS: 71.5380709272675, Tokens per sec: 177279.22032684006, Loss: 2.5609123706817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3351 , TFLOPS: 67.73064618270442, Tokens per sec: 167844.00238735287, Loss: 2.5518500804901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3352 , TFLOPS: 71.52935044578135, Tokens per sec: 177257.61001307706, Loss: 2.5751354694366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3353 , TFLOPS: 71.53930200279666, Tokens per sec: 177282.2710676155, Loss: 2.560363531112671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3354 , TFLOPS: 68.82233864819636, Tokens per sec: 170549.3365766947, Loss: 2.563542366027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3355 , TFLOPS: 70.41191668518067, Tokens per sec: 174488.48605881803, Loss: 2.5634355545043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3356 , TFLOPS: 71.53222819068085, Tokens per sec: 177264.7413819478, Loss: 2.5609874725341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3357 , TFLOPS: 70.44119382914069, Tokens per sec: 174561.0380466946, Loss: 2.5767321586608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3358 , TFLOPS: 69.63620464510589, Tokens per sec: 172566.18617177728, Loss: 2.5787744522094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3359 , TFLOPS: 71.54992573453578, Tokens per sec: 177308.5978451657, Loss: 2.5645015239715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3360 , TFLOPS: 71.51612574547373, Tokens per sec: 177224.83774889377, Loss: 2.570164680480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3361 , TFLOPS: 69.89866445579977, Tokens per sec: 173216.59049501168, Loss: 2.563995361328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3362 , TFLOPS: 70.47089538793301, Tokens per sec: 174634.6417812788, Loss: 2.5706167221069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3363 , TFLOPS: 71.56187478581755, Tokens per sec: 177338.20891053858, Loss: 2.574636220932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3364 , TFLOPS: 70.42752297319186, Tokens per sec: 174527.1601596843, Loss: 2.575366497039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3365 , TFLOPS: 70.70836925476021, Tokens per sec: 175223.1281831826, Loss: 2.5894839763641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3366 , TFLOPS: 70.50733436262266, Tokens per sec: 174724.94157464418, Loss: 2.5599231719970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3367 , TFLOPS: 70.72287505531902, Tokens per sec: 175259.07515491315, Loss: 2.5908734798431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3368 , TFLOPS: 70.43049083407644, Tokens per sec: 174534.51484589314, Loss: 2.535759925842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3369 , TFLOPS: 70.36509339343846, Tokens per sec: 174372.45278386943, Loss: 2.5677437782287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3370 , TFLOPS: 71.54996046747146, Tokens per sec: 177308.68391720648, Loss: 2.5684142112731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3371 , TFLOPS: 68.73699306758127, Tokens per sec: 170337.84082633868, Loss: 2.5774149894714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3372 , TFLOPS: 71.51216595770278, Tokens per sec: 177215.02495859071, Loss: 2.5559170246124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3373 , TFLOPS: 70.54680575325118, Tokens per sec: 174822.75602875426, Loss: 2.5771679878234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3374 , TFLOPS: 69.5825067581694, Tokens per sec: 172433.1168926381, Loss: 2.582965612411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3375 , TFLOPS: 70.76784348889659, Tokens per sec: 175370.51188700026, Loss: 2.57892107963562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3376 , TFLOPS: 71.51116380740007, Tokens per sec: 177212.54151974464, Loss: 2.5645360946655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3377 , TFLOPS: 70.5836215741503, Tokens per sec: 174913.9896885398, Loss: 2.5723249912261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3378 , TFLOPS: 69.84741788979895, Tokens per sec: 173089.59585918635, Loss: 2.5699496269226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3379 , TFLOPS: 71.51554525723407, Tokens per sec: 177223.39923508983, Loss: 2.5652310848236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3380 , TFLOPS: 71.5177884244777, Tokens per sec: 177228.95805062482, Loss: 2.582261323928833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3381 , TFLOPS: 68.16557033042403, Tokens per sec: 168921.79233625208, Loss: 2.5595927238464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3382 , TFLOPS: 71.5000413050341, Tokens per sec: 177184.978733637, Loss: 2.6025686264038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3383 , TFLOPS: 71.43822409279787, Tokens per sec: 177031.78887198755, Loss: 2.5532782077789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3384 , TFLOPS: 69.43858769795031, Tokens per sec: 172076.46960742195, Loss: 2.5473742485046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3385 , TFLOPS: 70.72403033430054, Tokens per sec: 175261.93806349387, Loss: 2.5629029273986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3386 , TFLOPS: 71.5368401357588, Tokens per sec: 177276.1702898986, Loss: 2.5825998783111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3387 , TFLOPS: 70.66374366515502, Tokens per sec: 175112.54105622642, Loss: 2.542616128921509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3388 , TFLOPS: 69.36908060905986, Tokens per sec: 171904.22338431346, Loss: 2.5579090118408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3389 , TFLOPS: 71.50528320389257, Tokens per sec: 177197.9687364511, Loss: 2.580704927444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3390 , TFLOPS: 71.52318206319354, Tokens per sec: 177242.32408152104, Loss: 2.5539777278900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3391 , TFLOPS: 68.72827958647416, Tokens per sec: 170316.24786028606, Loss: 2.549830913543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3392 , TFLOPS: 70.44260494077743, Tokens per sec: 174564.5349367772, Loss: 2.5568058490753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3393 , TFLOPS: 71.50766213820827, Tokens per sec: 177203.8639977467, Loss: 2.560368061065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3394 , TFLOPS: 70.65991948063804, Tokens per sec: 175103.06430572463, Loss: 2.5698394775390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3395 , TFLOPS: 69.77679196670668, Tokens per sec: 172914.577041676, Loss: 2.564497947692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3396 , TFLOPS: 70.41595137292407, Tokens per sec: 174498.48445950376, Loss: 2.571291923522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3397 , TFLOPS: 71.51230047364623, Tokens per sec: 177215.3583039168, Loss: 2.577925205230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3398 , TFLOPS: 69.4846203144044, Tokens per sec: 172190.54350190648, Loss: 2.5785789489746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3399 , TFLOPS: 70.49013920530285, Tokens per sec: 174682.33008060275, Loss: 2.558272123336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3400 , TFLOPS: 71.4218376388861, Tokens per sec: 176991.18143407724, Loss: 2.5564489364624023 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/model.pt +[rank0]:[2024-08-26 09:12:59,266] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006440389002818847, 'preprocessing_with_comm': 0.0013600999991467688, 'state_converting': 0.9924663779966068, : 1.0016746910005168}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400, took 4.64s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001400 + eval ppl=11.496617317199707, eval loss=2.4420528411865234 +------------------------------------------------------------------ +iteration: 3401 , TFLOPS: 64.57462557911333, Tokens per sec: 160023.03566728422, Loss: 2.5847666263580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3402 , TFLOPS: 70.07161671192574, Tokens per sec: 173645.18523795562, Loss: 2.57318377494812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3403 , TFLOPS: 71.49386288639049, Tokens per sec: 177169.6679316291, Loss: 2.5679311752319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3404 , TFLOPS: 69.69513732877313, Tokens per sec: 172712.22785387575, Loss: 2.573514699935913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3405 , TFLOPS: 70.11506234554999, Tokens per sec: 173752.8483040081, Loss: 2.5763161182403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3406 , TFLOPS: 70.41417866807794, Tokens per sec: 174494.09150161545, Loss: 2.595917224884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3407 , TFLOPS: 71.49058292334217, Tokens per sec: 177161.5398218776, Loss: 2.567060708999634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3408 , TFLOPS: 69.52916222922796, Tokens per sec: 172300.92327353626, Loss: 2.546304702758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3409 , TFLOPS: 71.5008644152603, Tokens per sec: 177187.01849145058, Loss: 2.5476138591766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3410 , TFLOPS: 70.39275649470511, Tokens per sec: 174441.00499615038, Loss: 2.558783531188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3411 , TFLOPS: 68.62579397000624, Tokens per sec: 170062.27721295593, Loss: 2.5297186374664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3412 , TFLOPS: 71.49019099370868, Tokens per sec: 177160.56857707124, Loss: 2.567237377166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3413 , TFLOPS: 71.51329383651586, Tokens per sec: 177217.81996653564, Loss: 2.5789260864257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3414 , TFLOPS: 68.21706305463681, Tokens per sec: 169049.3969792432, Loss: 2.552738666534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3415 , TFLOPS: 70.92483147002903, Tokens per sec: 175759.54539790022, Loss: 2.5622589588165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3416 , TFLOPS: 71.43740804380668, Tokens per sec: 177029.76661269215, Loss: 2.573143243789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3417 , TFLOPS: 69.51133338482826, Tokens per sec: 172256.74143310444, Loss: 2.584540367126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3418 , TFLOPS: 69.56366339372795, Tokens per sec: 172386.42095978424, Loss: 2.55120587348938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3419 , TFLOPS: 71.51480635354912, Tokens per sec: 177221.5681503616, Loss: 2.553274393081665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3420 , TFLOPS: 71.5342501215548, Tokens per sec: 177269.75194938754, Loss: 2.563662528991699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3421 , TFLOPS: 67.79477553993394, Tokens per sec: 168002.92199899955, Loss: 2.5991287231445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3422 , TFLOPS: 71.50430797767095, Tokens per sec: 177195.55201844388, Loss: 2.5755512714385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3423 , TFLOPS: 71.51207517949608, Tokens per sec: 177214.80000019446, Loss: 2.556898593902588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3424 , TFLOPS: 68.6024697890367, Tokens per sec: 170004.47732314278, Loss: 2.5550920963287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3425 , TFLOPS: 70.43565041091333, Tokens per sec: 174547.3008456693, Loss: 2.559014081954956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3426 , TFLOPS: 71.54171574135677, Tokens per sec: 177288.2525776627, Loss: 2.5606939792633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3427 , TFLOPS: 71.53825182329007, Tokens per sec: 177279.66860711132, Loss: 2.543255090713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3428 , TFLOPS: 68.60717112358653, Tokens per sec: 170016.12774805198, Loss: 2.5807478427886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3429 , TFLOPS: 71.51295243795981, Tokens per sec: 177216.9739433056, Loss: 2.557675361633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3430 , TFLOPS: 71.4835926540383, Tokens per sec: 177144.2171645009, Loss: 2.5786666870117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3431 , TFLOPS: 68.5968776739536, Tokens per sec: 169990.61944594453, Loss: 2.577357292175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3432 , TFLOPS: 70.38426241413782, Tokens per sec: 174419.95572880443, Loss: 2.5775575637817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3433 , TFLOPS: 71.52525354598251, Tokens per sec: 177247.45744406476, Loss: 2.555417776107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3434 , TFLOPS: 70.69411731465905, Tokens per sec: 175187.81030563108, Loss: 2.559540033340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3435 , TFLOPS: 69.66987797671304, Tokens per sec: 172649.6323969235, Loss: 2.5704307556152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3436 , TFLOPS: 70.40530615543534, Tokens per sec: 174472.10443789914, Loss: 2.557542085647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3437 , TFLOPS: 70.54579259459821, Tokens per sec: 174820.24531000358, Loss: 2.556504249572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3438 , TFLOPS: 69.6468873986508, Tokens per sec: 172592.65921186405, Loss: 2.573904514312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3439 , TFLOPS: 70.47028190819553, Tokens per sec: 174633.12151090894, Loss: 2.585233211517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3440 , TFLOPS: 71.51607270028828, Tokens per sec: 177224.7062970808, Loss: 2.56717586517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3441 , TFLOPS: 68.73665478072968, Tokens per sec: 170337.00251425506, Loss: 2.568286895751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3442 , TFLOPS: 71.49986133606828, Tokens per sec: 177184.53275071445, Loss: 2.5667643547058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3443 , TFLOPS: 70.44024654974459, Tokens per sec: 174558.69058399965, Loss: 2.5506882667541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3444 , TFLOPS: 70.60626902407391, Tokens per sec: 174970.112564273, Loss: 2.58750581741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3445 , TFLOPS: 69.91282882535033, Tokens per sec: 173251.69136309205, Loss: 2.586130142211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3446 , TFLOPS: 71.45584243416398, Tokens per sec: 177075.44906271328, Loss: 2.565753221511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3447 , TFLOPS: 70.37001198340707, Tokens per sec: 174384.64159163908, Loss: 2.5732316970825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3448 , TFLOPS: 69.56791240829855, Tokens per sec: 172396.95048595776, Loss: 2.5716943740844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3449 , TFLOPS: 71.45840655810797, Tokens per sec: 177081.80324430804, Loss: 2.5713248252868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3450 , TFLOPS: 70.51913883055732, Tokens per sec: 174754.1943465558, Loss: 2.561283588409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3451 , TFLOPS: 69.73507219427182, Tokens per sec: 172811.19084977012, Loss: 2.5483853816986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3452 , TFLOPS: 70.61048581775229, Tokens per sec: 174980.56224352625, Loss: 2.5760338306427 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3453 , TFLOPS: 71.50817648430319, Tokens per sec: 177205.13860402975, Loss: 2.5637869834899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3454 , TFLOPS: 69.5611724176741, Tokens per sec: 172380.24804671912, Loss: 2.5621182918548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3455 , TFLOPS: 69.52975928639334, Tokens per sec: 172302.40284696332, Loss: 2.5471253395080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3456 , TFLOPS: 71.54078049971744, Tokens per sec: 177285.9349458551, Loss: 2.542943000793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3457 , TFLOPS: 71.47763159639258, Tokens per sec: 177129.4450069901, Loss: 2.569517135620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3458 , TFLOPS: 68.11456179335775, Tokens per sec: 168795.38756234766, Loss: 2.5801351070404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3459 , TFLOPS: 71.50487796943074, Tokens per sec: 177196.96452081442, Loss: 2.5889482498168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3460 , TFLOPS: 71.46570852726656, Tokens per sec: 177099.89832826133, Loss: 2.574892044067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3461 , TFLOPS: 68.61729754599322, Tokens per sec: 170041.22213829332, Loss: 2.5760233402252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3462 , TFLOPS: 71.53221132567246, Tokens per sec: 177264.69958859918, Loss: 2.5781197547912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3463 , TFLOPS: 71.52177834243061, Tokens per sec: 177238.84550683788, Loss: 2.568802833557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3464 , TFLOPS: 70.58669148277173, Tokens per sec: 174921.597260282, Loss: 2.5664894580841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3465 , TFLOPS: 68.59250361453522, Tokens per sec: 169979.78004486335, Loss: 2.5637423992156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3466 , TFLOPS: 71.52347994342033, Tokens per sec: 177243.06226153724, Loss: 2.5691068172454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3467 , TFLOPS: 71.50359849926511, Tokens per sec: 177193.7938527987, Loss: 2.5807437896728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3468 , TFLOPS: 69.58378459450121, Tokens per sec: 172436.28351183364, Loss: 2.5567619800567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3469 , TFLOPS: 70.50699288618293, Tokens per sec: 174724.09535840977, Loss: 2.571685314178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3470 , TFLOPS: 71.54987614924185, Tokens per sec: 177308.47496734507, Loss: 2.575406312942505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3471 , TFLOPS: 69.75392084602285, Tokens per sec: 172857.89988515957, Loss: 2.55226731300354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3472 , TFLOPS: 70.42417361014672, Tokens per sec: 174518.86006909568, Loss: 2.574720859527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3473 , TFLOPS: 70.50289461015973, Tokens per sec: 174713.93937896195, Loss: 2.5632517337799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3474 , TFLOPS: 71.46426583286491, Tokens per sec: 177096.32317260813, Loss: 2.5462536811828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3475 , TFLOPS: 68.953167428558, Tokens per sec: 170873.5447064108, Loss: 2.538701057434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3476 , TFLOPS: 70.369600209473, Tokens per sec: 174383.62117046982, Loss: 2.5491580963134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3477 , TFLOPS: 71.52438083627194, Tokens per sec: 177245.29477326508, Loss: 2.554446220397949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3478 , TFLOPS: 68.78055613320984, Tokens per sec: 170445.79490183413, Loss: 2.577970504760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3479 , TFLOPS: 71.50766796203293, Tokens per sec: 177203.8784298257, Loss: 2.5760371685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3480 , TFLOPS: 70.4920449966692, Tokens per sec: 174687.05284154878, Loss: 2.565065383911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3481 , TFLOPS: 69.60765567141863, Tokens per sec: 172495.43867005117, Loss: 2.579152822494507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3482 , TFLOPS: 70.85617315640229, Tokens per sec: 175589.4025333955, Loss: 2.5777783393859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3483 , TFLOPS: 71.48857341232033, Tokens per sec: 177156.56003219963, Loss: 2.56180477142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3484 , TFLOPS: 69.53282791852581, Tokens per sec: 172310.0072554253, Loss: 2.5678091049194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3485 , TFLOPS: 69.62951363138937, Tokens per sec: 172549.60510271176, Loss: 2.5460779666900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3486 , TFLOPS: 71.50991861319224, Tokens per sec: 177209.4557913282, Loss: 2.5674099922180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3487 , TFLOPS: 71.51352302154211, Tokens per sec: 177218.3879122775, Loss: 2.5793371200561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3488 , TFLOPS: 68.65610304622085, Tokens per sec: 170137.38644263608, Loss: 2.5541515350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3489 , TFLOPS: 71.54019639214162, Tokens per sec: 177284.48746293742, Loss: 2.5429351329803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3490 , TFLOPS: 71.4275970969765, Tokens per sec: 177005.4540056819, Loss: 2.5661230087280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3491 , TFLOPS: 68.7010407125898, Tokens per sec: 170248.74693019097, Loss: 2.5490875244140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3492 , TFLOPS: 70.36674791059974, Tokens per sec: 174376.55285965523, Loss: 2.5653066635131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3493 , TFLOPS: 71.5446361314455, Tokens per sec: 177295.48962601012, Loss: 2.545041799545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3494 , TFLOPS: 71.53031618847373, Tokens per sec: 177260.00322985413, Loss: 2.594766855239868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3495 , TFLOPS: 68.43145260227242, Tokens per sec: 169580.67789524255, Loss: 2.5521676540374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3496 , TFLOPS: 71.46898203077028, Tokens per sec: 177108.01043054974, Loss: 2.569265365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3497 , TFLOPS: 71.52238774415125, Tokens per sec: 177240.35567143277, Loss: 2.5668070316314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3498 , TFLOPS: 67.76654060646898, Tokens per sec: 167932.95272353865, Loss: 2.584052085876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3499 , TFLOPS: 71.51235419903975, Tokens per sec: 177215.49144136024, Loss: 2.5728306770324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3500 , TFLOPS: 71.50284522195845, Tokens per sec: 177191.9271486525, Loss: 2.5564606189727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3501 , TFLOPS: 70.67415536984507, Tokens per sec: 175138.3424073923, Loss: 2.5527772903442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3502 , TFLOPS: 68.72253411745876, Tokens per sec: 170302.00995515028, Loss: 2.5533764362335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3503 , TFLOPS: 71.54055307336287, Tokens per sec: 177285.37135829538, Loss: 2.5631275177001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3504 , TFLOPS: 70.58920650156149, Tokens per sec: 174927.82975389523, Loss: 2.554201602935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3505 , TFLOPS: 69.59465025277032, Tokens per sec: 172463.20980997555, Loss: 2.563894748687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3506 , TFLOPS: 70.43257300277368, Tokens per sec: 174539.67468929474, Loss: 2.5515990257263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3507 , TFLOPS: 71.51161991071179, Tokens per sec: 177213.67179399502, Loss: 2.553072452545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3508 , TFLOPS: 69.5745428281195, Tokens per sec: 172413.3813966751, Loss: 2.5475966930389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3509 , TFLOPS: 70.52361811171305, Tokens per sec: 174765.29449869323, Loss: 2.55594539642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3510 , TFLOPS: 71.49286745665073, Tokens per sec: 177167.20114707833, Loss: 2.5671112537384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3511 , TFLOPS: 69.7530615104222, Tokens per sec: 172855.7703568775, Loss: 2.587376117706299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3512 , TFLOPS: 70.40314681684748, Tokens per sec: 174466.75335900765, Loss: 2.5621256828308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3513 , TFLOPS: 70.58427950687893, Tokens per sec: 174915.6201183186, Loss: 2.5758533477783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3514 , TFLOPS: 71.52227065923283, Tokens per sec: 177240.06552210695, Loss: 2.5665390491485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3515 , TFLOPS: 68.77638664179503, Tokens per sec: 170435.46244280084, Loss: 2.5833420753479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3516 , TFLOPS: 71.49921587706531, Tokens per sec: 177182.93323220193, Loss: 2.5856964588165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3517 , TFLOPS: 70.42583071010259, Tokens per sec: 174522.96654533094, Loss: 2.571162700653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3518 , TFLOPS: 69.80819890582758, Tokens per sec: 172992.40689657757, Loss: 2.551877975463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3519 , TFLOPS: 70.5824417261768, Tokens per sec: 174911.06589529043, Loss: 2.5916597843170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3520 , TFLOPS: 71.49575463458658, Tokens per sec: 177174.3558920523, Loss: 2.5719470977783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3521 , TFLOPS: 69.72803231903562, Tokens per sec: 172793.74526342851, Loss: 2.5679562091827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3522 , TFLOPS: 69.57821425514119, Tokens per sec: 172422.47959727593, Loss: 2.559185028076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3523 , TFLOPS: 71.50739684070847, Tokens per sec: 177203.2065613172, Loss: 2.5505309104919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3524 , TFLOPS: 70.39588180196621, Tokens per sec: 174448.7498518244, Loss: 2.588902235031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3525 , TFLOPS: 68.87861074219362, Tokens per sec: 170688.7850245028, Loss: 2.5695080757141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3526 , TFLOPS: 71.54369776806402, Tokens per sec: 177293.16425818155, Loss: 2.5571649074554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3527 , TFLOPS: 71.47638035171018, Tokens per sec: 177126.34428483114, Loss: 2.559804677963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3528 , TFLOPS: 68.62711350923955, Tokens per sec: 170065.54717653533, Loss: 2.5540482997894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3529 , TFLOPS: 71.48495214696385, Tokens per sec: 177147.58613773092, Loss: 2.5624630451202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3530 , TFLOPS: 71.48873196241597, Tokens per sec: 177156.95293680232, Loss: 2.577090263366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3531 , TFLOPS: 70.68898739023149, Tokens per sec: 175175.09778779178, Loss: 2.559295415878296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3532 , TFLOPS: 69.2573249145274, Tokens per sec: 171627.28046235596, Loss: 2.5719356536865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3533 , TFLOPS: 71.50121012102169, Tokens per sec: 177187.8751884099, Loss: 2.5695409774780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3534 , TFLOPS: 71.50540943680485, Tokens per sec: 177198.28155551344, Loss: 2.548996925354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3535 , TFLOPS: 68.69565139434323, Tokens per sec: 170235.39160589356, Loss: 2.564845085144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3536 , TFLOPS: 70.49090635046674, Tokens per sec: 174684.2311508269, Loss: 2.5684051513671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3537 , TFLOPS: 71.47712714649363, Tokens per sec: 177128.19492457077, Loss: 2.5679123401641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3538 , TFLOPS: 69.82578416456404, Tokens per sec: 173035.98510490265, Loss: 2.5498032569885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3539 , TFLOPS: 69.12078824556349, Tokens per sec: 171288.9275559077, Loss: 2.551464557647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3540 , TFLOPS: 71.52438615071972, Tokens per sec: 177245.3079430521, Loss: 2.5655291080474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3541 , TFLOPS: 71.52365348015213, Tokens per sec: 177243.4923046753, Loss: 2.5575037002563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3542 , TFLOPS: 68.97165421561637, Tokens per sec: 170919.3570012291, Loss: 2.563171148300171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3543 , TFLOPS: 69.38765479874856, Tokens per sec: 171950.25227247798, Loss: 2.567718744277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3544 , TFLOPS: 71.50722401058583, Tokens per sec: 177202.7782692345, Loss: 2.5519344806671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3545 , TFLOPS: 68.75281688900606, Tokens per sec: 170377.05399896714, Loss: 2.5709547996520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3546 , TFLOPS: 70.56208209574801, Tokens per sec: 174860.61248828948, Loss: 2.5578322410583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3547 , TFLOPS: 71.44042650318798, Tokens per sec: 177037.24668754978, Loss: 2.5744826793670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3548 , TFLOPS: 70.6216790040273, Tokens per sec: 175008.30019214697, Loss: 2.559906005859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3549 , TFLOPS: 70.61657783120079, Tokens per sec: 174995.65892394228, Loss: 2.5492427349090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3550 , TFLOPS: 70.2161220663499, Tokens per sec: 174003.28542479756, Loss: 2.577500581741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3551 , TFLOPS: 70.67145796199613, Tokens per sec: 175131.65793359862, Loss: 2.56319260597229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3552 , TFLOPS: 68.93155386191023, Tokens per sec: 170819.98390732668, Loss: 2.5708584785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3553 , TFLOPS: 71.5116749101349, Tokens per sec: 177213.80808862418, Loss: 2.576047897338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3554 , TFLOPS: 70.5415220739713, Tokens per sec: 174809.6624894534, Loss: 2.563444137573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3555 , TFLOPS: 68.69235427566069, Tokens per sec: 170227.22098259063, Loss: 2.577700138092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3556 , TFLOPS: 71.50769503372446, Tokens per sec: 177203.9455164595, Loss: 2.562124252319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3557 , TFLOPS: 71.5299588168624, Tokens per sec: 177259.11762363356, Loss: 2.555030345916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3558 , TFLOPS: 68.7048335967109, Tokens per sec: 170258.14611486375, Loss: 2.5436999797821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3559 , TFLOPS: 70.42265239412157, Tokens per sec: 174515.09032820957, Loss: 2.5870871543884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3560 , TFLOPS: 71.51777309269987, Tokens per sec: 177228.92005679043, Loss: 2.5587000846862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3561 , TFLOPS: 70.50798647002081, Tokens per sec: 174726.55756861233, Loss: 2.552548408508301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3562 , TFLOPS: 68.73094021122998, Tokens per sec: 170322.841181523, Loss: 2.579011917114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3563 , TFLOPS: 71.54481720568832, Tokens per sec: 177295.93834793076, Loss: 2.5846896171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3564 , TFLOPS: 71.5196508318771, Tokens per sec: 177233.57330132165, Loss: 2.570162057876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3565 , TFLOPS: 68.35387965222681, Tokens per sec: 169388.44357966367, Loss: 2.5660173892974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3566 , TFLOPS: 71.5141245740845, Tokens per sec: 177219.87862574405, Loss: 2.5649421215057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3567 , TFLOPS: 71.5085947176956, Tokens per sec: 177206.17503244837, Loss: 2.553135871887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3568 , TFLOPS: 70.57810378590221, Tokens per sec: 174900.31600142646, Loss: 2.5579497814178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3569 , TFLOPS: 69.37644268725906, Tokens per sec: 171922.46742509093, Loss: 2.571537494659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3570 , TFLOPS: 71.48381148554387, Tokens per sec: 177144.75945307798, Loss: 2.5482609272003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3571 , TFLOPS: 71.47466100755145, Tokens per sec: 177122.08356060443, Loss: 2.561828851699829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3572 , TFLOPS: 68.02349848158461, Tokens per sec: 168569.72264432188, Loss: 2.5500218868255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3573 , TFLOPS: 71.50958268469228, Tokens per sec: 177208.6233234997, Loss: 2.552119493484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3574 , TFLOPS: 71.50494087746037, Tokens per sec: 177197.12041384174, Loss: 2.590151786804199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3575 , TFLOPS: 68.62099609022646, Tokens per sec: 170050.38753833153, Loss: 2.558560848236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3576 , TFLOPS: 70.53155552098016, Tokens per sec: 174784.96427323492, Loss: 2.5736446380615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3577 , TFLOPS: 71.51729084632021, Tokens per sec: 177227.72499713782, Loss: 2.5560619831085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3578 , TFLOPS: 69.79634554746788, Tokens per sec: 172963.03296880698, Loss: 2.5667290687561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3579 , TFLOPS: 70.39500801781318, Tokens per sec: 174446.58451843794, Loss: 2.5626118183135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3580 , TFLOPS: 70.40550242467214, Tokens per sec: 174472.59081468877, Loss: 2.559256076812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3581 , TFLOPS: 71.5568601425865, Tokens per sec: 177325.7820722035, Loss: 2.577388048171997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3582 , TFLOPS: 68.0208255689943, Tokens per sec: 168563.09887247748, Loss: 2.582019805908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3583 , TFLOPS: 70.48328187255714, Tokens per sec: 174665.33685466135, Loss: 2.574821710586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3584 , TFLOPS: 71.45808270630805, Tokens per sec: 177081.00070387265, Loss: 2.575864315032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3585 , TFLOPS: 69.8012766299183, Tokens per sec: 172975.25273432274, Loss: 2.552419900894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3586 , TFLOPS: 70.68395357920001, Tokens per sec: 175162.6234495357, Loss: 2.5736124515533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3587 , TFLOPS: 70.48125378037494, Tokens per sec: 174660.31101882103, Loss: 2.5716967582702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3588 , TFLOPS: 70.58167800729107, Tokens per sec: 174909.17331576103, Loss: 2.568563461303711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3589 , TFLOPS: 70.36335482818434, Tokens per sec: 174368.144427651, Loss: 2.591315746307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3590 , TFLOPS: 71.52276427752307, Tokens per sec: 177241.2887626067, Loss: 2.561408042907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3591 , TFLOPS: 70.50607836123426, Tokens per sec: 174721.82906484374, Loss: 2.5491867065429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3592 , TFLOPS: 68.72119769565974, Tokens per sec: 170298.6981547133, Loss: 2.556701898574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3593 , TFLOPS: 71.5201511673912, Tokens per sec: 177234.8131878421, Loss: 2.5645945072174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3594 , TFLOPS: 71.49060737565544, Tokens per sec: 177161.60041740356, Loss: 2.5549535751342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3595 , TFLOPS: 67.83265499480451, Tokens per sec: 168096.79146093523, Loss: 2.572634696960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3596 , TFLOPS: 71.51690155092022, Tokens per sec: 177226.7602802522, Loss: 2.583554744720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3597 , TFLOPS: 71.5627728930935, Tokens per sec: 177340.43451930306, Loss: 2.553475856781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3598 , TFLOPS: 69.52863071481413, Tokens per sec: 172299.6061222676, Loss: 2.5319297313690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3599 , TFLOPS: 69.74679934902646, Tokens per sec: 172840.25203110487, Loss: 2.5798611640930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3600 , TFLOPS: 71.53349096951743, Tokens per sec: 177267.8706870114, Loss: 2.573122501373291 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/model.pt +[rank0]:[2024-08-26 09:38:09,867] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006399936999514466, 'preprocessing_with_comm': 0.0013913289985794108, 'state_converting': 0.9769180490002327, : 0.9860729530009849}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001600 + eval ppl=10.994260787963867, eval loss=2.3973734378814697 +------------------------------------------------------------------ +iteration: 3601 , TFLOPS: 66.55908645353462, Tokens per sec: 164940.74832051195, Loss: 2.585714340209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3602 , TFLOPS: 67.46878644818477, Tokens per sec: 167195.08511898213, Loss: 2.5401618480682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3603 , TFLOPS: 71.49963477811409, Tokens per sec: 177183.97131514654, Loss: 2.5562455654144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3604 , TFLOPS: 71.54725091510898, Tokens per sec: 177301.96934797041, Loss: 2.5884785652160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3605 , TFLOPS: 69.58294141682832, Tokens per sec: 172434.19402467652, Loss: 2.572054386138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3606 , TFLOPS: 70.39488260494419, Tokens per sec: 174446.27373153318, Loss: 2.572401523590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3607 , TFLOPS: 71.54671066519407, Tokens per sec: 177300.63054916784, Loss: 2.561525344848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3608 , TFLOPS: 70.52816651018237, Tokens per sec: 174776.56593114766, Loss: 2.5728280544281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3609 , TFLOPS: 68.84023392285731, Tokens per sec: 170593.68303863177, Loss: 2.550248384475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3610 , TFLOPS: 71.49170527526341, Tokens per sec: 177164.32113357572, Loss: 2.5611724853515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3611 , TFLOPS: 71.48793622682543, Tokens per sec: 177154.98101635143, Loss: 2.577051877975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3612 , TFLOPS: 69.01943008184243, Tokens per sec: 171037.75086068342, Loss: 2.5777807235717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3613 , TFLOPS: 70.39002534574672, Tokens per sec: 174434.2369081706, Loss: 2.549093723297119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3614 , TFLOPS: 71.53522326507982, Tokens per sec: 177272.16350624315, Loss: 2.5670359134674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3615 , TFLOPS: 70.40150779424016, Tokens per sec: 174462.69168043346, Loss: 2.5668888092041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3616 , TFLOPS: 70.5933412305482, Tokens per sec: 174938.0760678023, Loss: 2.551027774810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3617 , TFLOPS: 70.40272842934695, Tokens per sec: 174465.71654869214, Loss: 2.562314510345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3618 , TFLOPS: 70.69749246112809, Tokens per sec: 175196.1742903845, Loss: 2.5564327239990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3619 , TFLOPS: 70.40067154064474, Tokens per sec: 174460.61935190306, Loss: 2.5413312911987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3620 , TFLOPS: 71.50371387170789, Tokens per sec: 177194.07975842126, Loss: 2.5618529319763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3621 , TFLOPS: 70.4006552892756, Tokens per sec: 174460.5790792201, Loss: 2.568779945373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3622 , TFLOPS: 68.866501264105, Tokens per sec: 170658.77640383007, Loss: 2.5593388080596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3623 , TFLOPS: 71.49575914342061, Tokens per sec: 177174.36706543973, Loss: 2.5400729179382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3624 , TFLOPS: 70.42654190466733, Tokens per sec: 174524.72896380693, Loss: 2.570540428161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3625 , TFLOPS: 69.53172432307517, Tokens per sec: 172307.27242432756, Loss: 2.5839548110961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3626 , TFLOPS: 71.49692262488522, Tokens per sec: 177177.2503006771, Loss: 2.5600638389587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3627 , TFLOPS: 71.50818665445554, Tokens per sec: 177205.16380678763, Loss: 2.5303215980529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3628 , TFLOPS: 70.39806242113677, Tokens per sec: 174454.1536663441, Loss: 2.566749095916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3629 , TFLOPS: 69.02214076894391, Tokens per sec: 171044.46824192774, Loss: 2.567702054977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3630 , TFLOPS: 71.48930657101006, Tokens per sec: 177158.37688019688, Loss: 2.554226875305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3631 , TFLOPS: 70.36596695885856, Tokens per sec: 174374.61757521163, Loss: 2.548008918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3632 , TFLOPS: 68.69059044060226, Tokens per sec: 170222.85000501337, Loss: 2.5685017108917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3633 , TFLOPS: 71.54454764916619, Tokens per sec: 177295.27035717486, Loss: 2.564983367919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3634 , TFLOPS: 71.52205708018549, Tokens per sec: 177239.5362497, Loss: 2.5730741024017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3635 , TFLOPS: 69.34755481446771, Tokens per sec: 171850.8800940521, Loss: 2.563251495361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3636 , TFLOPS: 70.62990497873429, Tokens per sec: 175028.6850636368, Loss: 2.5588316917419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3637 , TFLOPS: 71.51727006612498, Tokens per sec: 177227.6735015249, Loss: 2.553946018218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3638 , TFLOPS: 70.56035783787851, Tokens per sec: 174856.33958734618, Loss: 2.5615503787994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3639 , TFLOPS: 70.3716173351151, Tokens per sec: 174388.6198300157, Loss: 2.567035436630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3640 , TFLOPS: 71.51468171031982, Tokens per sec: 177221.2592707088, Loss: 2.588078737258911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3641 , TFLOPS: 71.5201921804037, Tokens per sec: 177234.914822605, Loss: 2.5642173290252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3642 , TFLOPS: 69.45218201305906, Tokens per sec: 172110.15781779887, Loss: 2.562673568725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3643 , TFLOPS: 69.89624660761177, Tokens per sec: 173210.59880085383, Loss: 2.5788912773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3644 , TFLOPS: 71.47689956493701, Tokens per sec: 177127.63095240327, Loss: 2.5422487258911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3645 , TFLOPS: 70.54348486058875, Tokens per sec: 174814.52649091024, Loss: 2.5485520362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3646 , TFLOPS: 68.96735781147385, Tokens per sec: 170908.71003847595, Loss: 2.573810338973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3647 , TFLOPS: 71.50756897622186, Tokens per sec: 177203.6331320817, Loss: 2.5780115127563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3648 , TFLOPS: 71.48078644438635, Tokens per sec: 177137.26306785384, Loss: 2.5900397300720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3649 , TFLOPS: 68.71601520533476, Tokens per sec: 170285.85537278955, Loss: 2.5509915351867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3650 , TFLOPS: 70.52168271380161, Tokens per sec: 174760.49836946698, Loss: 2.5705227851867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3651 , TFLOPS: 71.46670592133259, Tokens per sec: 177102.369980629, Loss: 2.5502965450286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3652 , TFLOPS: 68.63554665377866, Tokens per sec: 170086.4454376908, Loss: 2.5387816429138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3653 , TFLOPS: 71.55399961173222, Tokens per sec: 177318.6933616331, Loss: 2.5536348819732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3654 , TFLOPS: 70.43428809741167, Tokens per sec: 174543.92488274095, Loss: 2.5617544651031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3655 , TFLOPS: 70.37038541343341, Tokens per sec: 174385.5669923811, Loss: 2.560549736022949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3656 , TFLOPS: 70.5706690827317, Tokens per sec: 174881.89198796978, Loss: 2.556368827819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3657 , TFLOPS: 71.50386824945416, Tokens per sec: 177194.46232348264, Loss: 2.5898659229278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3658 , TFLOPS: 69.67782201494882, Tokens per sec: 172669.31859877982, Loss: 2.5524072647094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3659 , TFLOPS: 70.40708728043904, Tokens per sec: 174476.51826186542, Loss: 2.5497257709503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3660 , TFLOPS: 71.53347585333158, Tokens per sec: 177267.8332274377, Loss: 2.5643088817596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3661 , TFLOPS: 70.42868299465974, Tokens per sec: 174530.03482066863, Loss: 2.5599465370178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3662 , TFLOPS: 70.3544181657173, Tokens per sec: 174345.99839360302, Loss: 2.5863072872161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3663 , TFLOPS: 70.87664841280962, Tokens per sec: 175640.14247430858, Loss: 2.5512447357177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3664 , TFLOPS: 71.54006107515548, Tokens per sec: 177284.15213253937, Loss: 2.5637450218200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3665 , TFLOPS: 69.62965735748809, Tokens per sec: 172549.96127181713, Loss: 2.5541656017303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3666 , TFLOPS: 70.40265081896256, Tokens per sec: 174465.52422161013, Loss: 2.5574636459350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3667 , TFLOPS: 71.5478715125452, Tokens per sec: 177303.50725678183, Loss: 2.561955690383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3668 , TFLOPS: 70.46627193296436, Tokens per sec: 174623.18435055087, Loss: 2.5521678924560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3669 , TFLOPS: 68.63678177733145, Tokens per sec: 170089.50620991472, Loss: 2.565467596054077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3670 , TFLOPS: 71.52492239170984, Tokens per sec: 177246.63680729741, Loss: 2.5701076984405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3671 , TFLOPS: 71.52454912090444, Tokens per sec: 177245.71180112247, Loss: 2.5755455493927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3672 , TFLOPS: 68.51205722878751, Tokens per sec: 169780.42503907956, Loss: 2.5809988975524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3673 , TFLOPS: 71.51937524307776, Tokens per sec: 177232.89036191828, Loss: 2.5659945011138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3674 , TFLOPS: 71.51108594590205, Tokens per sec: 177212.3485703754, Loss: 2.5577783584594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3675 , TFLOPS: 70.37503622854197, Tokens per sec: 174397.09222454965, Loss: 2.564692497253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3676 , TFLOPS: 69.7708013264904, Tokens per sec: 172899.73157529658, Loss: 2.5829739570617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3677 , TFLOPS: 71.49278701869382, Tokens per sec: 177167.00181296017, Loss: 2.566617250442505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3678 , TFLOPS: 71.54230138193958, Tokens per sec: 177289.70385954052, Loss: 2.5590224266052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3679 , TFLOPS: 69.86186771914424, Tokens per sec: 173125.4041280841, Loss: 2.5535390377044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3680 , TFLOPS: 70.39512401505868, Tokens per sec: 174446.87197239045, Loss: 2.5462305545806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3681 , TFLOPS: 71.49228013444409, Tokens per sec: 177165.7456979514, Loss: 2.545980215072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3682 , TFLOPS: 70.36911870429968, Tokens per sec: 174382.42794760814, Loss: 2.5777647495269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3683 , TFLOPS: 69.66246166882411, Tokens per sec: 172631.25396900097, Loss: 2.561293601989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3684 , TFLOPS: 71.4823205469685, Tokens per sec: 177141.06474304778, Loss: 2.5461463928222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3685 , TFLOPS: 70.70435026800996, Tokens per sec: 175213.16869128754, Loss: 2.5668420791625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3686 , TFLOPS: 70.3499967522332, Tokens per sec: 174335.04164393048, Loss: 2.545518398284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3687 , TFLOPS: 70.46129986396551, Tokens per sec: 174610.8630158531, Loss: 2.573836326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3688 , TFLOPS: 71.51606854634386, Tokens per sec: 177224.69600314894, Loss: 2.537022829055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3689 , TFLOPS: 68.69418576220085, Tokens per sec: 170231.75960799202, Loss: 2.57963228225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3690 , TFLOPS: 71.52990722011945, Tokens per sec: 177258.98976122064, Loss: 2.575718402862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3691 , TFLOPS: 70.49970497884362, Tokens per sec: 174706.0351211937, Loss: 2.567056179046631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3692 , TFLOPS: 69.63786977031491, Tokens per sec: 172570.31253547393, Loss: 2.5958755016326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3693 , TFLOPS: 70.80817836956602, Tokens per sec: 175470.466164553, Loss: 2.573014259338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3694 , TFLOPS: 71.52176877713681, Tokens per sec: 177238.82180298615, Loss: 2.582347869873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3695 , TFLOPS: 70.43306185137422, Tokens per sec: 174540.886109978, Loss: 2.5583605766296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3696 , TFLOPS: 69.88412484048776, Tokens per sec: 173180.55972659762, Loss: 2.5426061153411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3697 , TFLOPS: 71.48404172670315, Tokens per sec: 177145.33001603384, Loss: 2.567568063735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3698 , TFLOPS: 70.56679168420897, Tokens per sec: 174872.28336729968, Loss: 2.5534403324127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3699 , TFLOPS: 69.61198140818973, Tokens per sec: 172506.15832228924, Loss: 2.550163745880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3700 , TFLOPS: 71.51652918501843, Tokens per sec: 177225.83751652806, Loss: 2.5547235012054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3701 , TFLOPS: 71.55486616834183, Tokens per sec: 177320.8407843708, Loss: 2.5714097023010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3702 , TFLOPS: 69.32056697343558, Tokens per sec: 171784.00125101834, Loss: 2.55328369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3703 , TFLOPS: 70.69254412804206, Tokens per sec: 175183.9117758934, Loss: 2.5598809719085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3704 , TFLOPS: 71.50886145353387, Tokens per sec: 177206.836033239, Loss: 2.548454999923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3705 , TFLOPS: 70.41798748172236, Tokens per sec: 174503.5301614018, Loss: 2.582780361175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3706 , TFLOPS: 70.34456876911354, Tokens per sec: 174321.5905038183, Loss: 2.556285858154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3707 , TFLOPS: 71.53776300792083, Tokens per sec: 177278.45726877893, Loss: 2.5559473037719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3708 , TFLOPS: 71.51594734744269, Tokens per sec: 177224.3956589207, Loss: 2.559504985809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3709 , TFLOPS: 67.7218210652779, Tokens per sec: 167822.13277420303, Loss: 2.586254596710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3710 , TFLOPS: 71.52485359285544, Tokens per sec: 177246.46631615743, Loss: 2.5435502529144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3711 , TFLOPS: 71.53727029418394, Tokens per sec: 177277.236269862, Loss: 2.5969796180725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3712 , TFLOPS: 70.72122792336577, Tokens per sec: 175254.99338048647, Loss: 2.568256378173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3713 , TFLOPS: 68.78980351258602, Tokens per sec: 170468.71092661124, Loss: 2.5522210597991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3714 , TFLOPS: 71.53267491716883, Tokens per sec: 177265.848419396, Loss: 2.552845001220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3715 , TFLOPS: 71.51091617916018, Tokens per sec: 177211.92786968767, Loss: 2.5709147453308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3716 , TFLOPS: 69.61359003457491, Tokens per sec: 172510.14467567654, Loss: 2.5687148571014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3717 , TFLOPS: 70.44637756456495, Tokens per sec: 174573.88391411156, Loss: 2.5813474655151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3718 , TFLOPS: 71.48992762133705, Tokens per sec: 177159.91591132144, Loss: 2.5705883502960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3719 , TFLOPS: 69.58249680834282, Tokens per sec: 172433.0922358718, Loss: 2.563777446746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3720 , TFLOPS: 70.5155143065504, Tokens per sec: 174745.21237679233, Loss: 2.5598061084747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3721 , TFLOPS: 71.46846941573402, Tokens per sec: 177106.74011402065, Loss: 2.56192946434021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3722 , TFLOPS: 70.37577565928481, Tokens per sec: 174398.92461538545, Loss: 2.567657947540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3723 , TFLOPS: 70.72237153799455, Tokens per sec: 175257.8273835161, Loss: 2.5768704414367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3724 , TFLOPS: 70.55901533347847, Tokens per sec: 174853.0127135538, Loss: 2.5555880069732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3725 , TFLOPS: 71.47465034519811, Tokens per sec: 177122.0571381183, Loss: 2.5618605613708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3726 , TFLOPS: 69.77007717424912, Tokens per sec: 172897.93704627073, Loss: 2.570340156555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3727 , TFLOPS: 71.48786062634437, Tokens per sec: 177154.79367003133, Loss: 2.560777425765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3728 , TFLOPS: 70.45814282833976, Tokens per sec: 174603.0395337963, Loss: 2.545736789703369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3729 , TFLOPS: 69.51722619654235, Tokens per sec: 172271.34446967876, Loss: 2.5759196281433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3730 , TFLOPS: 70.92976805668484, Tokens per sec: 175771.77880344322, Loss: 2.5615386962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3731 , TFLOPS: 71.51780474161394, Tokens per sec: 177228.9984862859, Loss: 2.562004804611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3732 , TFLOPS: 69.71129611932066, Tokens per sec: 172752.27111689028, Loss: 2.550772190093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3733 , TFLOPS: 69.45034505150137, Tokens per sec: 172105.60562470087, Loss: 2.539700508117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3734 , TFLOPS: 71.49339846554322, Tokens per sec: 177168.51704560942, Loss: 2.5762484073638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3735 , TFLOPS: 70.40475856790911, Tokens per sec: 174470.7474556874, Loss: 2.5361595153808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3736 , TFLOPS: 69.5969430925833, Tokens per sec: 172468.89171960927, Loss: 2.562074661254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3737 , TFLOPS: 71.52710217572428, Tokens per sec: 177252.03855220825, Loss: 2.5571208000183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3738 , TFLOPS: 71.50780177537388, Tokens per sec: 177204.21003402534, Loss: 2.5734152793884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3739 , TFLOPS: 68.55254075577845, Tokens per sec: 169880.74767859178, Loss: 2.5672333240509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3740 , TFLOPS: 71.5527797216843, Tokens per sec: 177315.67033971238, Loss: 2.557223081588745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3741 , TFLOPS: 71.50355546766376, Tokens per sec: 177193.68721575025, Loss: 2.540497303009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3742 , TFLOPS: 70.25917125247906, Tokens per sec: 174109.966050285, Loss: 2.577505588531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3743 , TFLOPS: 70.62880596082576, Tokens per sec: 175025.96157619264, Loss: 2.553797483444214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3744 , TFLOPS: 71.4936155420808, Tokens per sec: 177169.0549851825, Loss: 2.5690178871154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3745 , TFLOPS: 71.51984711229527, Tokens per sec: 177234.05970581993, Loss: 2.5311498641967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3746 , TFLOPS: 68.46305558418109, Tokens per sec: 169658.9935657681, Loss: 2.5685606002807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3747 , TFLOPS: 71.54183405005216, Tokens per sec: 177288.54575964267, Loss: 2.5617482662200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3748 , TFLOPS: 71.50214602792678, Tokens per sec: 177190.19446881942, Loss: 2.5654256343841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3749 , TFLOPS: 69.58867734518765, Tokens per sec: 172448.4082870156, Loss: 2.55851411819458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3750 , TFLOPS: 69.6313487091685, Tokens per sec: 172554.15262759896, Loss: 2.566915988922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3751 , TFLOPS: 71.52841987604047, Tokens per sec: 177255.30395874847, Loss: 2.570620059967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3752 , TFLOPS: 70.60579495242753, Tokens per sec: 174968.93776251, Loss: 2.558790445327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3753 , TFLOPS: 69.53576130902786, Tokens per sec: 172317.27652022903, Loss: 2.5700013637542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3754 , TFLOPS: 70.50193800962478, Tokens per sec: 174711.5688174581, Loss: 2.546355724334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3755 , TFLOPS: 71.54074976819811, Tokens per sec: 177285.85878976484, Loss: 2.5501179695129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3756 , TFLOPS: 69.52994477824943, Tokens per sec: 172302.86251621632, Loss: 2.548766613006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3757 , TFLOPS: 70.5446625792785, Tokens per sec: 174817.44500755987, Loss: 2.543916940689087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3758 , TFLOPS: 71.51837574077435, Tokens per sec: 177230.41348510614, Loss: 2.549955129623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3759 , TFLOPS: 69.50308175426659, Tokens per sec: 172236.29298358047, Loss: 2.5616631507873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3760 , TFLOPS: 71.53761504716117, Tokens per sec: 177278.09060571724, Loss: 2.550903081893921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3761 , TFLOPS: 70.4107171845311, Tokens per sec: 174485.5135640725, Loss: 2.548431634902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3762 , TFLOPS: 71.53810172417975, Tokens per sec: 177279.2966449812, Loss: 2.5488271713256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3763 , TFLOPS: 69.66520556529228, Tokens per sec: 172638.05364671428, Loss: 2.535064458847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3764 , TFLOPS: 71.53677593432211, Tokens per sec: 177276.01119166616, Loss: 2.558880090713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3765 , TFLOPS: 70.5022662933023, Tokens per sec: 174712.3823405744, Loss: 2.5547072887420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3766 , TFLOPS: 69.60803199963955, Tokens per sec: 172496.3712528386, Loss: 2.5668387413024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3767 , TFLOPS: 71.52495933248355, Tokens per sec: 177246.72835060404, Loss: 2.538119077682495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3768 , TFLOPS: 71.50505799256656, Tokens per sec: 177197.4106379762, Loss: 2.542306423187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3769 , TFLOPS: 68.38793795493704, Tokens per sec: 169472.8438056114, Loss: 2.5377588272094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3770 , TFLOPS: 70.58550473769908, Tokens per sec: 174918.65637526126, Loss: 2.5787856578826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3771 , TFLOPS: 71.44925382732578, Tokens per sec: 177059.12176917325, Loss: 2.5556843280792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3772 , TFLOPS: 69.63568753955599, Tokens per sec: 172564.90472726076, Loss: 2.5559940338134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3773 , TFLOPS: 69.62217103703132, Tokens per sec: 172531.409345039, Loss: 2.564643383026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3774 , TFLOPS: 71.52038570059486, Tokens per sec: 177235.3943869567, Loss: 2.549408435821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3775 , TFLOPS: 71.54061853828614, Tokens per sec: 177285.5335875868, Loss: 2.5706162452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3776 , TFLOPS: 68.46705225836345, Tokens per sec: 169668.897764662, Loss: 2.547506809234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3777 , TFLOPS: 71.49915575029162, Tokens per sec: 177182.78423143324, Loss: 2.5734896659851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3778 , TFLOPS: 71.51032347985814, Tokens per sec: 177210.45909552602, Loss: 2.5489368438720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3779 , TFLOPS: 69.62165820483555, Tokens per sec: 172530.1384903647, Loss: 2.5720767974853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3780 , TFLOPS: 70.50096426429937, Tokens per sec: 174709.1557692748, Loss: 2.5759449005126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3781 , TFLOPS: 71.54454559950399, Tokens per sec: 177295.26527788615, Loss: 2.5617027282714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3782 , TFLOPS: 71.46898593808828, Tokens per sec: 177108.02011331418, Loss: 2.5541229248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3783 , TFLOPS: 68.43150722024563, Tokens per sec: 169580.81324459685, Loss: 2.5529980659484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3784 , TFLOPS: 71.53310318915055, Tokens per sec: 177266.90972454473, Loss: 2.5525684356689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3785 , TFLOPS: 71.53557406834118, Tokens per sec: 177273.03283536894, Loss: 2.560774803161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3786 , TFLOPS: 69.4516720282472, Tokens per sec: 172108.8940192556, Loss: 2.557924270629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3787 , TFLOPS: 70.6033349140506, Tokens per sec: 174962.84151641637, Loss: 2.5621585845947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3788 , TFLOPS: 71.52662831681286, Tokens per sec: 177250.8642776259, Loss: 2.5781097412109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3789 , TFLOPS: 69.59535298342333, Tokens per sec: 172464.95125394585, Loss: 2.57037353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3790 , TFLOPS: 70.58729262032112, Tokens per sec: 174923.0869453502, Loss: 2.597217321395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3791 , TFLOPS: 70.29991137167913, Tokens per sec: 174210.92455355736, Loss: 2.568087577819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3792 , TFLOPS: 70.51315748292328, Tokens per sec: 174739.37190821877, Loss: 2.5824971199035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3793 , TFLOPS: 69.53540483089691, Tokens per sec: 172316.39312815122, Loss: 2.5453569889068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3794 , TFLOPS: 70.44334566092435, Tokens per sec: 174566.37052289836, Loss: 2.569607734680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3795 , TFLOPS: 71.46498426434725, Tokens per sec: 177098.10352496314, Loss: 2.545527696609497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3796 , TFLOPS: 70.44550207647104, Tokens per sec: 174571.71435817186, Loss: 2.553575038909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3797 , TFLOPS: 70.8052023233453, Tokens per sec: 175463.09119417952, Loss: 2.5631051063537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3798 , TFLOPS: 70.46098015758761, Tokens per sec: 174610.07074823012, Loss: 2.5577268600463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3799 , TFLOPS: 70.69014642317964, Tokens per sec: 175177.96999911533, Loss: 2.556854724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3800 , TFLOPS: 70.38274420367833, Tokens per sec: 174416.19343604113, Loss: 2.5579028129577637 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/model.pt +[rank0]:[2024-08-26 10:03:17,616] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006406817999959458, 'preprocessing_with_comm': 0.0012740820020553656, 'state_converting': 0.9786697540002933, : 0.9877169370010961}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800, took 4.58s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0001800 + eval ppl=10.933914184570312, eval loss=2.391869306564331 +------------------------------------------------------------------ +iteration: 3801 , TFLOPS: 65.18608808568105, Tokens per sec: 161538.30711671468, Loss: 2.5430169105529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3802 , TFLOPS: 69.77327420839417, Tokens per sec: 172905.85964906454, Loss: 2.572462320327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3803 , TFLOPS: 68.621388001002, Tokens per sec: 170051.35873640588, Loss: 2.5477724075317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3804 , TFLOPS: 71.5010297985307, Tokens per sec: 177187.4283294131, Loss: 2.5804190635681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3805 , TFLOPS: 70.39387013630139, Tokens per sec: 174443.76472270372, Loss: 2.5383355617523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3806 , TFLOPS: 68.82571518222029, Tokens per sec: 170557.70399996196, Loss: 2.552396297454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3807 , TFLOPS: 71.46609379266064, Tokens per sec: 177100.85305834835, Loss: 2.577005624771118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3808 , TFLOPS: 71.52121024544945, Tokens per sec: 177237.43769993752, Loss: 2.571852922439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3809 , TFLOPS: 69.35400515566197, Tokens per sec: 171866.86475009518, Loss: 2.5400898456573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3810 , TFLOPS: 69.93613434196013, Tokens per sec: 173309.44500056573, Loss: 2.5654096603393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3811 , TFLOPS: 71.5144359027395, Tokens per sec: 177220.6501324468, Loss: 2.5543124675750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3812 , TFLOPS: 70.38275872514394, Tokens per sec: 174416.22942183257, Loss: 2.576918125152588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3813 , TFLOPS: 69.39972039229404, Tokens per sec: 171980.15214241852, Loss: 2.5707361698150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3814 , TFLOPS: 71.49649242056356, Tokens per sec: 177176.18420697708, Loss: 2.5667471885681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3815 , TFLOPS: 71.51063781802618, Tokens per sec: 177211.23806013362, Loss: 2.560802698135376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3816 , TFLOPS: 67.45759495216176, Tokens per sec: 167167.35135899205, Loss: 2.562809944152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3817 , TFLOPS: 71.46292899412819, Tokens per sec: 177093.01033895358, Loss: 2.552184581756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3818 , TFLOPS: 71.50685589832652, Tokens per sec: 177201.86604650502, Loss: 2.5754098892211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3819 , TFLOPS: 69.5610877680999, Tokens per sec: 172380.0382757493, Loss: 2.592529058456421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3820 , TFLOPS: 70.4231785370365, Tokens per sec: 174516.3941683122, Loss: 2.5506465435028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3821 , TFLOPS: 71.49732948785395, Tokens per sec: 177178.25855193322, Loss: 2.5528430938720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3822 , TFLOPS: 71.54935725796464, Tokens per sec: 177307.18909759956, Loss: 2.552989959716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3823 , TFLOPS: 68.91364412084363, Tokens per sec: 170775.60159604278, Loss: 2.5501203536987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3824 , TFLOPS: 70.42743415959389, Tokens per sec: 174526.9400698049, Loss: 2.559976577758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3825 , TFLOPS: 71.48229874474828, Tokens per sec: 177141.01071474436, Loss: 2.55521559715271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3826 , TFLOPS: 69.4047869192268, Tokens per sec: 171992.7075542819, Loss: 2.5308032035827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3827 , TFLOPS: 70.49563212424395, Tokens per sec: 174695.94213883375, Loss: 2.563842296600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3828 , TFLOPS: 71.54222179942269, Tokens per sec: 177289.50664529696, Loss: 2.5620086193084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3829 , TFLOPS: 70.39784856729237, Tokens per sec: 174453.62371295973, Loss: 2.588481903076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3830 , TFLOPS: 69.80588933688989, Tokens per sec: 172986.68353032996, Loss: 2.551232099533081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3831 , TFLOPS: 70.39502450993822, Tokens per sec: 174446.62538774055, Loss: 2.565798759460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3832 , TFLOPS: 71.48892451115097, Tokens per sec: 177157.43009377865, Loss: 2.535581350326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3833 , TFLOPS: 69.560581684536, Tokens per sec: 172378.7841449283, Loss: 2.54482364654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3834 , TFLOPS: 71.50431976908061, Tokens per sec: 177195.58123885596, Loss: 2.5641746520996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3835 , TFLOPS: 70.36595161194946, Tokens per sec: 174374.57954388027, Loss: 2.5525825023651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3836 , TFLOPS: 68.61001749712709, Tokens per sec: 170023.18137523945, Loss: 2.5713703632354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3837 , TFLOPS: 71.49986289070708, Tokens per sec: 177184.53660328066, Loss: 2.556225299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3838 , TFLOPS: 71.52181359877261, Tokens per sec: 177238.93287593717, Loss: 2.5588302612304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3839 , TFLOPS: 69.63076649988419, Tokens per sec: 172552.70984885676, Loss: 2.5460522174835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3840 , TFLOPS: 70.64885282029508, Tokens per sec: 175075.63990230148, Loss: 2.5419116020202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3841 , TFLOPS: 71.5307938073583, Tokens per sec: 177261.18682206413, Loss: 2.5551016330718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3842 , TFLOPS: 70.56441241484333, Tokens per sec: 174866.3872757135, Loss: 2.5610313415527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3843 , TFLOPS: 68.66925072346021, Tokens per sec: 170169.96783517147, Loss: 2.5425286293029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3844 , TFLOPS: 71.55147682770291, Tokens per sec: 177312.44162490088, Loss: 2.566969871520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3845 , TFLOPS: 71.55620543350658, Tokens per sec: 177324.15963098127, Loss: 2.5605108737945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3846 , TFLOPS: 68.60403284340775, Tokens per sec: 170008.35074405887, Loss: 2.541266918182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3847 , TFLOPS: 71.51054856011478, Tokens per sec: 177211.01686919664, Loss: 2.5543253421783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3848 , TFLOPS: 71.51566145074557, Tokens per sec: 177223.68717541112, Loss: 2.5676865577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3849 , TFLOPS: 70.32130852194013, Tokens per sec: 174263.94904899478, Loss: 2.532454252243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3850 , TFLOPS: 69.68548282073381, Tokens per sec: 172688.30291942245, Loss: 2.5513627529144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3851 , TFLOPS: 71.39902634524516, Tokens per sec: 176934.65253556354, Loss: 2.5576493740081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3852 , TFLOPS: 71.50337635975785, Tokens per sec: 177193.24336662906, Loss: 2.5503549575805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3853 , TFLOPS: 68.22949361187297, Tokens per sec: 169080.20126941334, Loss: 2.556415319442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3854 , TFLOPS: 71.50556964539071, Tokens per sec: 177198.67857003765, Loss: 2.551325798034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3855 , TFLOPS: 71.5138428939314, Tokens per sec: 177219.18059129504, Loss: 2.559061050415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3856 , TFLOPS: 69.59060028854545, Tokens per sec: 172453.1735524859, Loss: 2.5641000270843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3857 , TFLOPS: 69.72071633674366, Tokens per sec: 172775.6154533871, Loss: 2.546322822570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3858 , TFLOPS: 71.51959000408272, Tokens per sec: 177233.42256334645, Loss: 2.5462331771850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3859 , TFLOPS: 71.49769962625824, Tokens per sec: 177179.1757956732, Loss: 2.5552425384521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3860 , TFLOPS: 69.80623899398273, Tokens per sec: 172987.5500191239, Loss: 2.5497236251831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3861 , TFLOPS: 70.18846318055951, Tokens per sec: 173934.7436589312, Loss: 2.5570170879364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3862 , TFLOPS: 71.55010996844624, Tokens per sec: 177309.05439709086, Loss: 2.5354347229003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3863 , TFLOPS: 68.6524048562789, Tokens per sec: 170128.22192057074, Loss: 2.5395655632019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3864 , TFLOPS: 70.50561554916834, Tokens per sec: 174720.68216556145, Loss: 2.5512232780456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3865 , TFLOPS: 71.5011113944988, Tokens per sec: 177187.6305332106, Loss: 2.5664215087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3866 , TFLOPS: 69.60727446947239, Tokens per sec: 172494.49400963553, Loss: 2.5592164993286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3867 , TFLOPS: 71.4810439588499, Tokens per sec: 177137.90121706208, Loss: 2.5475008487701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3868 , TFLOPS: 70.44422941860347, Tokens per sec: 174568.56057178118, Loss: 2.5602619647979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3869 , TFLOPS: 71.5001025630666, Tokens per sec: 177185.13053778966, Loss: 2.580944299697876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3870 , TFLOPS: 68.95778754727334, Tokens per sec: 170884.99386953548, Loss: 2.5578441619873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3871 , TFLOPS: 71.52616194343858, Tokens per sec: 177249.70855302914, Loss: 2.5597212314605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3872 , TFLOPS: 70.53146732806502, Tokens per sec: 174784.74572147595, Loss: 2.562697649002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3873 , TFLOPS: 69.58164322760399, Tokens per sec: 172430.97696877128, Loss: 2.556148052215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3874 , TFLOPS: 71.52743421989155, Tokens per sec: 177252.86139423243, Loss: 2.557340383529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3875 , TFLOPS: 71.50299800256926, Tokens per sec: 177192.30575583625, Loss: 2.5530197620391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3876 , TFLOPS: 69.49346613013283, Tokens per sec: 172212.46440772855, Loss: 2.5520052909851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3877 , TFLOPS: 69.88923529451249, Tokens per sec: 173193.2239946339, Loss: 2.5817103385925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3878 , TFLOPS: 71.55405163005976, Tokens per sec: 177318.822268779, Loss: 2.579392671585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3879 , TFLOPS: 70.44754594072764, Tokens per sec: 174576.77927894995, Loss: 2.568502902984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3880 , TFLOPS: 69.58296274338936, Tokens per sec: 172434.24687424407, Loss: 2.5258939266204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3881 , TFLOPS: 71.52473870295756, Tokens per sec: 177246.1816063343, Loss: 2.567470073699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3882 , TFLOPS: 71.54635996543146, Tokens per sec: 177299.7614765233, Loss: 2.5669548511505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3883 , TFLOPS: 67.74744155096974, Tokens per sec: 167885.62316007688, Loss: 2.5593597888946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3884 , TFLOPS: 71.55672768333713, Tokens per sec: 177325.45382359196, Loss: 2.576946496963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3885 , TFLOPS: 71.51062894148271, Tokens per sec: 177211.2160630811, Loss: 2.576571226119995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3886 , TFLOPS: 68.89361204651551, Tokens per sec: 170725.95990914258, Loss: 2.552046060562134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3887 , TFLOPS: 71.54038096262498, Tokens per sec: 177284.9448489273, Loss: 2.5785093307495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3888 , TFLOPS: 71.53587397525273, Tokens per sec: 177273.77603773325, Loss: 2.5405895709991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3889 , TFLOPS: 71.52190166735316, Tokens per sec: 177239.15111958134, Loss: 2.577845573425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3890 , TFLOPS: 67.17335354933249, Tokens per sec: 166462.96985100574, Loss: 2.575387716293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3891 , TFLOPS: 71.51506508883668, Tokens per sec: 177222.2093249062, Loss: 2.5690107345581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3892 , TFLOPS: 71.55475624951997, Tokens per sec: 177320.56839342168, Loss: 2.5448262691497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3893 , TFLOPS: 69.58219231558887, Tokens per sec: 172432.33766928484, Loss: 2.5455703735351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3894 , TFLOPS: 70.42947326535517, Tokens per sec: 174531.9931985074, Loss: 2.548501491546631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3895 , TFLOPS: 71.50894942261591, Tokens per sec: 177207.05403031487, Loss: 2.5256686210632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3896 , TFLOPS: 70.4449959022276, Tokens per sec: 174570.46000263692, Loss: 2.554258108139038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3897 , TFLOPS: 69.6856994218623, Tokens per sec: 172688.83968087944, Loss: 2.5548791885375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3898 , TFLOPS: 70.56172417678648, Tokens per sec: 174859.7255256748, Loss: 2.583521604537964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3899 , TFLOPS: 71.50961638364673, Tokens per sec: 177208.70683322108, Loss: 2.5673022270202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3900 , TFLOPS: 69.56541152121602, Tokens per sec: 172390.75301227183, Loss: 2.5393903255462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3901 , TFLOPS: 70.49043419272644, Tokens per sec: 174683.06109193395, Loss: 2.5689001083374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3902 , TFLOPS: 71.55222633357627, Tokens per sec: 177314.29898301995, Loss: 2.5622639656066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3903 , TFLOPS: 68.62626627911416, Tokens per sec: 170063.4476469545, Loss: 2.532521963119507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3904 , TFLOPS: 71.4586657053178, Tokens per sec: 177082.4454396414, Loss: 2.5485808849334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3905 , TFLOPS: 70.53224538026238, Tokens per sec: 174786.67382053056, Loss: 2.577051877975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3906 , TFLOPS: 70.67487463603005, Tokens per sec: 175140.1248282324, Loss: 2.5661606788635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3907 , TFLOPS: 70.70538814607774, Tokens per sec: 175215.7406674712, Loss: 2.563347578048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3908 , TFLOPS: 71.44624646176514, Tokens per sec: 177051.66918602533, Loss: 2.55635142326355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3909 , TFLOPS: 70.39620763851532, Tokens per sec: 174449.55731068697, Loss: 2.5605578422546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3910 , TFLOPS: 68.68228741143908, Tokens per sec: 170202.27418409282, Loss: 2.562981605529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3911 , TFLOPS: 71.5018874053621, Tokens per sec: 177189.5535736145, Loss: 2.5605862140655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3912 , TFLOPS: 71.5007158924977, Tokens per sec: 177186.65043568372, Loss: 2.5634548664093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3913 , TFLOPS: 67.99488032977686, Tokens per sec: 168498.8037115926, Loss: 2.5386810302734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3914 , TFLOPS: 71.52409598333658, Tokens per sec: 177244.58887631207, Loss: 2.5544273853302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3915 , TFLOPS: 71.48534213704437, Tokens per sec: 177148.55257611143, Loss: 2.546725273132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3916 , TFLOPS: 69.14167816782451, Tokens per sec: 171340.69508448537, Loss: 2.562661647796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3917 , TFLOPS: 69.79092866571506, Tokens per sec: 172949.6093391047, Loss: 2.5227208137512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3918 , TFLOPS: 71.49786967782904, Tokens per sec: 177179.59720219838, Loss: 2.5586540699005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3919 , TFLOPS: 71.5114969593238, Tokens per sec: 177213.36710691132, Loss: 2.5474064350128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3920 , TFLOPS: 67.87163655383131, Tokens per sec: 168193.39205837686, Loss: 2.546967029571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3921 , TFLOPS: 71.5091340444139, Tokens per sec: 177207.51154346805, Loss: 2.5715739727020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3922 , TFLOPS: 71.4952106146793, Tokens per sec: 177173.0077507933, Loss: 2.578575849533081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3923 , TFLOPS: 69.23008329214197, Tokens per sec: 171559.77272117243, Loss: 2.5692405700683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3924 , TFLOPS: 70.94612535182715, Tokens per sec: 175812.31398271065, Loss: 2.5636584758758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3925 , TFLOPS: 71.56124001691684, Tokens per sec: 177336.63588327708, Loss: 2.5440731048583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3926 , TFLOPS: 71.47596368195164, Tokens per sec: 177125.31173126952, Loss: 2.539358139038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3927 , TFLOPS: 68.93083658849808, Tokens per sec: 170818.20642479716, Loss: 2.557295560836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3928 , TFLOPS: 71.48013652366711, Tokens per sec: 177135.6524927169, Loss: 2.5490808486938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3929 , TFLOPS: 71.51883059319245, Tokens per sec: 177231.54065950433, Loss: 2.542905330657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3930 , TFLOPS: 68.9106688677035, Tokens per sec: 170768.22859101038, Loss: 2.570322275161743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3931 , TFLOPS: 70.570730527077, Tokens per sec: 174882.04425382605, Loss: 2.5418989658355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3932 , TFLOPS: 71.53249645593742, Tokens per sec: 177265.40617280544, Loss: 2.547612428665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3933 , TFLOPS: 69.48505016704122, Tokens per sec: 172191.60872409272, Loss: 2.557400703430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3934 , TFLOPS: 71.52892681873053, Tokens per sec: 177256.56021857873, Loss: 2.561051607131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3935 , TFLOPS: 70.39891699759143, Tokens per sec: 174456.27140093807, Loss: 2.5416834354400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3936 , TFLOPS: 70.41437977226443, Tokens per sec: 174494.58985994305, Loss: 2.5418972969055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3937 , TFLOPS: 69.89176752790324, Tokens per sec: 173199.4991479247, Loss: 2.568301200866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3938 , TFLOPS: 70.59580899750513, Tokens per sec: 174944.19146616853, Loss: 2.54819917678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3939 , TFLOPS: 71.52490713420958, Tokens per sec: 177246.59899753105, Loss: 2.570418119430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3940 , TFLOPS: 69.58002455531985, Tokens per sec: 172426.96572053968, Loss: 2.5597352981567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3941 , TFLOPS: 71.54870636048014, Tokens per sec: 177305.57610193067, Loss: 2.538252353668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3942 , TFLOPS: 70.48932381923053, Tokens per sec: 174680.30946409336, Loss: 2.56264591217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3943 , TFLOPS: 70.47876841166638, Tokens per sec: 174654.15200137522, Loss: 2.560317039489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3944 , TFLOPS: 70.13863358009031, Tokens per sec: 173811.2604197849, Loss: 2.5262789726257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3945 , TFLOPS: 71.53714243117582, Tokens per sec: 177276.91941124262, Loss: 2.5566000938415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3946 , TFLOPS: 70.51029642705141, Tokens per sec: 174732.28189659698, Loss: 2.548686981201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3947 , TFLOPS: 69.47877205327634, Tokens per sec: 172176.050866592, Loss: 2.545225143432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3948 , TFLOPS: 71.52366530697832, Tokens per sec: 177243.52161285342, Loss: 2.5668115615844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3949 , TFLOPS: 71.53052023951213, Tokens per sec: 177260.50889080536, Loss: 2.5582799911499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3950 , TFLOPS: 67.91523954440827, Tokens per sec: 168301.44507229247, Loss: 2.5617239475250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3951 , TFLOPS: 71.5123208549195, Tokens per sec: 177215.4088109571, Loss: 2.575300455093384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3952 , TFLOPS: 71.46866090987258, Tokens per sec: 177107.21465759084, Loss: 2.566297769546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3953 , TFLOPS: 68.90439729800873, Tokens per sec: 170752.68695043627, Loss: 2.5691940784454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3954 , TFLOPS: 70.8792334706603, Tokens per sec: 175646.5485324302, Loss: 2.5541481971740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3955 , TFLOPS: 71.55104515427138, Tokens per sec: 177311.3718905848, Loss: 2.543850898742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3956 , TFLOPS: 71.52756072431042, Tokens per sec: 177253.17488611798, Loss: 2.5487160682678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3957 , TFLOPS: 68.86095062491667, Tokens per sec: 170645.021308468, Loss: 2.5595107078552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3958 , TFLOPS: 71.53291133981728, Tokens per sec: 177266.43430076295, Loss: 2.5552608966827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3959 , TFLOPS: 71.45255335499647, Tokens per sec: 177067.2983622142, Loss: 2.552515983581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3960 , TFLOPS: 68.66795004770998, Tokens per sec: 170166.74461738113, Loss: 2.5428829193115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3961 , TFLOPS: 71.50209049273518, Tokens per sec: 177190.0568464969, Loss: 2.543168783187866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3962 , TFLOPS: 71.56557187477941, Tokens per sec: 177347.37070425408, Loss: 2.557990789413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3963 , TFLOPS: 70.49138895123474, Tokens per sec: 174685.4270886928, Loss: 2.552694797515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3964 , TFLOPS: 68.82804169974865, Tokens per sec: 170563.4693666843, Loss: 2.5496506690979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3965 , TFLOPS: 71.47927774006199, Tokens per sec: 177133.5243323406, Loss: 2.5585920810699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3966 , TFLOPS: 71.52379714139137, Tokens per sec: 177243.8483130517, Loss: 2.5586376190185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3967 , TFLOPS: 69.6188306222047, Tokens per sec: 172523.1314291215, Loss: 2.537008762359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3968 , TFLOPS: 70.46621711366979, Tokens per sec: 174623.04850230008, Loss: 2.538583517074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3969 , TFLOPS: 71.50465906686019, Tokens per sec: 177196.4220561305, Loss: 2.5715599060058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3970 , TFLOPS: 68.68170923937572, Tokens per sec: 170200.84141002892, Loss: 2.543388843536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3971 , TFLOPS: 71.51519205181121, Tokens per sec: 177222.52395314322, Loss: 2.552077531814575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3972 , TFLOPS: 70.6013690686401, Tokens per sec: 174957.9699349334, Loss: 2.5769190788269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3973 , TFLOPS: 71.4960326095809, Tokens per sec: 177175.04474470968, Loss: 2.547420024871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3974 , TFLOPS: 69.74453644432631, Tokens per sec: 172834.64430397833, Loss: 2.5378682613372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3975 , TFLOPS: 70.46215715806719, Tokens per sec: 174612.9874850758, Loss: 2.541510581970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3976 , TFLOPS: 71.53190769960611, Tokens per sec: 177263.94716975954, Loss: 2.5541515350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3977 , TFLOPS: 68.68255200290433, Tokens per sec: 170202.9298708909, Loss: 2.5687484741210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3978 , TFLOPS: 71.479806703862, Tokens per sec: 177134.8351629072, Loss: 2.543403387069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3979 , TFLOPS: 70.54506834890849, Tokens per sec: 174818.4505494022, Loss: 2.5479068756103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3980 , TFLOPS: 69.43132770804603, Tokens per sec: 172058.47855268518, Loss: 2.540170907974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3981 , TFLOPS: 71.48099833899325, Tokens per sec: 177137.78816603188, Loss: 2.5710792541503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3982 , TFLOPS: 71.50339076796733, Tokens per sec: 177193.27907175935, Loss: 2.566606044769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3983 , TFLOPS: 69.33488026522166, Tokens per sec: 171819.47116480352, Loss: 2.5718705654144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3984 , TFLOPS: 69.69305299172582, Tokens per sec: 172707.06263706382, Loss: 2.5410404205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3985 , TFLOPS: 71.54674200131207, Tokens per sec: 177300.7082035202, Loss: 2.533295154571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3986 , TFLOPS: 71.49541759365263, Tokens per sec: 177173.52066748988, Loss: 2.549560785293579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3987 , TFLOPS: 68.61502330459781, Tokens per sec: 170035.58631758712, Loss: 2.5464823246002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3988 , TFLOPS: 71.57683417775034, Tokens per sec: 177375.27993166112, Loss: 2.563015937805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3989 , TFLOPS: 71.53945552892642, Tokens per sec: 177282.65152227733, Loss: 2.5662732124328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3990 , TFLOPS: 68.62974589865945, Tokens per sec: 170072.07052749948, Loss: 2.5527710914611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3991 , TFLOPS: 70.84826025762247, Tokens per sec: 175569.79349854137, Loss: 2.5422489643096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3992 , TFLOPS: 71.51737288512626, Tokens per sec: 177227.92829833654, Loss: 2.555511236190796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3993 , TFLOPS: 71.49120272148818, Tokens per sec: 177163.07574995985, Loss: 2.5747251510620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3994 , TFLOPS: 68.573967024247, Tokens per sec: 169933.84433215513, Loss: 2.5529232025146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3995 , TFLOPS: 71.51042265678957, Tokens per sec: 177210.70486688736, Loss: 2.5553133487701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3996 , TFLOPS: 71.53314388840401, Tokens per sec: 177267.01058177822, Loss: 2.555370330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3997 , TFLOPS: 67.26602699179604, Tokens per sec: 166692.62485025285, Loss: 2.5604238510131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3998 , TFLOPS: 71.53981229943587, Tokens per sec: 177283.5356389013, Loss: 2.5554661750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3999 , TFLOPS: 71.56505015366076, Tokens per sec: 177346.07782184973, Loss: 2.562638282775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4000 , TFLOPS: 69.62296704330156, Tokens per sec: 172533.38193626393, Loss: 2.5836470127105713 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/model.pt +[rank0]:[2024-08-26 10:28:26,769] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006459721000283025, 'preprocessing_with_comm': 0.0012840449999202974, 'state_converting': 0.9798419819999253, : 0.988963056999637}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000, took 4.60s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002000 + eval ppl=9.899835586547852, eval loss=2.292518138885498 +------------------------------------------------------------------ +iteration: 4001 , TFLOPS: 65.99939954718411, Tokens per sec: 163553.7825119729, Loss: 2.5666747093200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4002 , TFLOPS: 71.52321070225229, Tokens per sec: 177242.39505226345, Loss: 2.5544538497924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4003 , TFLOPS: 70.43060069894987, Tokens per sec: 174534.78710315216, Loss: 2.5502209663391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4004 , TFLOPS: 69.76185069476053, Tokens per sec: 172877.55092387646, Loss: 2.5529699325561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4005 , TFLOPS: 70.40432394565515, Tokens per sec: 174469.67041386451, Loss: 2.5536141395568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4006 , TFLOPS: 71.50630138535895, Tokens per sec: 177200.49190228674, Loss: 2.5721325874328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4007 , TFLOPS: 68.78756913105758, Tokens per sec: 170463.17388304631, Loss: 2.563394546508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4008 , TFLOPS: 71.54768773488378, Tokens per sec: 177303.05183549205, Loss: 2.564950466156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4009 , TFLOPS: 70.39566225178568, Tokens per sec: 174448.2057822916, Loss: 2.5430846214294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4010 , TFLOPS: 68.9160414540595, Tokens per sec: 170781.54245183972, Loss: 2.5476131439208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4011 , TFLOPS: 71.52092862543216, Tokens per sec: 177236.73981451167, Loss: 2.546790599822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4012 , TFLOPS: 71.48901482515835, Tokens per sec: 177157.65390183782, Loss: 2.553738594055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4013 , TFLOPS: 69.36507092318062, Tokens per sec: 171894.28694100134, Loss: 2.5523955821990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4014 , TFLOPS: 70.86186529163679, Tokens per sec: 175603.50827154706, Loss: 2.5508203506469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4015 , TFLOPS: 71.53643444687384, Tokens per sec: 177275.16494815142, Loss: 2.556224822998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4016 , TFLOPS: 70.47302315353764, Tokens per sec: 174639.9146188398, Loss: 2.563103199005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4017 , TFLOPS: 68.993921783989, Tokens per sec: 170974.53848863163, Loss: 2.5463597774505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4018 , TFLOPS: 71.38325693083155, Tokens per sec: 176895.57418950225, Loss: 2.554717779159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4019 , TFLOPS: 71.49929664809814, Tokens per sec: 177183.13339171896, Loss: 2.5537219047546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4020 , TFLOPS: 68.57786555119343, Tokens per sec: 169943.50531138657, Loss: 2.543706178665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4021 , TFLOPS: 71.5124815387612, Tokens per sec: 177215.8070032176, Loss: 2.5517022609710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4022 , TFLOPS: 71.47572628462126, Tokens per sec: 177124.72343453334, Loss: 2.5243778228759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4023 , TFLOPS: 70.5298279628143, Tokens per sec: 174780.68319379364, Loss: 2.569805860519409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4024 , TFLOPS: 69.54303197552771, Tokens per sec: 172335.29403274573, Loss: 2.569401979446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4025 , TFLOPS: 71.50483721541552, Tokens per sec: 177196.8635278753, Loss: 2.5507898330688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4026 , TFLOPS: 71.5320310497591, Tokens per sec: 177264.25284502681, Loss: 2.546663284301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4027 , TFLOPS: 69.5049868212808, Tokens per sec: 172241.0139495021, Loss: 2.5666611194610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4028 , TFLOPS: 70.44597659446794, Tokens per sec: 174572.8902660406, Loss: 2.551223039627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4029 , TFLOPS: 71.4891009692318, Tokens per sec: 177157.8673763414, Loss: 2.547822952270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4030 , TFLOPS: 68.61459938267787, Tokens per sec: 170034.53579237065, Loss: 2.5262935161590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4031 , TFLOPS: 70.47317663058044, Tokens per sec: 174640.29495185873, Loss: 2.5710031986236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4032 , TFLOPS: 71.57880137999886, Tokens per sec: 177380.15487553817, Loss: 2.5716280937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4033 , TFLOPS: 71.53628361114124, Tokens per sec: 177274.7911605901, Loss: 2.564176082611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4034 , TFLOPS: 69.47395611723644, Tokens per sec: 172164.11644656598, Loss: 2.5374042987823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4035 , TFLOPS: 70.47939595321434, Tokens per sec: 174655.7071184605, Loss: 2.555170774459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4036 , TFLOPS: 71.54634703852365, Tokens per sec: 177299.72944222172, Loss: 2.542356014251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4037 , TFLOPS: 68.6045022305658, Tokens per sec: 170009.5139371438, Loss: 2.554183006286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4038 , TFLOPS: 70.38418075543731, Tokens per sec: 174419.75336954911, Loss: 2.549642324447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4039 , TFLOPS: 71.51208640359684, Tokens per sec: 177214.8278147526, Loss: 2.573633909225464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4040 , TFLOPS: 69.59125614864534, Tokens per sec: 172454.79884606242, Loss: 2.562105894088745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4041 , TFLOPS: 71.52873054648076, Tokens per sec: 177256.07383432265, Loss: 2.5273680686950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4042 , TFLOPS: 70.5811983943316, Tokens per sec: 174907.98478201372, Loss: 2.5409131050109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4043 , TFLOPS: 70.49152594273289, Tokens per sec: 174685.76656871612, Loss: 2.553981304168701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4044 , TFLOPS: 69.67812099942648, Tokens per sec: 172670.05951525134, Loss: 2.5474839210510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4045 , TFLOPS: 71.55076736055696, Tokens per sec: 177310.68348715894, Loss: 2.5561652183532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4046 , TFLOPS: 70.47666004740891, Tokens per sec: 174648.92724248974, Loss: 2.53994083404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4047 , TFLOPS: 69.62990210993374, Tokens per sec: 172550.56779533913, Loss: 2.564695358276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4048 , TFLOPS: 71.49544621048099, Tokens per sec: 177173.59158314293, Loss: 2.564201831817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4049 , TFLOPS: 71.53402178949189, Tokens per sec: 177269.18611738278, Loss: 2.5623276233673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4050 , TFLOPS: 68.63522453540817, Tokens per sec: 170085.64719288444, Loss: 2.535672664642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4051 , TFLOPS: 70.84626351444555, Tokens per sec: 175564.84534898956, Loss: 2.548389196395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4052 , TFLOPS: 71.52232348169075, Tokens per sec: 177240.19642197655, Loss: 2.565866708755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4053 , TFLOPS: 70.4775968418337, Tokens per sec: 174651.2487222708, Loss: 2.5710391998291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4054 , TFLOPS: 69.62854400147626, Tokens per sec: 172547.2022529735, Loss: 2.5840110778808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4055 , TFLOPS: 71.50536070514197, Tokens per sec: 177198.16079308456, Loss: 2.5203914642333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4056 , TFLOPS: 71.5122709353284, Tokens per sec: 177215.28510471128, Loss: 2.5612027645111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4057 , TFLOPS: 68.0959097644627, Tokens per sec: 168749.1657800551, Loss: 2.5653727054595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4058 , TFLOPS: 71.53958907456266, Tokens per sec: 177282.9824630752, Loss: 2.5368192195892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4059 , TFLOPS: 71.49349443858736, Tokens per sec: 177168.7548773851, Loss: 2.5637686252593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4060 , TFLOPS: 69.60656437823167, Tokens per sec: 172492.73432531802, Loss: 2.54636812210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4061 , TFLOPS: 69.38109111810857, Tokens per sec: 171933.98674880294, Loss: 2.5567190647125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4062 , TFLOPS: 71.58029509760067, Tokens per sec: 177383.85647230205, Loss: 2.5706629753112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4063 , TFLOPS: 70.02379653968431, Tokens per sec: 173526.68158331452, Loss: 2.5490341186523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4064 , TFLOPS: 69.66913485016582, Tokens per sec: 172647.79084747736, Loss: 2.5694737434387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4065 , TFLOPS: 70.36445308189191, Tokens per sec: 174370.86602132095, Loss: 2.5601418018341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4066 , TFLOPS: 71.5539454368907, Tokens per sec: 177318.55911040798, Loss: 2.5378386974334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4067 , TFLOPS: 69.0229240945153, Tokens per sec: 171046.40940898407, Loss: 2.5541672706604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4068 , TFLOPS: 69.90676389841255, Tokens per sec: 173236.66180603337, Loss: 2.547565460205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4069 , TFLOPS: 71.51167954653212, Tokens per sec: 177213.81957812724, Loss: 2.541121006011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4070 , TFLOPS: 70.42393309676486, Tokens per sec: 174518.26405044063, Loss: 2.5743257999420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4071 , TFLOPS: 69.9265506316386, Tokens per sec: 173285.69551065617, Loss: 2.5514092445373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4072 , TFLOPS: 70.41617916960867, Tokens per sec: 174499.04896478204, Loss: 2.5369396209716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4073 , TFLOPS: 71.51742527529936, Tokens per sec: 177228.05812695672, Loss: 2.551267385482788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4074 , TFLOPS: 69.6100630581274, Tokens per sec: 172501.40443951104, Loss: 2.540851593017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4075 , TFLOPS: 70.4770135295126, Tokens per sec: 174649.80321008197, Loss: 2.537663221359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4076 , TFLOPS: 71.52610322081026, Tokens per sec: 177249.56303188732, Loss: 2.5313401222229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4077 , TFLOPS: 68.72078034602343, Tokens per sec: 170297.66391633963, Loss: 2.5454139709472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4078 , TFLOPS: 71.46324974519618, Tokens per sec: 177093.80519543373, Loss: 2.574632167816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4079 , TFLOPS: 70.47770013627184, Tokens per sec: 174651.50469726732, Loss: 2.5263559818267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4080 , TFLOPS: 69.53890304660675, Tokens per sec: 172325.0620920405, Loss: 2.529108762741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4081 , TFLOPS: 71.50234713711554, Tokens per sec: 177190.69283954316, Loss: 2.547710657119751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4082 , TFLOPS: 71.55394343509971, Tokens per sec: 177318.5541497494, Loss: 2.566887855529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4083 , TFLOPS: 70.51465225748136, Tokens per sec: 174743.07612423672, Loss: 2.5561563968658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4084 , TFLOPS: 69.02676600469552, Tokens per sec: 171055.93008563196, Loss: 2.544137716293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4085 , TFLOPS: 71.46188691065257, Tokens per sec: 177090.42794130603, Loss: 2.549912691116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4086 , TFLOPS: 71.49600912868347, Tokens per sec: 177174.98655645922, Loss: 2.551417589187622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4087 , TFLOPS: 68.57654254002084, Tokens per sec: 169940.2267439592, Loss: 2.547874689102173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4088 , TFLOPS: 71.51929632027611, Tokens per sec: 177232.69478252175, Loss: 2.542980909347534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4089 , TFLOPS: 71.49375915025318, Tokens per sec: 177169.4108620534, Loss: 2.5626657009124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4090 , TFLOPS: 69.40851961632158, Tokens per sec: 172001.9575888733, Loss: 2.5543410778045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4091 , TFLOPS: 69.87037812871584, Tokens per sec: 173146.4938605015, Loss: 2.557243585586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4092 , TFLOPS: 71.52332381905687, Tokens per sec: 177242.67536816606, Loss: 2.545105218887329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4093 , TFLOPS: 71.55877270331746, Tokens per sec: 177330.52160838078, Loss: 2.537370443344116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4094 , TFLOPS: 68.53819132331952, Tokens per sec: 169845.18820423807, Loss: 2.5369911193847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4095 , TFLOPS: 71.56276521270084, Tokens per sec: 177340.41548644402, Loss: 2.558739423751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4096 , TFLOPS: 71.51851229925505, Tokens per sec: 177230.7518920645, Loss: 2.5489938259124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4097 , TFLOPS: 68.69418421132087, Tokens per sec: 170231.7557647406, Loss: 2.5292654037475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4098 , TFLOPS: 70.53663914511306, Tokens per sec: 174797.56205384366, Loss: 2.5455939769744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4099 , TFLOPS: 71.51442473585139, Tokens per sec: 177220.62245966794, Loss: 2.557249069213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4100 , TFLOPS: 70.72295051079567, Tokens per sec: 175259.2621418964, Loss: 2.5596888065338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4101 , TFLOPS: 70.73806709045869, Tokens per sec: 175296.7226915301, Loss: 2.5611743927001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4102 , TFLOPS: 70.36098156143852, Tokens per sec: 174362.26321121788, Loss: 2.5435984134674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4103 , TFLOPS: 71.49685139848951, Tokens per sec: 177177.0737938223, Loss: 2.5562992095947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4104 , TFLOPS: 68.73552135174857, Tokens per sec: 170334.1937523824, Loss: 2.5617527961730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4105 , TFLOPS: 70.58619828775873, Tokens per sec: 174920.3750687099, Loss: 2.5485739707946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4106 , TFLOPS: 71.50039897646568, Tokens per sec: 177185.8650828452, Loss: 2.562211036682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4107 , TFLOPS: 69.6065496261967, Tokens per sec: 172492.69776815028, Loss: 2.5294933319091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4108 , TFLOPS: 71.50905854399812, Tokens per sec: 177207.32444512067, Loss: 2.560471534729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4109 , TFLOPS: 70.41560818686897, Tokens per sec: 174497.63400665415, Loss: 2.559774875640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4110 , TFLOPS: 70.43516318495674, Tokens per sec: 174546.0934460765, Loss: 2.530233383178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4111 , TFLOPS: 69.84861506460012, Tokens per sec: 173092.56259022333, Loss: 2.572061061859131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4112 , TFLOPS: 70.40095144732899, Tokens per sec: 174461.31299150092, Loss: 2.5702483654022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4113 , TFLOPS: 71.52484757296067, Tokens per sec: 177246.451398195, Loss: 2.5662119388580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4114 , TFLOPS: 69.44370857814194, Tokens per sec: 172089.15971264828, Loss: 2.553544759750366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4115 , TFLOPS: 71.57449910673418, Tokens per sec: 177369.49336845375, Loss: 2.555281400680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4116 , TFLOPS: 70.51714856386623, Tokens per sec: 174749.26224644925, Loss: 2.543243885040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4117 , TFLOPS: 69.53165214597034, Tokens per sec: 172307.09356151082, Loss: 2.5410871505737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4118 , TFLOPS: 70.92881995389081, Tokens per sec: 175769.42930027746, Loss: 2.5486133098602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4119 , TFLOPS: 71.55464077031846, Tokens per sec: 177320.28222323922, Loss: 2.5433480739593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4120 , TFLOPS: 70.55554901705626, Tokens per sec: 174844.4227996133, Loss: 2.561424732208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4121 , TFLOPS: 69.88008031575743, Tokens per sec: 173170.53694877468, Loss: 2.5427849292755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4122 , TFLOPS: 71.51027337956208, Tokens per sec: 177210.33494147338, Loss: 2.5624825954437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4123 , TFLOPS: 71.54774666700686, Tokens per sec: 177303.19787578494, Loss: 2.547287940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4124 , TFLOPS: 67.81660660013078, Tokens per sec: 168057.0217710571, Loss: 2.5537109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4125 , TFLOPS: 71.53191790544639, Tokens per sec: 177263.97246095605, Loss: 2.5299479961395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4126 , TFLOPS: 71.52275419988848, Tokens per sec: 177241.26378911803, Loss: 2.535926342010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4127 , TFLOPS: 69.47867226798513, Tokens per sec: 172175.80358764812, Loss: 2.517854690551758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4128 , TFLOPS: 71.48437486722688, Tokens per sec: 177146.1555749501, Loss: 2.5525381565093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4129 , TFLOPS: 71.55846466375482, Tokens per sec: 177329.7582524112, Loss: 2.5446369647979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4130 , TFLOPS: 71.55795197321763, Tokens per sec: 177328.4877487825, Loss: 2.5459465980529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4131 , TFLOPS: 67.89783287839434, Tokens per sec: 168258.3094364074, Loss: 2.5647499561309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4132 , TFLOPS: 71.55110941034518, Tokens per sec: 177311.53112421403, Loss: 2.5467848777770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4133 , TFLOPS: 71.56040758469805, Tokens per sec: 177334.57302453893, Loss: 2.5390915870666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4134 , TFLOPS: 69.6025075371387, Tokens per sec: 172482.68102619174, Loss: 2.5461082458496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4135 , TFLOPS: 70.4374426207787, Tokens per sec: 174551.7421391439, Loss: 2.5589795112609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4136 , TFLOPS: 71.50791583818584, Tokens per sec: 177204.49269423846, Loss: 2.553652763366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4137 , TFLOPS: 70.47888085718571, Tokens per sec: 174654.43065375905, Loss: 2.5459671020507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4138 , TFLOPS: 69.08605532323864, Tokens per sec: 171202.85554823943, Loss: 2.5295259952545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4139 , TFLOPS: 70.52606964800073, Tokens per sec: 174771.36967567308, Loss: 2.534210681915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4140 , TFLOPS: 71.53554231670395, Tokens per sec: 177272.95415131416, Loss: 2.541060209274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4141 , TFLOPS: 69.57704092465069, Tokens per sec: 172419.57195506745, Loss: 2.5618016719818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4142 , TFLOPS: 70.45556622929529, Tokens per sec: 174596.65443752828, Loss: 2.5460948944091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4143 , TFLOPS: 71.51629802929176, Tokens per sec: 177225.2646871735, Loss: 2.5565128326416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4144 , TFLOPS: 68.69972607565664, Tokens per sec: 170245.4891150514, Loss: 2.5638949871063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4145 , TFLOPS: 71.53470572684314, Tokens per sec: 177270.88098948108, Loss: 2.533163547515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4146 , TFLOPS: 70.59614889668475, Tokens per sec: 174945.03377377865, Loss: 2.53578519821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4147 , TFLOPS: 69.56295766517773, Tokens per sec: 172384.67208669437, Loss: 2.5586307048797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4148 , TFLOPS: 71.52347107339025, Tokens per sec: 177243.04028062557, Loss: 2.541620969772339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4149 , TFLOPS: 70.31463925818285, Tokens per sec: 174247.42187872424, Loss: 2.543611764907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4150 , TFLOPS: 71.49587073482044, Tokens per sec: 177174.6436012213, Loss: 2.5542492866516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4151 , TFLOPS: 68.84502214456353, Tokens per sec: 170605.54877948598, Loss: 2.5389890670776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4152 , TFLOPS: 71.51603865719626, Tokens per sec: 177224.62193454866, Loss: 2.563014268875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4153 , TFLOPS: 70.50934078075956, Tokens per sec: 174729.9136998126, Loss: 2.5575616359710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4154 , TFLOPS: 69.47758272688219, Tokens per sec: 172173.10358477218, Loss: 2.554694175720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4155 , TFLOPS: 71.51758968218859, Tokens per sec: 177228.46554533913, Loss: 2.5321388244628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4156 , TFLOPS: 71.50834137017034, Tokens per sec: 177205.54720937225, Loss: 2.5420775413513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4157 , TFLOPS: 69.36913905169068, Tokens per sec: 171904.3682115906, Loss: 2.531217575073242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4158 , TFLOPS: 69.666092759638, Tokens per sec: 172640.25221203547, Loss: 2.569689989089966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4159 , TFLOPS: 71.53449057543051, Tokens per sec: 177270.3478205797, Loss: 2.5494232177734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4160 , TFLOPS: 71.53633344822063, Tokens per sec: 177274.91466236283, Loss: 2.5517818927764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4161 , TFLOPS: 68.83388365938224, Tokens per sec: 170577.94638620128, Loss: 2.548398494720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4162 , TFLOPS: 71.57335264427, Tokens per sec: 177366.65230817426, Loss: 2.556057929992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4163 , TFLOPS: 71.53052834785998, Tokens per sec: 177260.52898418455, Loss: 2.5681495666503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4164 , TFLOPS: 67.15615703155882, Tokens per sec: 166420.35498560977, Loss: 2.5601298809051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4165 , TFLOPS: 71.52295925131388, Tokens per sec: 177241.77192913846, Loss: 2.5439138412475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4166 , TFLOPS: 71.52767333305958, Tokens per sec: 177253.4539430033, Loss: 2.5445008277893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4167 , TFLOPS: 70.63848224348767, Tokens per sec: 175049.9404705594, Loss: 2.5676729679107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4168 , TFLOPS: 69.01807894788048, Tokens per sec: 171034.40260188602, Loss: 2.5568244457244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4169 , TFLOPS: 71.54222178011439, Tokens per sec: 177289.50659744884, Loss: 2.5386834144592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4170 , TFLOPS: 71.53327404703087, Tokens per sec: 177267.33312919372, Loss: 2.5633301734924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4171 , TFLOPS: 68.69530003732089, Tokens per sec: 170234.52090448703, Loss: 2.563370704650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4172 , TFLOPS: 70.43314905807895, Tokens per sec: 174541.10221779897, Loss: 2.564626455307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4173 , TFLOPS: 71.47307101031117, Tokens per sec: 177118.14337229036, Loss: 2.566746234893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4174 , TFLOPS: 69.56141999737969, Tokens per sec: 172380.8615765027, Loss: 2.548705577850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4175 , TFLOPS: 71.4988123431376, Tokens per sec: 177181.93323067614, Loss: 2.557961940765381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4176 , TFLOPS: 70.49093111852886, Tokens per sec: 174684.29252881327, Loss: 2.5571553707122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4177 , TFLOPS: 70.44250534823034, Tokens per sec: 174564.28813547455, Loss: 2.557799816131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4178 , TFLOPS: 69.666286313301, Tokens per sec: 172640.7318593341, Loss: 2.56258487701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4179 , TFLOPS: 70.56050682426734, Tokens per sec: 174856.70879202985, Loss: 2.5653135776519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4180 , TFLOPS: 71.54173511463223, Tokens per sec: 177288.30058677346, Loss: 2.581111192703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4181 , TFLOPS: 69.57717308696574, Tokens per sec: 172419.89946784306, Loss: 2.5368540287017822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4182 , TFLOPS: 71.52188337727824, Tokens per sec: 177239.10579476086, Loss: 2.572183847427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4183 , TFLOPS: 70.41139823942066, Tokens per sec: 174487.2012931132, Loss: 2.545105218887329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4184 , TFLOPS: 69.49096578581799, Tokens per sec: 172206.26827908074, Loss: 2.54290509223938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4185 , TFLOPS: 70.8487328788502, Tokens per sec: 175570.96470600722, Loss: 2.5528011322021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4186 , TFLOPS: 70.57120823875263, Tokens per sec: 174883.22807598257, Loss: 2.558669090270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4187 , TFLOPS: 71.49684039160873, Tokens per sec: 177177.0465175592, Loss: 2.5593700408935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4188 , TFLOPS: 69.85904553530978, Tokens per sec: 173118.41044565308, Loss: 2.5563700199127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4189 , TFLOPS: 71.5359420510681, Tokens per sec: 177273.94473710292, Loss: 2.5520613193511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4190 , TFLOPS: 70.45347570683688, Tokens per sec: 174591.4738925863, Loss: 2.540257453918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4191 , TFLOPS: 68.66943955257949, Tokens per sec: 170170.43577453028, Loss: 2.549312114715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4192 , TFLOPS: 71.53354385689704, Tokens per sec: 177268.0017477641, Loss: 2.536585807800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4193 , TFLOPS: 71.48288561343722, Tokens per sec: 177142.4650400045, Loss: 2.536231279373169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4194 , TFLOPS: 68.67328622414895, Tokens per sec: 170179.96825042527, Loss: 2.544597625732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4195 , TFLOPS: 71.54496556369055, Tokens per sec: 177296.3059954033, Loss: 2.54494309425354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4196 , TFLOPS: 71.51999396575425, Tokens per sec: 177234.42362486856, Loss: 2.5526514053344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4197 , TFLOPS: 70.01433426636348, Tokens per sec: 173503.23302767004, Loss: 2.5339884757995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4198 , TFLOPS: 69.39074359566084, Tokens per sec: 171957.90665148565, Loss: 2.5561678409576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4199 , TFLOPS: 71.5427291987687, Tokens per sec: 177290.76403677103, Loss: 2.554509162902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4200 , TFLOPS: 71.51624247214963, Tokens per sec: 177225.12701045515, Loss: 2.5695853233337402 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/model.pt +[rank0]:[2024-08-26 10:53:35,622] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006397605997335631, 'preprocessing_with_comm': 0.0012998629972571507, 'state_converting': 0.989365083994926, : 0.9984190940012923}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200, took 4.71s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002200 + eval ppl=11.20853042602539, eval loss=2.416675090789795 +------------------------------------------------------------------ +iteration: 4201 , TFLOPS: 63.362691796255916, Tokens per sec: 157019.7302478362, Loss: 2.5474154949188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4202 , TFLOPS: 71.48691079801904, Tokens per sec: 177152.43989080892, Loss: 2.559230327606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4203 , TFLOPS: 71.5135837520761, Tokens per sec: 177218.53840923076, Loss: 2.539337635040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4204 , TFLOPS: 68.76085120776253, Tokens per sec: 170396.96392589895, Loss: 2.558530330657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4205 , TFLOPS: 69.85485597379383, Tokens per sec: 173108.02825069826, Loss: 2.5228304862976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4206 , TFLOPS: 71.49081113263584, Tokens per sec: 177162.10534964668, Loss: 2.5541017055511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4207 , TFLOPS: 71.48081556101516, Tokens per sec: 177137.33522206752, Loss: 2.5391178131103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4208 , TFLOPS: 68.98761603965663, Tokens per sec: 170958.91215954095, Loss: 2.528548002243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4209 , TFLOPS: 70.4975665150085, Tokens per sec: 174700.73577223907, Loss: 2.5413174629211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4210 , TFLOPS: 71.4221332495646, Tokens per sec: 176991.91398990277, Loss: 2.5496785640716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4211 , TFLOPS: 67.78910625121782, Tokens per sec: 167988.87287703727, Loss: 2.551760196685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4212 , TFLOPS: 70.39915093719956, Tokens per sec: 174456.85112905764, Loss: 2.5241894721984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4213 , TFLOPS: 71.52564634327388, Tokens per sec: 177248.43083902297, Loss: 2.539879322052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4214 , TFLOPS: 69.03619031948286, Tokens per sec: 171079.284575849, Loss: 2.537287712097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4215 , TFLOPS: 71.5280231442476, Tokens per sec: 177254.32081366208, Loss: 2.549252986907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4216 , TFLOPS: 70.55864731809278, Tokens per sec: 174852.10073088782, Loss: 2.529816150665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4217 , TFLOPS: 69.54597724441845, Tokens per sec: 172342.5927335051, Loss: 2.533752918243408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4218 , TFLOPS: 69.80440002451415, Tokens per sec: 172982.99285020155, Loss: 2.5580670833587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4219 , TFLOPS: 70.5679887204751, Tokens per sec: 174875.24975503163, Loss: 2.5537233352661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4220 , TFLOPS: 71.56053611120628, Tokens per sec: 177334.8915273847, Loss: 2.548638105392456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4221 , TFLOPS: 69.48194740646078, Tokens per sec: 172183.91974157706, Loss: 2.5458526611328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4222 , TFLOPS: 71.52182067128275, Tokens per sec: 177238.9504023964, Loss: 2.5630393028259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4223 , TFLOPS: 70.44096134385467, Tokens per sec: 174560.46192254548, Loss: 2.5481951236724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4224 , TFLOPS: 69.35425695020554, Tokens per sec: 171867.4887247116, Loss: 2.537773370742798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4225 , TFLOPS: 70.08479062326914, Tokens per sec: 173677.8316414907, Loss: 2.532489776611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4226 , TFLOPS: 71.46398894404811, Tokens per sec: 177095.6370116183, Loss: 2.5267293453216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4227 , TFLOPS: 69.9904242904846, Tokens per sec: 173443.98147354092, Loss: 2.5305449962615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4228 , TFLOPS: 68.77935795913392, Tokens per sec: 170442.82569448423, Loss: 2.5495753288269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4229 , TFLOPS: 71.4801851148556, Tokens per sec: 177135.77290703476, Loss: 2.5478920936584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4230 , TFLOPS: 71.48881963095931, Tokens per sec: 177157.1701891102, Loss: 2.5296216011047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4231 , TFLOPS: 67.79665023384209, Tokens per sec: 168007.5676970185, Loss: 2.5694258213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4232 , TFLOPS: 71.45249921799495, Tokens per sec: 177067.16420476077, Loss: 2.5565288066864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4233 , TFLOPS: 71.5107131479443, Tokens per sec: 177211.42473596902, Loss: 2.5344462394714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4234 , TFLOPS: 68.69165419943953, Tokens per sec: 170225.4861165948, Loss: 2.541944742202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4235 , TFLOPS: 71.49983076517212, Tokens per sec: 177184.45699266612, Loss: 2.56412935256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4236 , TFLOPS: 71.52719764983088, Tokens per sec: 177252.27514756174, Loss: 2.557708263397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4237 , TFLOPS: 70.59457101191462, Tokens per sec: 174941.12360149992, Loss: 2.542695999145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4238 , TFLOPS: 67.9947013225193, Tokens per sec: 168498.3601118891, Loss: 2.5230722427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4239 , TFLOPS: 71.5364403236666, Tokens per sec: 177275.1795114912, Loss: 2.533048391342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4240 , TFLOPS: 71.48727205437977, Tokens per sec: 177153.33512386648, Loss: 2.542477607727051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4241 , TFLOPS: 69.57988536844464, Tokens per sec: 172426.62080013021, Loss: 2.554335594177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4242 , TFLOPS: 70.49065211023567, Tokens per sec: 174683.60111552742, Loss: 2.5350844860076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4243 , TFLOPS: 71.45778576606301, Tokens per sec: 177080.26485323475, Loss: 2.557662010192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4244 , TFLOPS: 69.47174556844168, Tokens per sec: 172158.6384631405, Loss: 2.5362327098846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4245 , TFLOPS: 69.8211876950576, Tokens per sec: 173024.59454712278, Loss: 2.5393788814544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4246 , TFLOPS: 70.47134589712824, Tokens per sec: 174635.75819269227, Loss: 2.52860426902771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4247 , TFLOPS: 71.49205177093424, Tokens per sec: 177165.1797880177, Loss: 2.5249526500701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4248 , TFLOPS: 68.63087931155845, Tokens per sec: 170074.87924951894, Loss: 2.5212254524230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4249 , TFLOPS: 70.53249114433237, Tokens per sec: 174787.2828509691, Loss: 2.544081687927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4250 , TFLOPS: 71.51657848258837, Tokens per sec: 177225.95968133697, Loss: 2.5402698516845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4251 , TFLOPS: 67.89156615113477, Tokens per sec: 168242.7797959212, Loss: 2.5714972019195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4252 , TFLOPS: 71.5230953634794, Tokens per sec: 177242.10923007864, Loss: 2.551909923553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4253 , TFLOPS: 70.45202777507977, Tokens per sec: 174587.8857581893, Loss: 2.518080949783325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4254 , TFLOPS: 70.5889022071004, Tokens per sec: 174927.07567869985, Loss: 2.54422664642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4255 , TFLOPS: 70.68072372810386, Tokens per sec: 175154.619522722, Loss: 2.547070264816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4256 , TFLOPS: 70.49173635843455, Tokens per sec: 174686.288002004, Loss: 2.558016777038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4257 , TFLOPS: 71.50443795412463, Tokens per sec: 177195.8741144142, Loss: 2.5424578189849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4258 , TFLOPS: 68.08512828311034, Tokens per sec: 168722.44808158572, Loss: 2.5436296463012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4259 , TFLOPS: 71.46190504463958, Tokens per sec: 177090.47287932347, Loss: 2.5484352111816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4260 , TFLOPS: 70.55752478955176, Tokens per sec: 174849.31898150645, Loss: 2.5442354679107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4261 , TFLOPS: 69.5184353469581, Tokens per sec: 172274.3408776079, Loss: 2.531683921813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4262 , TFLOPS: 71.46728530820931, Tokens per sec: 177103.80576513425, Loss: 2.5375843048095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4263 , TFLOPS: 71.5080826835672, Tokens per sec: 177204.906155473, Loss: 2.552100419998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4264 , TFLOPS: 68.5403112361041, Tokens per sec: 169850.44158164592, Loss: 2.549370765686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4265 , TFLOPS: 69.74632724801725, Tokens per sec: 172839.0821127978, Loss: 2.535346031188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4266 , TFLOPS: 71.52406619370835, Tokens per sec: 177244.51505433174, Loss: 2.5387825965881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4267 , TFLOPS: 71.53387213344811, Tokens per sec: 177268.81525322032, Loss: 2.575502872467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4268 , TFLOPS: 68.51987426127947, Tokens per sec: 169799.79650671186, Loss: 2.550755739212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4269 , TFLOPS: 71.53352930126714, Tokens per sec: 177267.96567730975, Loss: 2.540177583694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4270 , TFLOPS: 71.52626739701789, Tokens per sec: 177249.96987861537, Loss: 2.552568197250366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4271 , TFLOPS: 67.3313605174086, Tokens per sec: 166854.5285238028, Loss: 2.5468099117279053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4272 , TFLOPS: 70.90546935746076, Tokens per sec: 175711.56395004326, Loss: 2.54211688041687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4273 , TFLOPS: 71.52917538349898, Tokens per sec: 177257.17618945666, Loss: 2.5335912704467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4274 , TFLOPS: 71.5521278335721, Tokens per sec: 177314.05488915896, Loss: 2.514425039291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4275 , TFLOPS: 68.92600245879963, Tokens per sec: 170806.22691888106, Loss: 2.5711686611175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4276 , TFLOPS: 71.47041286520444, Tokens per sec: 177111.55619589702, Loss: 2.5308594703674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4277 , TFLOPS: 71.51496832369504, Tokens per sec: 177221.96953022564, Loss: 2.521857261657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4278 , TFLOPS: 67.93649347217304, Tokens per sec: 168354.11464660702, Loss: 2.5448479652404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4279 , TFLOPS: 70.49141125126972, Tokens per sec: 174685.4823506361, Loss: 2.542534112930298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4280 , TFLOPS: 71.53413715383805, Tokens per sec: 177269.47200294098, Loss: 2.5359652042388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4281 , TFLOPS: 69.37145530547565, Tokens per sec: 171910.1081436262, Loss: 2.5534660816192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4282 , TFLOPS: 71.50655135742565, Tokens per sec: 177201.11136060473, Loss: 2.5563526153564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4283 , TFLOPS: 70.57262029343185, Tokens per sec: 174886.72730302837, Loss: 2.558377742767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4284 , TFLOPS: 70.6808013582885, Tokens per sec: 175154.8118988712, Loss: 2.5552077293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4285 , TFLOPS: 69.07093263542741, Tokens per sec: 171165.37986194165, Loss: 2.5403201580047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4286 , TFLOPS: 70.47518209013786, Tokens per sec: 174645.26470156162, Loss: 2.547996759414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4287 , TFLOPS: 71.49935516260136, Tokens per sec: 177183.27839710383, Loss: 2.545679807662964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4288 , TFLOPS: 67.80697791471667, Tokens per sec: 168033.16082791367, Loss: 2.55216646194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4289 , TFLOPS: 71.54056064263948, Tokens per sec: 177285.39011579665, Loss: 2.5310380458831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4290 , TFLOPS: 70.53374613239355, Tokens per sec: 174790.39284963306, Loss: 2.5660414695739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4291 , TFLOPS: 69.6310726817201, Tokens per sec: 172553.46860117486, Loss: 2.5467824935913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4292 , TFLOPS: 70.02747627167281, Tokens per sec: 173535.80036453798, Loss: 2.564936876296997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4293 , TFLOPS: 70.3972438283751, Tokens per sec: 174452.1251033052, Loss: 2.5627553462982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4294 , TFLOPS: 71.50753574281472, Tokens per sec: 177203.55077603782, Loss: 2.5419235229492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4295 , TFLOPS: 69.48868176563674, Tokens per sec: 172200.608225466, Loss: 2.5537941455841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4296 , TFLOPS: 71.47624919383115, Tokens per sec: 177126.01926115868, Loss: 2.5383102893829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4297 , TFLOPS: 70.46424080558359, Tokens per sec: 174618.15099315415, Loss: 2.554084062576294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4298 , TFLOPS: 67.85650126905884, Tokens per sec: 168155.88515542142, Loss: 2.5510730743408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4299 , TFLOPS: 71.55094723775395, Tokens per sec: 177311.1292426682, Loss: 2.5261173248291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4300 , TFLOPS: 71.4992614235769, Tokens per sec: 177183.046101475, Loss: 2.570859909057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4301 , TFLOPS: 68.6290932690972, Tokens per sec: 170070.45323955113, Loss: 2.5544776916503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4302 , TFLOPS: 71.5087245938171, Tokens per sec: 177206.4968797845, Loss: 2.5180695056915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4303 , TFLOPS: 71.50573453859509, Tokens per sec: 177199.08719356256, Loss: 2.5428457260131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4304 , TFLOPS: 71.49542134938658, Tokens per sec: 177173.5299746123, Loss: 2.553741693496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4305 , TFLOPS: 67.16738879535899, Tokens per sec: 166448.1885335875, Loss: 2.5347049236297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4306 , TFLOPS: 71.51724192452681, Tokens per sec: 177227.60376354458, Loss: 2.5600924491882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4307 , TFLOPS: 71.55134381781599, Tokens per sec: 177312.11201174883, Loss: 2.5696020126342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4308 , TFLOPS: 68.5800164526257, Tokens per sec: 169948.83548207712, Loss: 2.5499281883239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4309 , TFLOPS: 71.50413405194043, Tokens per sec: 177195.12101132417, Loss: 2.5432844161987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4310 , TFLOPS: 71.4984664385598, Tokens per sec: 177181.07604102785, Loss: 2.5486180782318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4311 , TFLOPS: 69.42494494625657, Tokens per sec: 172042.66136585167, Loss: 2.5443077087402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4312 , TFLOPS: 68.73692072828096, Tokens per sec: 170337.66156158372, Loss: 2.5690250396728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4313 , TFLOPS: 71.50003736533867, Tokens per sec: 177184.9689706377, Loss: 2.5527894496917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4314 , TFLOPS: 71.4844347278106, Tokens per sec: 177146.30391607073, Loss: 2.5559701919555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4315 , TFLOPS: 69.59010294083066, Tokens per sec: 172451.9410700613, Loss: 2.5564098358154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4316 , TFLOPS: 70.5098418420387, Tokens per sec: 174731.15538485878, Loss: 2.5351920127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4317 , TFLOPS: 71.48523750863166, Tokens per sec: 177148.29329537897, Loss: 2.5589241981506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4318 , TFLOPS: 67.91083087963831, Tokens per sec: 168290.519915338, Loss: 2.5406179428100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4319 , TFLOPS: 71.5473792106773, Tokens per sec: 177302.28727852166, Loss: 2.5543975830078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4320 , TFLOPS: 70.43511071924462, Tokens per sec: 174545.9634302623, Loss: 2.537076711654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4321 , TFLOPS: 70.51586409300468, Tokens per sec: 174746.0791861582, Loss: 2.540581464767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4322 , TFLOPS: 70.33228237386875, Tokens per sec: 174291.14346294352, Loss: 2.5504086017608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4323 , TFLOPS: 70.53481191650697, Tokens per sec: 174793.03398007192, Loss: 2.552475690841675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4324 , TFLOPS: 71.42801339569473, Tokens per sec: 177006.4856397648, Loss: 2.5360233783721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4325 , TFLOPS: 67.84072861329558, Tokens per sec: 168116.79877693858, Loss: 2.5565037727355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4326 , TFLOPS: 71.48363139556739, Tokens per sec: 177144.3131702777, Loss: 2.5798356533050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4327 , TFLOPS: 70.47796519429829, Tokens per sec: 174652.16154025545, Loss: 2.544417381286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4328 , TFLOPS: 68.62831359047792, Tokens per sec: 170068.52111004386, Loss: 2.524980306625366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4329 , TFLOPS: 71.44662251203663, Tokens per sec: 177052.60108002342, Loss: 2.531033754348755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4330 , TFLOPS: 69.98581650043909, Tokens per sec: 173432.56286221812, Loss: 2.538501501083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4331 , TFLOPS: 69.63455686381087, Tokens per sec: 172562.10278821015, Loss: 2.5542497634887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4332 , TFLOPS: 69.64971288104759, Tokens per sec: 172599.66106849586, Loss: 2.548184871673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4333 , TFLOPS: 71.52440780468669, Tokens per sec: 177245.3616039676, Loss: 2.571141242980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4334 , TFLOPS: 70.54623475574475, Tokens per sec: 174821.34103403278, Loss: 2.5336360931396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4335 , TFLOPS: 69.4698021604239, Tokens per sec: 172153.82248398836, Loss: 2.555429220199585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4336 , TFLOPS: 71.50695127004526, Tokens per sec: 177202.10238813012, Loss: 2.5612292289733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4337 , TFLOPS: 71.47748370083579, Tokens per sec: 177129.07850550834, Loss: 2.5392565727233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4338 , TFLOPS: 68.42617209705041, Tokens per sec: 169567.59222160676, Loss: 2.5406265258789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4339 , TFLOPS: 70.84722654281596, Tokens per sec: 175567.23183937775, Loss: 2.523444652557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4340 , TFLOPS: 71.54191000638392, Tokens per sec: 177288.73398780002, Loss: 2.5527100563049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4341 , TFLOPS: 70.53656488544166, Tokens per sec: 174797.37803019758, Loss: 2.5598649978637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4342 , TFLOPS: 69.85058038870072, Tokens per sec: 173097.43287984293, Loss: 2.5409491062164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4343 , TFLOPS: 71.51336164716182, Tokens per sec: 177217.98800878625, Loss: 2.555354118347168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4344 , TFLOPS: 71.50164862876261, Tokens per sec: 177188.96185889756, Loss: 2.549570083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4345 , TFLOPS: 67.006764955541, Tokens per sec: 166050.1449643419, Loss: 2.565234661102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4346 , TFLOPS: 71.51107766809194, Tokens per sec: 177212.3280570501, Loss: 2.566775321960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4347 , TFLOPS: 71.51547310501635, Tokens per sec: 177223.22043394615, Loss: 2.5581343173980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4348 , TFLOPS: 68.64959122043041, Tokens per sec: 170121.24942099059, Loss: 2.5502123832702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4349 , TFLOPS: 70.48533132655784, Tokens per sec: 174670.41562744105, Loss: 2.5508627891540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4350 , TFLOPS: 71.50552739367275, Tokens per sec: 177198.57386562612, Loss: 2.558220386505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4351 , TFLOPS: 69.65737438961496, Tokens per sec: 172618.64713071063, Loss: 2.5499513149261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4352 , TFLOPS: 69.75825499280195, Tokens per sec: 172868.64037832382, Loss: 2.5535264015197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4353 , TFLOPS: 70.51520118944676, Tokens per sec: 174744.4364381168, Loss: 2.5515286922454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4354 , TFLOPS: 71.51984000151268, Tokens per sec: 177234.0420845174, Loss: 2.525667667388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4355 , TFLOPS: 69.4508800159998, Tokens per sec: 172106.93132565918, Loss: 2.544097900390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4356 , TFLOPS: 70.53688266562165, Tokens per sec: 174798.16552448994, Loss: 2.5321528911590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4357 , TFLOPS: 71.49945287034224, Tokens per sec: 177183.52052764912, Loss: 2.4958949089050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4358 , TFLOPS: 69.57966868395735, Tokens per sec: 172426.08383210097, Loss: 2.5379786491394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4359 , TFLOPS: 69.99282616416261, Tokens per sec: 173449.93358110322, Loss: 2.5585620403289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4360 , TFLOPS: 70.39765894930889, Tokens per sec: 174453.1538187085, Loss: 2.554262638092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4361 , TFLOPS: 71.52649812436448, Tokens per sec: 177250.54164639692, Loss: 2.521082878112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4362 , TFLOPS: 69.43396143679477, Tokens per sec: 172065.0052226538, Loss: 2.5564024448394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4363 , TFLOPS: 71.52573638287258, Tokens per sec: 177248.65396706725, Loss: 2.528780937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4364 , TFLOPS: 70.50430662703954, Tokens per sec: 174717.43851233658, Loss: 2.5287411212921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4365 , TFLOPS: 67.9696654730351, Tokens per sec: 168436.3185189408, Loss: 2.537480592727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4366 , TFLOPS: 71.48793139081346, Tokens per sec: 177154.96903218105, Loss: 2.5465168952941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4367 , TFLOPS: 71.38981540841243, Tokens per sec: 176911.8268194236, Loss: 2.5474584102630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4368 , TFLOPS: 68.57419108838407, Tokens per sec: 169934.3995877695, Loss: 2.5237584114074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4369 , TFLOPS: 70.97991764566005, Tokens per sec: 175896.05500935705, Loss: 2.529536008834839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4370 , TFLOPS: 71.45187025475879, Tokens per sec: 177065.60556457553, Loss: 2.5503368377685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4371 , TFLOPS: 69.64012242409581, Tokens per sec: 172575.89485969328, Loss: 2.5832929611206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4372 , TFLOPS: 69.02410360222737, Tokens per sec: 171049.33235902808, Loss: 2.5400233268737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4373 , TFLOPS: 71.46362901308824, Tokens per sec: 177094.74506305004, Loss: 2.5570743083953857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4374 , TFLOPS: 71.48944188910541, Tokens per sec: 177158.71221334362, Loss: 2.558635711669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4375 , TFLOPS: 69.37480652168848, Tokens per sec: 171918.41282656856, Loss: 2.5542280673980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4376 , TFLOPS: 71.51363822317518, Tokens per sec: 177218.67339461474, Loss: 2.553769826889038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4377 , TFLOPS: 71.4789236189736, Tokens per sec: 177132.64678127388, Loss: 2.5289275646209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4378 , TFLOPS: 68.60995645002079, Tokens per sec: 170023.03009378514, Loss: 2.531686305999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4379 , TFLOPS: 69.76214929769773, Tokens per sec: 172878.29089484867, Loss: 2.544694423675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4380 , TFLOPS: 71.49873240357103, Tokens per sec: 177181.7351316241, Loss: 2.545079231262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4381 , TFLOPS: 71.45174255228076, Tokens per sec: 177065.2891037674, Loss: 2.551957130432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4382 , TFLOPS: 68.52324440830884, Tokens per sec: 169808.14810230298, Loss: 2.5526247024536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4383 , TFLOPS: 71.54774184217551, Tokens per sec: 177303.18591932135, Loss: 2.5561788082122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4384 , TFLOPS: 71.5674854817044, Tokens per sec: 177352.1128330154, Loss: 2.542362689971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4385 , TFLOPS: 67.9872323564925, Tokens per sec: 168479.85119130017, Loss: 2.527737855911255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4386 , TFLOPS: 70.52149609163872, Tokens per sec: 174760.03589918924, Loss: 2.5414559841156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4387 , TFLOPS: 71.53467430460579, Tokens per sec: 177270.80312171546, Loss: 2.52559757232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4388 , TFLOPS: 70.52350324061062, Tokens per sec: 174765.0098354473, Loss: 2.5278799533843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4389 , TFLOPS: 69.84000287408337, Tokens per sec: 173071.22063341172, Loss: 2.5528767108917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4390 , TFLOPS: 70.55107487353199, Tokens per sec: 174833.33537909255, Loss: 2.5308685302734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4391 , TFLOPS: 71.52435373268148, Tokens per sec: 177245.22760758217, Loss: 2.5631020069122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4392 , TFLOPS: 68.16029582637384, Tokens per sec: 168908.72153418115, Loss: 2.535684585571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4393 , TFLOPS: 70.41365208832806, Tokens per sec: 174492.78657898773, Loss: 2.5263447761535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4394 , TFLOPS: 71.54053683278072, Tokens per sec: 177285.33111234376, Loss: 2.529432773590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4395 , TFLOPS: 69.49574327648342, Tokens per sec: 172218.10742723374, Loss: 2.533906936645508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4396 , TFLOPS: 71.56799077252671, Tokens per sec: 177353.36499933546, Loss: 2.546814203262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4397 , TFLOPS: 70.50893739395943, Tokens per sec: 174728.91406288516, Loss: 2.535080909729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4398 , TFLOPS: 69.55126625126353, Tokens per sec: 172355.6994750994, Loss: 2.5408363342285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4399 , TFLOPS: 69.69657578583642, Tokens per sec: 172715.79250893768, Loss: 2.5187556743621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4400 , TFLOPS: 71.523946736019, Tokens per sec: 177244.2190250182, Loss: 2.520115613937378 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/optimizer.pt +[rank0]:[2024-08-26 11:18:47,815] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006368803005898371, 'preprocessing_with_comm': 0.0012646489994949661, 'state_converting': 0.9676959690041258, : 0.9766826880004373}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400, took 4.63s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002400 + eval ppl=11.599221229553223, eval loss=2.4509379863739014 +------------------------------------------------------------------ +iteration: 4401 , TFLOPS: 67.27381971347494, Tokens per sec: 166711.93607301233, Loss: 2.5513064861297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4402 , TFLOPS: 68.83561787164994, Tokens per sec: 170582.24395523992, Loss: 2.567462921142578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4403 , TFLOPS: 71.48839288330113, Tokens per sec: 177156.11266140314, Loss: 2.547016143798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4404 , TFLOPS: 70.44654535152736, Tokens per sec: 174574.29970868764, Loss: 2.5542855262756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4405 , TFLOPS: 69.5082459984062, Tokens per sec: 172249.09054944688, Loss: 2.5217082500457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4406 , TFLOPS: 70.86829679388914, Tokens per sec: 175619.4462426169, Loss: 2.5334582328796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4407 , TFLOPS: 71.54105533510725, Tokens per sec: 177286.61601822663, Loss: 2.5351569652557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4408 , TFLOPS: 70.59646539713513, Tokens per sec: 174945.8180967601, Loss: 2.5381534099578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4409 , TFLOPS: 68.99113390162475, Tokens per sec: 170967.62980902244, Loss: 2.540800094604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4410 , TFLOPS: 71.51817457996626, Tokens per sec: 177229.91498646402, Loss: 2.5478196144104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4411 , TFLOPS: 71.5020249051577, Tokens per sec: 177189.89431325486, Loss: 2.5471858978271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4412 , TFLOPS: 66.93605127885311, Tokens per sec: 165874.90868375474, Loss: 2.5657691955566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4413 , TFLOPS: 71.53531717564184, Tokens per sec: 177272.39622696093, Loss: 2.542959451675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4414 , TFLOPS: 71.46048961688366, Tokens per sec: 177086.96529342208, Loss: 2.56473970413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4415 , TFLOPS: 68.69116752339697, Tokens per sec: 170224.2800797496, Loss: 2.5508434772491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4416 , TFLOPS: 70.49773513721644, Tokens per sec: 174701.15363664558, Loss: 2.5529303550720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4417 , TFLOPS: 71.51700575997776, Tokens per sec: 177227.0185217764, Loss: 2.546473979949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4418 , TFLOPS: 70.61828825975546, Tokens per sec: 174999.89755432, Loss: 2.5583295822143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4419 , TFLOPS: 67.78553082188938, Tokens per sec: 167980.01256929533, Loss: 2.532529354095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4420 , TFLOPS: 71.5138330506428, Tokens per sec: 177219.15619854152, Loss: 2.5543313026428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4421 , TFLOPS: 71.46372834023148, Tokens per sec: 177094.9912066528, Loss: 2.5495963096618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4422 , TFLOPS: 69.55349495755404, Tokens per sec: 172361.22245479954, Loss: 2.551922082901001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4423 , TFLOPS: 70.48484494266623, Tokens per sec: 174669.21031457803, Loss: 2.5378012657165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4424 , TFLOPS: 71.54663042513276, Tokens per sec: 177300.43170545672, Loss: 2.5414111614227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4425 , TFLOPS: 70.61309353819506, Tokens per sec: 174987.02446204753, Loss: 2.558544635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4426 , TFLOPS: 69.83585723339063, Tokens per sec: 173060.94727909507, Loss: 2.5533909797668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4427 , TFLOPS: 70.47447738951998, Tokens per sec: 174643.51837580136, Loss: 2.5498909950256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4428 , TFLOPS: 71.49985477002447, Tokens per sec: 177184.5164793346, Loss: 2.5447206497192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4429 , TFLOPS: 68.69977282324777, Tokens per sec: 170245.6049607319, Loss: 2.5447916984558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4430 , TFLOPS: 70.4668012883742, Tokens per sec: 174624.49615156974, Loss: 2.52780818939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4431 , TFLOPS: 71.53989284321062, Tokens per sec: 177283.73523524773, Loss: 2.540879011154175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4432 , TFLOPS: 67.08260685280214, Tokens per sec: 166238.08954042968, Loss: 2.54091215133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4433 , TFLOPS: 71.49291646089834, Tokens per sec: 177167.32258500223, Loss: 2.575758457183838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4434 , TFLOPS: 70.55634116894723, Tokens per sec: 174846.38583926184, Loss: 2.525308847427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4435 , TFLOPS: 69.75765264475885, Tokens per sec: 172867.14769351893, Loss: 2.5373682975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4436 , TFLOPS: 69.94358396937182, Tokens per sec: 173327.9059979361, Loss: 2.534438133239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4437 , TFLOPS: 71.51135698897484, Tokens per sec: 177213.0202449676, Loss: 2.559013605117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4438 , TFLOPS: 70.3528292950322, Tokens per sec: 174342.06099701676, Loss: 2.5527353286743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4439 , TFLOPS: 68.31139700687596, Tokens per sec: 169283.16690463453, Loss: 2.5243797302246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4440 , TFLOPS: 71.48920098509299, Tokens per sec: 177158.1152266631, Loss: 2.5275754928588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4441 , TFLOPS: 71.55826602421062, Tokens per sec: 177329.26600173872, Loss: 2.5579323768615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4442 , TFLOPS: 68.46780920462186, Tokens per sec: 169670.77356087483, Loss: 2.5715713500976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4443 , TFLOPS: 71.52937292413476, Tokens per sec: 177257.66571691303, Loss: 2.562908887863159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4444 , TFLOPS: 71.53020582314107, Tokens per sec: 177259.72973240205, Loss: 2.5473568439483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4445 , TFLOPS: 69.54358696743724, Tokens per sec: 172336.6693638349, Loss: 2.5466933250427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4446 , TFLOPS: 69.48435905500672, Tokens per sec: 172189.89607233892, Loss: 2.5214157104492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4447 , TFLOPS: 71.56234065780274, Tokens per sec: 177339.36339263793, Loss: 2.5407118797302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4448 , TFLOPS: 71.53263357093276, Tokens per sec: 177265.74595886836, Loss: 2.5415380001068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4449 , TFLOPS: 68.5875632101222, Tokens per sec: 169967.5371784974, Loss: 2.5380420684814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4450 , TFLOPS: 71.50616707549086, Tokens per sec: 177200.1590676382, Loss: 2.537977933883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4451 , TFLOPS: 71.54001646268297, Tokens per sec: 177284.04157791805, Loss: 2.5569419860839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4452 , TFLOPS: 67.67237791421543, Tokens per sec: 167699.60719925753, Loss: 2.526049852371216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4453 , TFLOPS: 71.52900756029355, Tokens per sec: 177256.76030506633, Loss: 2.536412000656128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4454 , TFLOPS: 71.54683936494959, Tokens per sec: 177300.94948133946, Loss: 2.540523052215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4455 , TFLOPS: 70.59956070277524, Tokens per sec: 174953.48860511233, Loss: 2.531552314758301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4456 , TFLOPS: 68.13619324689182, Tokens per sec: 168848.99268710637, Loss: 2.535937786102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4457 , TFLOPS: 71.52410629328314, Tokens per sec: 177244.6144254954, Loss: 2.5554816722869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4458 , TFLOPS: 71.55650621947127, Tokens per sec: 177324.90501173612, Loss: 2.5476324558258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4459 , TFLOPS: 68.11135415442975, Tokens per sec: 168787.43867973346, Loss: 2.5280895233154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4460 , TFLOPS: 70.55312291293096, Tokens per sec: 174838.41064633336, Loss: 2.5446434020996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4461 , TFLOPS: 71.52364989538206, Tokens per sec: 177243.48342122018, Loss: 2.574401617050171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4462 , TFLOPS: 70.51180107233054, Tokens per sec: 174736.01057334902, Loss: 2.5440049171447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4463 , TFLOPS: 70.71057777626487, Tokens per sec: 175228.60114275987, Loss: 2.540994644165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4464 , TFLOPS: 70.43452245961667, Tokens per sec: 174544.50565810208, Loss: 2.521620750427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4465 , TFLOPS: 70.73953692643852, Tokens per sec: 175300.3651070039, Loss: 2.5533909797668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4466 , TFLOPS: 69.58318155186933, Tokens per sec: 172434.78910576119, Loss: 2.5414021015167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4467 , TFLOPS: 70.57752606267644, Tokens per sec: 174898.8843396315, Loss: 2.54335355758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4468 , TFLOPS: 71.44485229596297, Tokens per sec: 177048.21428958751, Loss: 2.554596424102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4469 , TFLOPS: 68.68863874957337, Tokens per sec: 170218.01349964726, Loss: 2.543128490447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4470 , TFLOPS: 71.57812684734708, Tokens per sec: 177378.48330932116, Loss: 2.5292742252349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4471 , TFLOPS: 70.4193473008147, Tokens per sec: 174506.89994290905, Loss: 2.5399630069732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4472 , TFLOPS: 68.40920027805764, Tokens per sec: 169525.53418454246, Loss: 2.5587782859802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4473 , TFLOPS: 70.8298931759866, Tokens per sec: 175524.27784694664, Loss: 2.548788070678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4474 , TFLOPS: 71.54547071053159, Tokens per sec: 177297.5578049219, Loss: 2.544851303100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4475 , TFLOPS: 70.52868041963754, Tokens per sec: 174777.83945538933, Loss: 2.5517578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4476 , TFLOPS: 67.90253705122207, Tokens per sec: 168269.96689488195, Loss: 2.528046131134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4477 , TFLOPS: 71.54498979209761, Tokens per sec: 177296.366036065, Loss: 2.5322391986846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4478 , TFLOPS: 70.46238255302242, Tokens per sec: 174613.54603860417, Loss: 2.560595989227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4479 , TFLOPS: 68.08542711672156, Tokens per sec: 168723.1886241936, Loss: 2.528268575668335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4480 , TFLOPS: 71.56528185681539, Tokens per sec: 177346.65200779107, Loss: 2.5324227809906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4481 , TFLOPS: 71.49064047750608, Tokens per sec: 177161.68244743592, Loss: 2.5447938442230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4482 , TFLOPS: 69.5559765289452, Tokens per sec: 172367.3720620745, Loss: 2.5605273246765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4483 , TFLOPS: 69.89385843492829, Tokens per sec: 173204.6806458652, Loss: 2.5456786155700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4484 , TFLOPS: 71.53485740643748, Tokens per sec: 177271.25686822477, Loss: 2.5557639598846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4485 , TFLOPS: 70.61211745456156, Tokens per sec: 174984.6056192792, Loss: 2.547743082046509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4486 , TFLOPS: 69.35683504456073, Tokens per sec: 171873.8775265243, Loss: 2.5346312522888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4487 , TFLOPS: 71.49791954830195, Tokens per sec: 177179.72078672392, Loss: 2.508709192276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4488 , TFLOPS: 71.50510448335548, Tokens per sec: 177197.5258472725, Loss: 2.5412447452545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4489 , TFLOPS: 68.69697292611995, Tokens per sec: 170238.66650721504, Loss: 2.5343148708343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4490 , TFLOPS: 71.51103767012958, Tokens per sec: 177212.2289376932, Loss: 2.5244336128234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4491 , TFLOPS: 71.52240011162996, Tokens per sec: 177240.38631940744, Loss: 2.5207457542419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4492 , TFLOPS: 69.61449013919716, Tokens per sec: 172512.37523408516, Loss: 2.5617940425872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4493 , TFLOPS: 69.07135350101255, Tokens per sec: 171166.4228132238, Loss: 2.543142318725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4494 , TFLOPS: 71.47883264530036, Tokens per sec: 177132.42133849004, Loss: 2.5395383834838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4495 , TFLOPS: 71.51988545838597, Tokens per sec: 177234.1547316569, Loss: 2.503690004348755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4496 , TFLOPS: 67.95354545725763, Tokens per sec: 168396.37134407787, Loss: 2.5491435527801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4497 , TFLOPS: 70.54156556267958, Tokens per sec: 174809.77025926323, Loss: 2.5312209129333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4498 , TFLOPS: 71.5573968706246, Tokens per sec: 177327.11214340737, Loss: 2.5325205326080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4499 , TFLOPS: 68.0670110389984, Tokens per sec: 168677.55155489716, Loss: 2.5357508659362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4500 , TFLOPS: 71.53858248232075, Tokens per sec: 177280.48801661446, Loss: 2.5375585556030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4501 , TFLOPS: 70.21468354805761, Tokens per sec: 173999.7206180034, Loss: 2.5344760417938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4502 , TFLOPS: 70.53485873282561, Tokens per sec: 174793.14999606676, Loss: 2.564678907394409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4503 , TFLOPS: 70.43458528816211, Tokens per sec: 174544.6613541588, Loss: 2.556614637374878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4504 , TFLOPS: 70.58480366359332, Tokens per sec: 174916.91903639748, Loss: 2.524709939956665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4505 , TFLOPS: 71.50278415234244, Tokens per sec: 177191.77581141665, Loss: 2.539492130279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4506 , TFLOPS: 68.85139200408521, Tokens per sec: 170621.33399307865, Loss: 2.547924757003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4507 , TFLOPS: 71.54382480381302, Tokens per sec: 177293.4790667617, Loss: 2.5553557872772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4508 , TFLOPS: 70.62571902849979, Tokens per sec: 175018.3118178352, Loss: 2.514570474624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4509 , TFLOPS: 68.84301010623783, Tokens per sec: 170600.56272687056, Loss: 2.5640084743499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4510 , TFLOPS: 71.4681130476603, Tokens per sec: 177105.8569946768, Loss: 2.5430526733398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4511 , TFLOPS: 71.47824001443563, Tokens per sec: 177130.95273392356, Loss: 2.550590753555298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4512 , TFLOPS: 69.59643835350215, Tokens per sec: 172467.6409205646, Loss: 2.5455753803253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4513 , TFLOPS: 68.95979075237804, Tokens per sec: 170889.95803245617, Loss: 2.5417964458465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4514 , TFLOPS: 71.5556747968375, Tokens per sec: 177322.84465486082, Loss: 2.537369966506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4515 , TFLOPS: 71.5680374999402, Tokens per sec: 177353.48079501351, Loss: 2.510317325592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4516 , TFLOPS: 66.92637765513207, Tokens per sec: 165850.93637853637, Loss: 2.5411744117736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4517 , TFLOPS: 71.49645576985122, Tokens per sec: 177176.0933824745, Loss: 2.5454585552215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4518 , TFLOPS: 71.51824650577443, Tokens per sec: 177230.09322653993, Loss: 2.536442279815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4519 , TFLOPS: 67.64452614809595, Tokens per sec: 167630.58745468812, Loss: 2.5520577430725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4520 , TFLOPS: 71.52506708226692, Tokens per sec: 177246.99536643684, Loss: 2.524923324584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4521 , TFLOPS: 71.53421452432589, Tokens per sec: 177269.6637355329, Loss: 2.549099922180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4522 , TFLOPS: 71.53493184991548, Tokens per sec: 177271.44134736384, Loss: 2.5527403354644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4523 , TFLOPS: 69.66984300980721, Tokens per sec: 172649.54574507894, Loss: 2.5724921226501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4524 , TFLOPS: 71.51364767036134, Tokens per sec: 177218.69680578276, Loss: 2.5642380714416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4525 , TFLOPS: 71.52790315141615, Tokens per sec: 177254.02345820746, Loss: 2.520907402038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4526 , TFLOPS: 67.85765525076765, Tokens per sec: 168158.74484921744, Loss: 2.534785747528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4527 , TFLOPS: 71.54964610256694, Tokens per sec: 177307.90488634293, Loss: 2.5457897186279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4528 , TFLOPS: 71.49931617228903, Tokens per sec: 177183.18177481473, Loss: 2.569042682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4529 , TFLOPS: 69.77677938632952, Tokens per sec: 172914.54586611557, Loss: 2.5343549251556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4530 , TFLOPS: 69.74898593063091, Tokens per sec: 172845.67062119252, Loss: 2.5563554763793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4531 , TFLOPS: 71.55211857927362, Tokens per sec: 177314.03195598788, Loss: 2.5568997859954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4532 , TFLOPS: 70.71563511161929, Tokens per sec: 175241.13377687958, Loss: 2.5308682918548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4533 , TFLOPS: 68.80100316876376, Tokens per sec: 170496.46490836365, Loss: 2.5244455337524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4534 , TFLOPS: 70.43223894010659, Tokens per sec: 174538.84684520564, Loss: 2.5331857204437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4535 , TFLOPS: 71.51701714019482, Tokens per sec: 177227.0467232079, Loss: 2.562180280685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4536 , TFLOPS: 67.94588945261329, Tokens per sec: 168377.39892119064, Loss: 2.550474166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4537 , TFLOPS: 71.55153620116337, Tokens per sec: 177312.58875887643, Loss: 2.5292680263519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4538 , TFLOPS: 70.56194217787724, Tokens per sec: 174860.26575639247, Loss: 2.5453813076019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4539 , TFLOPS: 68.53256526421868, Tokens per sec: 169831.2462100246, Loss: 2.5293455123901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4540 , TFLOPS: 70.97165076855832, Tokens per sec: 175875.5687772273, Loss: 2.5458574295043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4541 , TFLOPS: 69.69529331734789, Tokens per sec: 172712.61441074757, Loss: 2.5548248291015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4542 , TFLOPS: 71.48295716913091, Tokens per sec: 177142.64236289595, Loss: 2.5479955673217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4543 , TFLOPS: 69.4598167182218, Tokens per sec: 172129.07745822432, Loss: 2.5473978519439697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4544 , TFLOPS: 71.49572111279608, Tokens per sec: 177174.27282136268, Loss: 2.5382230281829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4545 , TFLOPS: 70.50136677129217, Tokens per sec: 174710.15322594275, Loss: 2.5406768321990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4546 , TFLOPS: 68.91945925349428, Tokens per sec: 170790.01213534083, Loss: 2.557389259338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4547 , TFLOPS: 71.5048842177737, Tokens per sec: 177196.98000489658, Loss: 2.5470094680786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4548 , TFLOPS: 71.49659979495348, Tokens per sec: 177176.4502925436, Loss: 2.5230894088745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4549 , TFLOPS: 68.76570804462295, Tokens per sec: 170408.99970266383, Loss: 2.538548469543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4550 , TFLOPS: 70.41414279085396, Tokens per sec: 174494.00259390223, Loss: 2.550931453704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4551 , TFLOPS: 71.5113793787486, Tokens per sec: 177213.07572929346, Loss: 2.543832302093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4552 , TFLOPS: 70.4883872948686, Tokens per sec: 174677.98865355778, Loss: 2.53951358795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4553 , TFLOPS: 67.95912144919029, Tokens per sec: 168410.18926633318, Loss: 2.5351059436798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4554 , TFLOPS: 71.46277195722752, Tokens per sec: 177092.6211842147, Loss: 2.5345447063446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4555 , TFLOPS: 71.48796195153491, Tokens per sec: 177155.04476501537, Loss: 2.5350468158721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4556 , TFLOPS: 67.62569837356965, Tokens per sec: 167583.9301553614, Loss: 2.5495762825012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4557 , TFLOPS: 71.55974958016633, Tokens per sec: 177332.94241682414, Loss: 2.543450355529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4558 , TFLOPS: 71.50276176795057, Tokens per sec: 177191.72034042774, Loss: 2.5636098384857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4559 , TFLOPS: 69.68624675960956, Tokens per sec: 172690.19604411136, Loss: 2.5252251625061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4560 , TFLOPS: 68.98944412053272, Tokens per sec: 170963.44234533166, Loss: 2.5469000339508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4561 , TFLOPS: 71.51520267797672, Tokens per sec: 177222.5502859519, Loss: 2.5396885871887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4562 , TFLOPS: 71.49330906732592, Tokens per sec: 177168.29550697896, Loss: 2.5415947437286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4563 , TFLOPS: 68.71538487133782, Tokens per sec: 170284.2933357081, Loss: 2.545630931854248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4564 , TFLOPS: 71.47989640613054, Tokens per sec: 177135.05745501016, Loss: 2.556553602218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4565 , TFLOPS: 71.53687315482873, Tokens per sec: 177276.2521147913, Loss: 2.54402494430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4566 , TFLOPS: 68.81450026177181, Tokens per sec: 170529.91219166512, Loss: 2.535289764404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4567 , TFLOPS: 70.50126868356938, Tokens per sec: 174709.91015376046, Loss: 2.5481245517730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4568 , TFLOPS: 71.47033585463713, Tokens per sec: 177111.3653552278, Loss: 2.530820369720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4569 , TFLOPS: 69.55081249962313, Tokens per sec: 172354.57502854965, Loss: 2.5411019325256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4570 , TFLOPS: 70.52906705452118, Tokens per sec: 174778.79757922236, Loss: 2.543407917022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4571 , TFLOPS: 70.57638154855607, Tokens per sec: 174896.0481075626, Loss: 2.5489823818206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4572 , TFLOPS: 71.49121797430541, Tokens per sec: 177163.1135481212, Loss: 2.5223031044006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4573 , TFLOPS: 68.0354220174285, Tokens per sec: 168599.27047962896, Loss: 2.539902687072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4574 , TFLOPS: 71.51605874099577, Tokens per sec: 177224.67170441625, Loss: 2.543584108352661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4575 , TFLOPS: 70.15297028602153, Tokens per sec: 173846.78835640097, Loss: 2.554696559906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4576 , TFLOPS: 67.7736196961433, Tokens per sec: 167950.49548757772, Loss: 2.5519368648529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4577 , TFLOPS: 71.54073577032682, Tokens per sec: 177285.82410149774, Loss: 2.5617010593414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4578 , TFLOPS: 69.1705707884963, Tokens per sec: 171412.29418129503, Loss: 2.543236255645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4579 , TFLOPS: 70.68808095476513, Tokens per sec: 175172.85154085403, Loss: 2.551907777786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4580 , TFLOPS: 69.17005998778784, Tokens per sec: 171411.02836087006, Loss: 2.5633556842803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4581 , TFLOPS: 71.55851909691393, Tokens per sec: 177329.89314377576, Loss: 2.56021785736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4582 , TFLOPS: 70.63362439615052, Tokens per sec: 175037.9021897219, Loss: 2.5471205711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4583 , TFLOPS: 69.43694534869721, Tokens per sec: 172072.39968505307, Loss: 2.528653144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4584 , TFLOPS: 71.45081526396572, Tokens per sec: 177062.99118117348, Loss: 2.5525412559509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4585 , TFLOPS: 71.4710020376147, Tokens per sec: 177113.01623003234, Loss: 2.532526731491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4586 , TFLOPS: 67.51179706513203, Tokens per sec: 167301.670165788, Loss: 2.549149513244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4587 , TFLOPS: 71.49894158515629, Tokens per sec: 177182.2535066356, Loss: 2.5463314056396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4588 , TFLOPS: 71.54950441640182, Tokens per sec: 177307.55377241774, Loss: 2.549520254135132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4589 , TFLOPS: 70.56905103397634, Tokens per sec: 174877.88228491117, Loss: 2.552100419998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4590 , TFLOPS: 69.52644174816066, Tokens per sec: 172294.18162176575, Loss: 2.511975049972534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4591 , TFLOPS: 71.51379708727436, Tokens per sec: 177219.06707735284, Loss: 2.542325258255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4592 , TFLOPS: 71.51824805158275, Tokens per sec: 177230.0970572232, Loss: 2.5446057319641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4593 , TFLOPS: 67.85167454004534, Tokens per sec: 168143.92398921787, Loss: 2.540928363800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4594 , TFLOPS: 71.51278765631592, Tokens per sec: 177216.56559623987, Loss: 2.5303640365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4595 , TFLOPS: 71.48046887508502, Tokens per sec: 177136.47609614203, Loss: 2.5540199279785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4596 , TFLOPS: 68.86745424709373, Tokens per sec: 170661.13800065592, Loss: 2.5457496643066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4597 , TFLOPS: 69.80805712019064, Tokens per sec: 172992.05553615023, Loss: 2.544043779373169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4598 , TFLOPS: 71.51208766896501, Tokens per sec: 177214.8309504743, Loss: 2.5462584495544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4599 , TFLOPS: 70.6314675576607, Tokens per sec: 175032.5573063487, Loss: 2.562009572982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4600 , TFLOPS: 67.64911006485728, Tokens per sec: 167641.94690538282, Loss: 2.546828031539917 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/model.pt +[rank0]:[2024-08-26 11:44:00,627] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006500155999674462, 'preprocessing_with_comm': 0.0014144789965939708, 'state_converting': 0.9862020799992024, : 0.9955061189975822}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600, took 4.72s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002600 + eval ppl=10.646491050720215, eval loss=2.3652303218841553 +------------------------------------------------------------------ +iteration: 4601 , TFLOPS: 67.26578044037288, Tokens per sec: 166692.01386866637, Loss: 2.539039134979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4602 , TFLOPS: 71.46982926931949, Tokens per sec: 177110.1099810057, Loss: 2.5429325103759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4603 , TFLOPS: 67.92074640104813, Tokens per sec: 168315.09166967642, Loss: 2.5513365268707275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4604 , TFLOPS: 70.12840059358531, Tokens per sec: 173785.901951969, Loss: 2.525987148284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4605 , TFLOPS: 71.56495796425853, Tokens per sec: 177345.84936635545, Loss: 2.5522587299346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4606 , TFLOPS: 69.83667949602697, Tokens per sec: 173062.98493648766, Loss: 2.517460584640503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4607 , TFLOPS: 69.90252818688205, Tokens per sec: 173226.16524625866, Loss: 2.526414394378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4608 , TFLOPS: 70.39244106947037, Tokens per sec: 174440.22333767175, Loss: 2.5394062995910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4609 , TFLOPS: 71.50356666007129, Tokens per sec: 177193.71495176907, Loss: 2.5472793579101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4610 , TFLOPS: 68.64111678473147, Tokens per sec: 170100.24883579244, Loss: 2.5271360874176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4611 , TFLOPS: 70.4539560326466, Tokens per sec: 174592.66419285507, Loss: 2.5454771518707275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4612 , TFLOPS: 71.50901314909498, Tokens per sec: 177207.21195155007, Loss: 2.521427631378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4613 , TFLOPS: 68.11496692223079, Tokens per sec: 168796.39151632367, Loss: 2.5377166271209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4614 , TFLOPS: 71.47270168655896, Tokens per sec: 177117.228147348, Loss: 2.5354506969451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4615 , TFLOPS: 70.5198176261223, Tokens per sec: 174755.87647674128, Loss: 2.547666072845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4616 , TFLOPS: 68.99738492592569, Tokens per sec: 170983.12053584732, Loss: 2.5344011783599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4617 , TFLOPS: 70.50273419699826, Tokens per sec: 174713.54185747687, Loss: 2.5608208179473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4618 , TFLOPS: 70.46920670516973, Tokens per sec: 174630.45703936758, Loss: 2.516622543334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4619 , TFLOPS: 71.49719625439376, Tokens per sec: 177177.92838474203, Loss: 2.544846534729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4620 , TFLOPS: 68.81219955889614, Tokens per sec: 170524.21079649587, Loss: 2.5471062660217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4621 , TFLOPS: 71.50433648571386, Tokens per sec: 177195.62266451467, Loss: 2.5305469036102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4622 , TFLOPS: 70.5239292462157, Tokens per sec: 174766.06552426508, Loss: 2.5425891876220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4623 , TFLOPS: 67.93768496901275, Tokens per sec: 168357.06730703005, Loss: 2.5322601795196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4624 , TFLOPS: 71.46675950106419, Tokens per sec: 177102.50275710615, Loss: 2.547452449798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4625 , TFLOPS: 71.461700854379, Tokens per sec: 177089.96687336435, Loss: 2.5510146617889404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4626 , TFLOPS: 69.61011927163771, Tokens per sec: 172501.54374278203, Loss: 2.5434083938598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4627 , TFLOPS: 69.8254019418312, Tokens per sec: 173035.0379148654, Loss: 2.5353310108184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4628 , TFLOPS: 71.50734734329527, Tokens per sec: 177203.0839012747, Loss: 2.56309175491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4629 , TFLOPS: 71.5008240424408, Tokens per sec: 177186.91844315644, Loss: 2.54502010345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4630 , TFLOPS: 67.7900534928557, Tokens per sec: 167991.22024616323, Loss: 2.5526957511901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4631 , TFLOPS: 71.53599183928615, Tokens per sec: 177274.06811779187, Loss: 2.558551073074341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4632 , TFLOPS: 71.54548480694687, Tokens per sec: 177297.59273739182, Loss: 2.5482892990112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4633 , TFLOPS: 67.37043124285103, Tokens per sec: 166951.3500854448, Loss: 2.5503828525543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4634 , TFLOPS: 71.54261013306068, Tokens per sec: 177290.46897883038, Loss: 2.565135955810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4635 , TFLOPS: 71.49963690238023, Tokens per sec: 177183.9765793121, Loss: 2.5294551849365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4636 , TFLOPS: 69.94299536281689, Tokens per sec: 173326.44736605263, Loss: 2.553415536880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4637 , TFLOPS: 68.80475985938482, Tokens per sec: 170505.7744015563, Loss: 2.5458083152770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4638 , TFLOPS: 71.53750854514215, Tokens per sec: 177277.82668198197, Loss: 2.530770778656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4639 , TFLOPS: 71.52352492183266, Tokens per sec: 177243.17372299777, Loss: 2.521378993988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4640 , TFLOPS: 68.70427608599124, Tokens per sec: 170256.76454188555, Loss: 2.551832437515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4641 , TFLOPS: 70.40025535432333, Tokens per sec: 174459.58799635194, Loss: 2.5424275398254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4642 , TFLOPS: 71.44216188123696, Tokens per sec: 177041.54714552278, Loss: 2.5180320739746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4643 , TFLOPS: 69.16413449010422, Tokens per sec: 171396.34432486264, Loss: 2.5111474990844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4644 , TFLOPS: 69.73560552859459, Tokens per sec: 172812.51251097355, Loss: 2.544682741165161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4645 , TFLOPS: 71.51287220591892, Tokens per sec: 177216.77511947, Loss: 2.5252649784088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4646 , TFLOPS: 70.42838816780505, Tokens per sec: 174529.30420724486, Loss: 2.5435376167297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4647 , TFLOPS: 69.42873499548148, Tokens per sec: 172052.05352533938, Loss: 2.5435824394226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4648 , TFLOPS: 70.264211545985, Tokens per sec: 174122.45645282688, Loss: 2.55751633644104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4649 , TFLOPS: 71.47407110905122, Tokens per sec: 177120.62172713818, Loss: 2.564229726791382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4650 , TFLOPS: 68.62631917144984, Tokens per sec: 170063.5787199889, Loss: 2.5573017597198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4651 , TFLOPS: 71.47336336908924, Tokens per sec: 177118.86786954838, Loss: 2.518887519836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4652 , TFLOPS: 70.49793061729154, Tokens per sec: 174701.6380578056, Loss: 2.5231056213378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4653 , TFLOPS: 68.3941545040075, Tokens per sec: 169488.2490990174, Loss: 2.5330066680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4654 , TFLOPS: 71.50783458409168, Tokens per sec: 177204.29133764224, Loss: 2.5434682369232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4655 , TFLOPS: 70.53685935942863, Tokens per sec: 174798.10776917628, Loss: 2.5355615615844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4656 , TFLOPS: 70.65589609864192, Tokens per sec: 175093.0939219261, Loss: 2.5266926288604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4657 , TFLOPS: 69.09955293082747, Tokens per sec: 171236.30410672878, Loss: 2.5384342670440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4658 , TFLOPS: 71.44579046886687, Tokens per sec: 177050.53918539168, Loss: 2.529317617416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4659 , TFLOPS: 70.42459718939271, Tokens per sec: 174519.9097451283, Loss: 2.524392604827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4660 , TFLOPS: 68.6493654527708, Tokens per sec: 170120.6899438597, Loss: 2.5194180011749268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4661 , TFLOPS: 71.54489791542213, Tokens per sec: 177296.1383555421, Loss: 2.5623042583465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4662 , TFLOPS: 71.54109772550083, Tokens per sec: 177286.72106629165, Loss: 2.5371642112731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4663 , TFLOPS: 68.20952787475863, Tokens per sec: 169030.72397343648, Loss: 2.5231528282165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4664 , TFLOPS: 70.32639754908334, Tokens per sec: 174276.5602188583, Loss: 2.534400701522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4665 , TFLOPS: 71.52749794075545, Tokens per sec: 177253.01930155262, Loss: 2.531419038772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4666 , TFLOPS: 70.5208146712696, Tokens per sec: 174758.3472644499, Loss: 2.5287840366363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4667 , TFLOPS: 69.24992601170302, Tokens per sec: 171608.94516610075, Loss: 2.5272421836853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4668 , TFLOPS: 71.49443708515115, Tokens per sec: 177171.09085941126, Loss: 2.542564630508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4669 , TFLOPS: 71.45823441801294, Tokens per sec: 177081.37666218975, Loss: 2.560697555541992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4670 , TFLOPS: 67.81665995437957, Tokens per sec: 168057.15398876308, Loss: 2.5325374603271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4671 , TFLOPS: 71.51636406809484, Tokens per sec: 177225.4283386023, Loss: 2.561065912246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4672 , TFLOPS: 71.51482106571358, Tokens per sec: 177221.60460872576, Loss: 2.5596044063568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4673 , TFLOPS: 69.63338558874266, Tokens per sec: 172559.2002395645, Loss: 2.5230188369750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4674 , TFLOPS: 69.66913877404872, Tokens per sec: 172647.80057129142, Loss: 2.536569118499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4675 , TFLOPS: 71.52207268435082, Tokens per sec: 177239.5749185406, Loss: 2.5512619018554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4676 , TFLOPS: 71.49680499567243, Tokens per sec: 177176.95880252987, Loss: 2.5427870750427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4677 , TFLOPS: 68.29602844171225, Tokens per sec: 169245.08190717164, Loss: 2.5448622703552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4678 , TFLOPS: 70.3983253827782, Tokens per sec: 174454.80531425917, Loss: 2.519228219985962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4679 , TFLOPS: 71.50529665670803, Tokens per sec: 177198.00207400977, Loss: 2.554492712020874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4680 , TFLOPS: 68.64196866223116, Tokens per sec: 170102.35988207874, Loss: 2.528878688812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4681 , TFLOPS: 71.50331725931555, Tokens per sec: 177193.09690922257, Loss: 2.5204432010650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4682 , TFLOPS: 69.93989060631611, Tokens per sec: 173318.75343742652, Loss: 2.539832592010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4683 , TFLOPS: 69.05019043994227, Tokens per sec: 171113.97841658798, Loss: 2.5150485038757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4684 , TFLOPS: 70.3548647654306, Tokens per sec: 174347.10511688958, Loss: 2.525003433227539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4685 , TFLOPS: 69.91778976961885, Tokens per sec: 173263.98512948258, Loss: 2.4958834648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4686 , TFLOPS: 71.48922757942343, Tokens per sec: 177158.18113034352, Loss: 2.531803846359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4687 , TFLOPS: 68.94988447091733, Tokens per sec: 170865.4091757295, Loss: 2.5296120643615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4688 , TFLOPS: 71.49917739857415, Tokens per sec: 177182.8378782621, Loss: 2.521836280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4689 , TFLOPS: 70.46349245298269, Tokens per sec: 174616.2964929713, Loss: 2.538665294647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4690 , TFLOPS: 68.65221521282919, Tokens per sec: 170127.75196321137, Loss: 2.5451998710632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4691 , TFLOPS: 71.47059705651658, Tokens per sec: 177112.01264225884, Loss: 2.5423429012298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4692 , TFLOPS: 70.57745915897132, Tokens per sec: 174898.71854488016, Loss: 2.5558080673217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4693 , TFLOPS: 70.64520584435881, Tokens per sec: 175066.60229418386, Loss: 2.5379793643951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4694 , TFLOPS: 69.67114443717819, Tokens per sec: 172652.7708254683, Loss: 2.5441079139709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4695 , TFLOPS: 71.50279416648914, Tokens per sec: 177191.8006275753, Loss: 2.5387673377990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4696 , TFLOPS: 70.47468030173354, Tokens per sec: 174644.0212146192, Loss: 2.537858486175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4697 , TFLOPS: 68.0537797407723, Tokens per sec: 168644.76294034504, Loss: 2.529337167739868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4698 , TFLOPS: 71.43402095384187, Tokens per sec: 177021.3730306406, Loss: 2.522068977355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4699 , TFLOPS: 71.52066456274733, Tokens per sec: 177236.08543808974, Loss: 2.533752202987671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4700 , TFLOPS: 67.7421546623342, Tokens per sec: 167872.52166763917, Loss: 2.534977674484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4701 , TFLOPS: 71.53345689050178, Tokens per sec: 177267.78623545653, Loss: 2.546433687210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4702 , TFLOPS: 71.457889860737, Tokens per sec: 177080.52281130388, Loss: 2.5335919857025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4703 , TFLOPS: 69.77438147568965, Tokens per sec: 172908.6035793983, Loss: 2.548905372619629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4704 , TFLOPS: 68.74073768375224, Tokens per sec: 170347.1203977139, Loss: 2.547795057296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4705 , TFLOPS: 71.52817695050554, Tokens per sec: 177254.70196251245, Loss: 2.5544633865356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4706 , TFLOPS: 71.52314058293189, Tokens per sec: 177242.22128886325, Loss: 2.524322748184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4707 , TFLOPS: 67.77532204679953, Tokens per sec: 167954.71410003328, Loss: 2.5439229011535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4708 , TFLOPS: 71.4359562168396, Tokens per sec: 177026.16882553577, Loss: 2.54634952545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4709 , TFLOPS: 71.51104632458866, Tokens per sec: 177212.25038439617, Loss: 2.559115409851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4710 , TFLOPS: 69.65621189609608, Tokens per sec: 172615.76634371164, Loss: 2.539351224899292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4711 , TFLOPS: 69.7546716550399, Tokens per sec: 172859.76047261222, Loss: 2.537856101989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4712 , TFLOPS: 71.48488519133622, Tokens per sec: 177147.42021430988, Loss: 2.5367984771728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4713 , TFLOPS: 70.59355771394998, Tokens per sec: 174938.61253751954, Loss: 2.5446417331695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4714 , TFLOPS: 69.57073857149284, Tokens per sec: 172403.9540296808, Loss: 2.5362415313720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4715 , TFLOPS: 70.36670821908542, Tokens per sec: 174376.45449971038, Loss: 2.562572956085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4716 , TFLOPS: 71.5296088031836, Tokens per sec: 177258.25025118026, Loss: 2.5402698516845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4717 , TFLOPS: 67.85466382442164, Tokens per sec: 168151.3317651992, Loss: 2.545780658721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4718 , TFLOPS: 70.56060849995575, Tokens per sec: 174856.96075558627, Loss: 2.521298885345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4719 , TFLOPS: 71.54435919965213, Tokens per sec: 177294.8033585196, Loss: 2.5252509117126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4720 , TFLOPS: 68.78368314115072, Tokens per sec: 170453.54397198, Loss: 2.5199971199035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4721 , TFLOPS: 71.5158746630707, Tokens per sec: 177224.21553904007, Loss: 2.514500617980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4722 , TFLOPS: 70.5020048453324, Tokens per sec: 174711.7344437041, Loss: 2.5259850025177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4723 , TFLOPS: 70.56318120118534, Tokens per sec: 174863.33619263963, Loss: 2.5644469261169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4724 , TFLOPS: 68.82231688177752, Tokens per sec: 170549.28263711106, Loss: 2.5131006240844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4725 , TFLOPS: 71.55634224292214, Tokens per sec: 177324.49865978386, Loss: 2.5324227809906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4726 , TFLOPS: 70.45793953309516, Tokens per sec: 174602.53574578534, Loss: 2.5406110286712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4727 , TFLOPS: 68.86384465338483, Tokens per sec: 170652.19302981565, Loss: 2.549511432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4728 , TFLOPS: 71.52653746325035, Tokens per sec: 177250.63913248962, Loss: 2.5218029022216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4729 , TFLOPS: 70.43196362868828, Tokens per sec: 174538.16459318303, Loss: 2.526283025741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4730 , TFLOPS: 69.56966419576909, Tokens per sec: 172401.2916082851, Loss: 2.5459609031677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4731 , TFLOPS: 70.38396797779201, Tokens per sec: 174419.22608310467, Loss: 2.5460753440856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4732 , TFLOPS: 71.48513949225229, Tokens per sec: 177148.05039999346, Loss: 2.5408403873443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4733 , TFLOPS: 69.72857833830189, Tokens per sec: 172795.0983593193, Loss: 2.508114814758301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4734 , TFLOPS: 69.55476767141954, Tokens per sec: 172364.37637995929, Loss: 2.5342578887939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4735 , TFLOPS: 71.46869688569953, Tokens per sec: 177107.3038096531, Loss: 2.5261383056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4736 , TFLOPS: 71.52409212795621, Tokens per sec: 177244.5793222548, Loss: 2.5501291751861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4737 , TFLOPS: 66.44639137058368, Tokens per sec: 164661.4774905713, Loss: 2.5462887287139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4738 , TFLOPS: 71.46356836736936, Tokens per sec: 177094.594776278, Loss: 2.5378150939941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4739 , TFLOPS: 71.42284694153766, Tokens per sec: 176993.68259723164, Loss: 2.542795181274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4740 , TFLOPS: 69.619447374289, Tokens per sec: 172524.65980872727, Loss: 2.55757212638855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4741 , TFLOPS: 69.93969143642809, Tokens per sec: 173318.25987250364, Loss: 2.5481925010681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4742 , TFLOPS: 71.5365643564856, Tokens per sec: 177275.48687848003, Loss: 2.5465304851531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4743 , TFLOPS: 71.5033197990574, Tokens per sec: 177193.10320298263, Loss: 2.553764820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4744 , TFLOPS: 67.085207880057, Tokens per sec: 166244.535172494, Loss: 2.5404653549194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4745 , TFLOPS: 71.5260589349243, Tokens per sec: 177249.45328658342, Loss: 2.5389671325683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4746 , TFLOPS: 71.52560113991704, Tokens per sec: 177248.3188201252, Loss: 2.517631769180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4747 , TFLOPS: 68.40274928892663, Tokens per sec: 169509.5479228407, Loss: 2.5466930866241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4748 , TFLOPS: 70.57101033904321, Tokens per sec: 174882.7376587022, Loss: 2.541879177093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4749 , TFLOPS: 71.49811694535262, Tokens per sec: 177180.20995836056, Loss: 2.5453288555145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4750 , TFLOPS: 69.53372133319758, Tokens per sec: 172312.22123539972, Loss: 2.541780948638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4751 , TFLOPS: 70.37830441958452, Tokens per sec: 174405.19116197407, Loss: 2.5299932956695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4752 , TFLOPS: 70.51492689150618, Tokens per sec: 174743.75669760365, Loss: 2.5127241611480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4753 , TFLOPS: 71.46010625416643, Tokens per sec: 177086.0152783779, Loss: 2.5521295070648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4754 , TFLOPS: 68.9950628428568, Tokens per sec: 170977.36615820404, Loss: 2.5335800647735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4755 , TFLOPS: 71.47547044889707, Tokens per sec: 177124.0894454262, Loss: 2.537047863006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4756 , TFLOPS: 70.3764674326772, Tokens per sec: 174400.6389060569, Loss: 2.5254600048065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4757 , TFLOPS: 67.77548605358457, Tokens per sec: 167955.12052691347, Loss: 2.546783924102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4758 , TFLOPS: 71.4446178433074, Tokens per sec: 177047.6332900799, Loss: 2.54430890083313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4759 , TFLOPS: 70.40556889032615, Tokens per sec: 174472.75552390123, Loss: 2.5435969829559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4760 , TFLOPS: 70.67689876216524, Tokens per sec: 175145.14083577128, Loss: 2.529402256011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4761 , TFLOPS: 69.83789355747247, Tokens per sec: 173065.9935144895, Loss: 2.5588972568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4762 , TFLOPS: 71.50363366043071, Tokens per sec: 177193.8809860404, Loss: 2.532022476196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4763 , TFLOPS: 70.53308855877918, Tokens per sec: 174788.76330977897, Loss: 2.533290386199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4764 , TFLOPS: 67.80068382116632, Tokens per sec: 168017.56337074426, Loss: 2.5281565189361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4765 , TFLOPS: 71.5335164660198, Tokens per sec: 177267.93387015298, Loss: 2.5544943809509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4766 , TFLOPS: 70.58239633335909, Tokens per sec: 174910.95340688774, Loss: 2.540020227432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4767 , TFLOPS: 68.79708043825181, Tokens per sec: 170486.74395003062, Loss: 2.556199550628662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4768 , TFLOPS: 71.50224274864127, Tokens per sec: 177190.43415340455, Loss: 2.5415523052215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4769 , TFLOPS: 71.49101952446127, Tokens per sec: 177162.62176754634, Loss: 2.5357754230499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4770 , TFLOPS: 69.44131588986835, Tokens per sec: 172083.23036753, Loss: 2.555683135986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4771 , TFLOPS: 69.76775546284385, Tokens per sec: 172892.18358965084, Loss: 2.5429046154022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4772 , TFLOPS: 71.49770457947322, Tokens per sec: 177179.18807028557, Loss: 2.5357613563537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4773 , TFLOPS: 71.5049199595517, Tokens per sec: 177197.06857695978, Loss: 2.5282344818115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4774 , TFLOPS: 67.8949928297483, Tokens per sec: 168251.27148300537, Loss: 2.533393621444702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4775 , TFLOPS: 71.5524093028997, Tokens per sec: 177314.75240115912, Loss: 2.5413296222686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4776 , TFLOPS: 71.4442952830645, Tokens per sec: 177046.83395026508, Loss: 2.552602767944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4777 , TFLOPS: 68.92321848335187, Tokens per sec: 170799.32792104132, Loss: 2.5273523330688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4778 , TFLOPS: 69.63377806701125, Tokens per sec: 172560.1728439491, Loss: 2.509817600250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4779 , TFLOPS: 71.51822405377914, Tokens per sec: 177230.0375880223, Loss: 2.5324008464813232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4780 , TFLOPS: 70.575874169188, Tokens per sec: 174894.79076559606, Loss: 2.5448927879333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4781 , TFLOPS: 68.20425577418564, Tokens per sec: 169017.65912744682, Loss: 2.555786371231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4782 , TFLOPS: 71.53499755586037, Tokens per sec: 177271.6041739334, Loss: 2.536348581314087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4783 , TFLOPS: 71.4920440478354, Tokens per sec: 177165.16064932803, Loss: 2.522374153137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4784 , TFLOPS: 67.79709281959498, Tokens per sec: 168008.66447326902, Loss: 2.5307962894439697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4785 , TFLOPS: 70.43936213285946, Tokens per sec: 174556.4989015317, Loss: 2.549262285232544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4786 , TFLOPS: 71.57862955943483, Tokens per sec: 177379.72908525288, Loss: 2.5287370681762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4787 , TFLOPS: 68.69623266240114, Tokens per sec: 170236.83205217303, Loss: 2.540194272994995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4788 , TFLOPS: 71.5397688085555, Tokens per sec: 177283.42786370878, Loss: 2.520148992538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4789 , TFLOPS: 70.46001534514609, Tokens per sec: 174607.67983671714, Loss: 2.5436277389526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4790 , TFLOPS: 71.48747276133344, Tokens per sec: 177153.83249780734, Loss: 2.5436360836029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4791 , TFLOPS: 68.87065700735268, Tokens per sec: 170669.07479339073, Loss: 2.5524513721466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4792 , TFLOPS: 70.49459549529202, Tokens per sec: 174693.37325809678, Loss: 2.5119380950927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4793 , TFLOPS: 71.46971980457329, Tokens per sec: 177109.8387153061, Loss: 2.5415706634521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4794 , TFLOPS: 68.7247799501449, Tokens per sec: 170307.57537594545, Loss: 2.5333266258239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4795 , TFLOPS: 71.54908542397727, Tokens per sec: 177306.51546303387, Loss: 2.5400495529174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4796 , TFLOPS: 70.56520708879401, Tokens per sec: 174868.35656530387, Loss: 2.5365183353424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4797 , TFLOPS: 68.92970152094038, Tokens per sec: 170815.3936023512, Loss: 2.537559747695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4798 , TFLOPS: 70.40958823382101, Tokens per sec: 174482.7158998487, Loss: 2.532660722732544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4799 , TFLOPS: 71.50844284032024, Tokens per sec: 177205.79866358158, Loss: 2.5386266708374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4800 , TFLOPS: 70.42766881340351, Tokens per sec: 174527.52156779444, Loss: 2.531597137451172 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/model.pt +[rank0]:[2024-08-26 12:09:14,112] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00648292600089917, 'preprocessing_with_comm': 0.0012969220042577945, 'state_converting': 0.9739873049984453, : 0.9831627849998767}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800, took 4.57s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0002800 + eval ppl=10.543620109558105, eval loss=2.3555209636688232 +------------------------------------------------------------------ +iteration: 4801 , TFLOPS: 63.299601963591556, Tokens per sec: 156863.3866925751, Loss: 2.5201048851013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4802 , TFLOPS: 71.45509158040676, Tokens per sec: 177073.58836438952, Loss: 2.5234954357147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4803 , TFLOPS: 70.17229710902205, Tokens per sec: 173894.6823528216, Loss: 2.5430307388305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4804 , TFLOPS: 68.77779745746636, Tokens per sec: 170438.95859944855, Loss: 2.5430221557617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4805 , TFLOPS: 71.502262631685, Tokens per sec: 177190.48342577723, Loss: 2.5618457794189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4806 , TFLOPS: 71.52173173336567, Tokens per sec: 177238.73000444056, Loss: 2.536123275756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4807 , TFLOPS: 68.66410221956609, Tokens per sec: 170157.2092753673, Loss: 2.5503745079040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4808 , TFLOPS: 71.54661454915131, Tokens per sec: 177300.3923630258, Loss: 2.532689332962036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4809 , TFLOPS: 71.50958094610569, Tokens per sec: 177208.61901509057, Loss: 2.5503318309783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4810 , TFLOPS: 70.73303713581093, Tokens per sec: 175284.2579098177, Loss: 2.542726755142212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4811 , TFLOPS: 68.36565099097072, Tokens per sec: 169417.61425379105, Loss: 2.5374083518981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4812 , TFLOPS: 71.52567236563785, Tokens per sec: 177248.4953253075, Loss: 2.550163984298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4813 , TFLOPS: 71.53589293946953, Tokens per sec: 177273.82303315154, Loss: 2.525218963623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4814 , TFLOPS: 68.64002155747546, Tokens per sec: 170097.5347420028, Loss: 2.5350966453552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4815 , TFLOPS: 70.50594971666489, Tokens per sec: 174721.51026942948, Loss: 2.539882183074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4816 , TFLOPS: 71.49120266361074, Tokens per sec: 177163.07560653315, Loss: 2.5504136085510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4817 , TFLOPS: 69.5884388094873, Tokens per sec: 172447.81716927324, Loss: 2.4964447021484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4818 , TFLOPS: 69.17739867251055, Tokens per sec: 171429.21443003582, Loss: 2.5476574897766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4819 , TFLOPS: 71.51632937743403, Tokens per sec: 177225.34237132332, Loss: 2.5240135192871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4820 , TFLOPS: 71.53748689287727, Tokens per sec: 177277.77302528438, Loss: 2.537445068359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4821 , TFLOPS: 69.02899757455091, Tokens per sec: 171061.46016156138, Loss: 2.532449960708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4822 , TFLOPS: 70.56831015106967, Tokens per sec: 174876.04629545347, Loss: 2.5368974208831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4823 , TFLOPS: 71.52211676237864, Tokens per sec: 177239.68414874916, Loss: 2.526540756225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4824 , TFLOPS: 67.62478251363815, Tokens per sec: 167581.66055356083, Loss: 2.543972969055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4825 , TFLOPS: 71.49056478442044, Tokens per sec: 177161.4948716314, Loss: 2.528660535812378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4826 , TFLOPS: 70.49834780900946, Tokens per sec: 174702.67190484007, Loss: 2.5361459255218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4827 , TFLOPS: 69.4968510120391, Tokens per sec: 172220.85251796784, Loss: 2.5411627292633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4828 , TFLOPS: 70.98528109065505, Tokens per sec: 175909.34621688692, Loss: 2.548645257949829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4829 , TFLOPS: 71.50993322682105, Tokens per sec: 177209.4920055102, Loss: 2.552647829055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4830 , TFLOPS: 69.78529830933722, Tokens per sec: 172935.65669576547, Loss: 2.525934934616089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4831 , TFLOPS: 68.7334528977986, Tokens per sec: 170329.06789563535, Loss: 2.530226707458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4832 , TFLOPS: 71.53315985496005, Tokens per sec: 177267.05014866294, Loss: 2.511890411376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4833 , TFLOPS: 70.45059182725615, Tokens per sec: 174584.32732129967, Loss: 2.537553071975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4834 , TFLOPS: 68.78252129909492, Tokens per sec: 170450.66479937828, Loss: 2.520951509475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4835 , TFLOPS: 71.57220137114786, Tokens per sec: 177363.79932655438, Loss: 2.541428804397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4836 , TFLOPS: 71.50254250173295, Tokens per sec: 177191.1769745863, Loss: 2.5097622871398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4837 , TFLOPS: 70.52521123016102, Tokens per sec: 174769.24242170373, Loss: 2.543487071990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4838 , TFLOPS: 68.96635140174071, Tokens per sec: 170906.21604429206, Loss: 2.5321669578552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4839 , TFLOPS: 71.51465640116412, Tokens per sec: 177221.19655183292, Loss: 2.513882875442505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4840 , TFLOPS: 71.538978857667, Tokens per sec: 177281.4702783868, Loss: 2.5492336750030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4841 , TFLOPS: 67.8615862494674, Tokens per sec: 168168.48629703064, Loss: 2.5072567462921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4842 , TFLOPS: 71.52047534052595, Tokens per sec: 177235.61652458063, Loss: 2.5493297576904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4843 , TFLOPS: 71.51830903166852, Tokens per sec: 177230.2481725933, Loss: 2.543095350265503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4844 , TFLOPS: 68.0917678326687, Tokens per sec: 168738.9016167981, Loss: 2.5212464332580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4845 , TFLOPS: 70.38178699664226, Tokens per sec: 174413.82137156068, Loss: 2.5256400108337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4846 , TFLOPS: 71.50097682407497, Tokens per sec: 177187.2970528762, Loss: 2.5492846965789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4847 , TFLOPS: 70.40004312541609, Tokens per sec: 174459.06206974082, Loss: 2.550459384918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4848 , TFLOPS: 69.04175954990708, Tokens per sec: 171093.08574234162, Loss: 2.5407192707061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4849 , TFLOPS: 71.45586467351313, Tokens per sec: 177075.50417427035, Loss: 2.5337791442871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4850 , TFLOPS: 71.5197456291089, Tokens per sec: 177233.80821930483, Loss: 2.542612075805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4851 , TFLOPS: 68.32043735804717, Tokens per sec: 169305.5698907137, Loss: 2.5454533100128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4852 , TFLOPS: 70.46975205965057, Tokens per sec: 174631.8084878469, Loss: 2.5345444679260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4853 , TFLOPS: 71.5290054245785, Tokens per sec: 177256.75501252917, Loss: 2.5546233654022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4854 , TFLOPS: 68.76155415089563, Tokens per sec: 170398.70589641837, Loss: 2.5408740043640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4855 , TFLOPS: 70.60778076256268, Tokens per sec: 174973.85881877996, Loss: 2.531599521636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4856 , TFLOPS: 71.523036626518, Tokens per sec: 177241.9636734179, Loss: 2.5257320404052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4857 , TFLOPS: 70.73394338265564, Tokens per sec: 175286.50368933022, Loss: 2.528101921081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4858 , TFLOPS: 68.76548049431156, Tokens per sec: 170408.43580792547, Loss: 2.535137176513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4859 , TFLOPS: 70.49708436464307, Tokens per sec: 174699.54095052026, Loss: 2.5544381141662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4860 , TFLOPS: 71.4805708451017, Tokens per sec: 177136.72878907627, Loss: 2.5365004539489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4861 , TFLOPS: 68.68979071148699, Tokens per sec: 170220.8681881683, Loss: 2.5517311096191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4862 , TFLOPS: 71.50807039826923, Tokens per sec: 177204.87571115125, Loss: 2.5445544719696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4863 , TFLOPS: 70.52914613706186, Tokens per sec: 174778.9935544698, Loss: 2.5129599571228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4864 , TFLOPS: 68.96027945481765, Tokens per sec: 170891.16909093654, Loss: 2.5247459411621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4865 , TFLOPS: 70.88216967017958, Tokens per sec: 175653.82475828807, Loss: 2.561002254486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4866 , TFLOPS: 71.5001116396888, Tokens per sec: 177185.15303065933, Loss: 2.541141986846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4867 , TFLOPS: 69.43151993211205, Tokens per sec: 172058.95490509603, Loss: 2.51621150970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4868 , TFLOPS: 69.73794953537407, Tokens per sec: 172818.32121798676, Loss: 2.531782627105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4869 , TFLOPS: 71.45820438424214, Tokens per sec: 177081.30223519722, Loss: 2.538346529006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4870 , TFLOPS: 70.54685089347228, Tokens per sec: 174822.8678911948, Loss: 2.5446393489837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4871 , TFLOPS: 68.0935024144828, Tokens per sec: 168743.20010161342, Loss: 2.5298283100128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4872 , TFLOPS: 71.50190225712419, Tokens per sec: 177189.59037791702, Loss: 2.5225791931152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4873 , TFLOPS: 71.51552659363429, Tokens per sec: 177223.35298463362, Loss: 2.533722162246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4874 , TFLOPS: 68.60808923032104, Tokens per sec: 170018.40291767794, Loss: 2.542841911315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4875 , TFLOPS: 71.5142360779294, Tokens per sec: 177220.15494455484, Loss: 2.5224955081939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4876 , TFLOPS: 71.50164274798892, Tokens per sec: 177188.94728569253, Loss: 2.527416229248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4877 , TFLOPS: 70.66740954327092, Tokens per sec: 175121.6255060274, Loss: 2.5415916442871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4878 , TFLOPS: 67.9048022706456, Tokens per sec: 168275.58035814797, Loss: 2.5348339080810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4879 , TFLOPS: 71.51271230280504, Tokens per sec: 177216.378861939, Loss: 2.540708541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4880 , TFLOPS: 71.42804580365751, Tokens per sec: 177006.5659502666, Loss: 2.543433427810669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4881 , TFLOPS: 67.49501025201978, Tokens per sec: 167260.07059367574, Loss: 2.515241861343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4882 , TFLOPS: 71.41458612066276, Tokens per sec: 176973.21137309916, Loss: 2.524506092071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4883 , TFLOPS: 71.5034096859114, Tokens per sec: 177193.32595250872, Loss: 2.5297865867614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4884 , TFLOPS: 69.57564627463825, Tokens per sec: 172416.11585870324, Loss: 2.5197622776031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4885 , TFLOPS: 69.28581737227829, Tokens per sec: 171697.887911366, Loss: 2.5182948112487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4886 , TFLOPS: 71.49005021863587, Tokens per sec: 177160.21972093315, Loss: 2.537416934967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4887 , TFLOPS: 71.49652458163085, Tokens per sec: 177176.26390564468, Loss: 2.534498929977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4888 , TFLOPS: 68.91398221453822, Tokens per sec: 170776.43942946204, Loss: 2.555114269256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4889 , TFLOPS: 70.57713393805255, Tokens per sec: 174897.91261161744, Loss: 2.5317325592041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4890 , TFLOPS: 71.46991339902651, Tokens per sec: 177110.31846368735, Loss: 2.5302202701568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4891 , TFLOPS: 68.07735896880412, Tokens per sec: 168703.1948648743, Loss: 2.547116756439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4892 , TFLOPS: 70.42034746412503, Tokens per sec: 174509.37845776914, Loss: 2.5531163215637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4893 , TFLOPS: 71.4414170412324, Tokens per sec: 177039.7013499406, Loss: 2.5393097400665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4894 , TFLOPS: 68.63262243848476, Tokens per sec: 170079.1989100636, Loss: 2.5424821376800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4895 , TFLOPS: 71.51629665749772, Tokens per sec: 177225.26128771674, Loss: 2.5489423274993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4896 , TFLOPS: 70.58746467428453, Tokens per sec: 174923.51331402484, Loss: 2.5150158405303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4897 , TFLOPS: 70.64759449636618, Tokens per sec: 175072.52163698987, Loss: 2.546487331390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4898 , TFLOPS: 68.89528065807795, Tokens per sec: 170730.09491240702, Loss: 2.513808250427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4899 , TFLOPS: 71.520993436102, Tokens per sec: 177236.90042249102, Loss: 2.5351834297180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4900 , TFLOPS: 70.23272950161271, Tokens per sec: 174044.44047883886, Loss: 2.5442981719970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4901 , TFLOPS: 68.69528100734068, Tokens per sec: 170234.47374609977, Loss: 2.537602186203003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4902 , TFLOPS: 71.50315711878955, Tokens per sec: 177192.7000633582, Loss: 2.554269313812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4903 , TFLOPS: 71.54185932048847, Tokens per sec: 177288.60838256762, Loss: 2.5133512020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4904 , TFLOPS: 69.71673944240484, Tokens per sec: 172765.7602711244, Loss: 2.5446927547454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4905 , TFLOPS: 69.22374316853261, Tokens per sec: 171544.06119644616, Loss: 2.542896270751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4906 , TFLOPS: 71.51040500044306, Tokens per sec: 177210.66111251566, Loss: 2.5384347438812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4907 , TFLOPS: 70.46345491252453, Tokens per sec: 174616.20346358055, Loss: 2.5131232738494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4908 , TFLOPS: 68.688011760574, Tokens per sec: 170216.4597518383, Loss: 2.537719488143921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4909 , TFLOPS: 71.53511672190349, Tokens per sec: 177271.89948051548, Loss: 2.526468276977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4910 , TFLOPS: 71.52312963558609, Tokens per sec: 177242.19416013442, Loss: 2.5444393157958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4911 , TFLOPS: 67.18048398783132, Tokens per sec: 166480.63986308847, Loss: 2.5345516204833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4912 , TFLOPS: 71.49896221949626, Tokens per sec: 177182.30464080314, Loss: 2.5469367504119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4913 , TFLOPS: 71.53756914235223, Tokens per sec: 177277.97684854385, Loss: 2.537783622741699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4914 , TFLOPS: 69.30272723588499, Tokens per sec: 171739.79241615866, Loss: 2.536757230758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4915 , TFLOPS: 70.45545711169915, Tokens per sec: 174596.38403210478, Loss: 2.5312061309814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4916 , TFLOPS: 71.55111447737872, Tokens per sec: 177311.54368088135, Loss: 2.545670509338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4917 , TFLOPS: 70.61717749042073, Tokens per sec: 174997.1449455472, Loss: 2.528752088546753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4918 , TFLOPS: 68.8595222364352, Tokens per sec: 170641.48160446095, Loss: 2.5305278301239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4919 , TFLOPS: 70.51208237049835, Tokens per sec: 174736.7076611964, Loss: 2.5298404693603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4920 , TFLOPS: 71.54455130366759, Tokens per sec: 177295.27941343188, Loss: 2.522029161453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4921 , TFLOPS: 68.6139922582266, Tokens per sec: 170033.03127110013, Loss: 2.53991436958313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4922 , TFLOPS: 70.44155753684792, Tokens per sec: 174561.93935445885, Loss: 2.540931224822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4923 , TFLOPS: 71.50593219129279, Tokens per sec: 177199.5769987207, Loss: 2.524934768676758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4924 , TFLOPS: 70.58039017777426, Tokens per sec: 174905.98193235224, Loss: 2.539771556854248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4925 , TFLOPS: 69.01109427215717, Tokens per sec: 171017.09380601812, Loss: 2.5446419715881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4926 , TFLOPS: 70.57295033631311, Tokens per sec: 174887.54518564534, Loss: 2.5404326915740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4927 , TFLOPS: 71.47604503729634, Tokens per sec: 177125.51333877572, Loss: 2.5376927852630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4928 , TFLOPS: 68.61780031315081, Tokens per sec: 170042.4680506942, Loss: 2.549323081970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4929 , TFLOPS: 70.57434307154608, Tokens per sec: 174890.99653697506, Loss: 2.5509090423583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4930 , TFLOPS: 71.53381147485004, Tokens per sec: 177268.6649345322, Loss: 2.5275063514709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4931 , TFLOPS: 68.80187565873365, Tokens per sec: 170498.62703462175, Loss: 2.55582857131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4932 , TFLOPS: 70.9868231511943, Tokens per sec: 175913.1676127752, Loss: 2.551621437072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4933 , TFLOPS: 70.46371679682326, Tokens per sec: 174616.85244172174, Loss: 2.5100901126861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4934 , TFLOPS: 70.68754950594388, Tokens per sec: 175171.53455213102, Loss: 2.516263723373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4935 , TFLOPS: 69.80410302934364, Tokens per sec: 172982.2568634523, Loss: 2.5372583866119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4936 , TFLOPS: 71.52735206135881, Tokens per sec: 177252.6577963378, Loss: 2.539808750152588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4937 , TFLOPS: 70.5389040172206, Tokens per sec: 174803.1746564224, Loss: 2.5282201766967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4938 , TFLOPS: 68.25716035437402, Tokens per sec: 169148.76250507424, Loss: 2.5256824493408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4939 , TFLOPS: 71.4938063117989, Tokens per sec: 177169.527733559, Loss: 2.5244271755218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4940 , TFLOPS: 71.5132995164045, Tokens per sec: 177217.8340419254, Loss: 2.5359580516815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4941 , TFLOPS: 67.52370419789229, Tokens per sec: 167331.17735244692, Loss: 2.536511182785034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4942 , TFLOPS: 71.55784400822036, Tokens per sec: 177328.22019962594, Loss: 2.5433130264282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4943 , TFLOPS: 71.58108083186805, Tokens per sec: 177385.80360837237, Loss: 2.528770685195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4944 , TFLOPS: 69.72454950177749, Tokens per sec: 172785.11445859887, Loss: 2.5697641372680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4945 , TFLOPS: 68.96973205501372, Tokens per sec: 170914.5936755123, Loss: 2.528447151184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4946 , TFLOPS: 71.54538310600688, Tokens per sec: 177297.34071125925, Loss: 2.5378522872924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4947 , TFLOPS: 71.574880456188, Tokens per sec: 177370.43839440946, Loss: 2.5532360076904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4948 , TFLOPS: 68.5343789690998, Tokens per sec: 169835.74077051613, Loss: 2.5402917861938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4949 , TFLOPS: 71.50273714318944, Tokens per sec: 177191.659317557, Loss: 2.5392932891845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4950 , TFLOPS: 71.5132325462843, Tokens per sec: 177217.6680825902, Loss: 2.532705783843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4951 , TFLOPS: 69.60973659667938, Tokens per sec: 172500.59543208015, Loss: 2.527865409851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4952 , TFLOPS: 69.5291517285739, Tokens per sec: 172300.8972517587, Loss: 2.5171236991882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4953 , TFLOPS: 71.47664363396261, Tokens per sec: 177126.9967272556, Loss: 2.5356855392456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4954 , TFLOPS: 71.46095809542257, Tokens per sec: 177088.12623484863, Loss: 2.529271364212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4955 , TFLOPS: 68.24281729209422, Tokens per sec: 169113.21881672612, Loss: 2.532099485397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4956 , TFLOPS: 71.53503332781939, Tokens per sec: 177271.69282078848, Loss: 2.52091121673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4957 , TFLOPS: 71.5387517811043, Tokens per sec: 177280.90755764992, Loss: 2.54534649848938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4958 , TFLOPS: 67.87471443127725, Tokens per sec: 168201.0193777441, Loss: 2.5233800411224365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4959 , TFLOPS: 70.53511484268144, Tokens per sec: 174793.78466450234, Loss: 2.5165555477142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4960 , TFLOPS: 71.57524563087668, Tokens per sec: 177371.3433375153, Loss: 2.518885374069214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4961 , TFLOPS: 68.89590565400626, Tokens per sec: 170731.64372116633, Loss: 2.5124564170837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4962 , TFLOPS: 71.54763717615569, Tokens per sec: 177302.92654539432, Loss: 2.545038938522339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4963 , TFLOPS: 70.43718049134473, Tokens per sec: 174551.0925535306, Loss: 2.528026819229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4964 , TFLOPS: 70.56591530755054, Tokens per sec: 174870.1116093993, Loss: 2.548330783843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4965 , TFLOPS: 68.90544891932522, Tokens per sec: 170755.29298390422, Loss: 2.538347005844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4966 , TFLOPS: 70.52792853669763, Tokens per sec: 174775.9762066375, Loss: 2.557225465774536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4967 , TFLOPS: 71.50159799020383, Tokens per sec: 177188.83637097062, Loss: 2.544191598892212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4968 , TFLOPS: 68.6632638537127, Tokens per sec: 170155.131712429, Loss: 2.508047580718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4969 , TFLOPS: 71.54803543096257, Tokens per sec: 177303.91346467717, Loss: 2.537835121154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4970 , TFLOPS: 70.47463512253259, Tokens per sec: 174643.90925558232, Loss: 2.528125524520874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4971 , TFLOPS: 70.59439332262882, Tokens per sec: 174940.6832678756, Loss: 2.54292368888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4972 , TFLOPS: 69.06774113531486, Tokens per sec: 171157.47097308998, Loss: 2.524519443511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4973 , TFLOPS: 71.51416545455143, Tokens per sec: 177219.97993204446, Loss: 2.5282599925994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4974 , TFLOPS: 70.49083433634314, Tokens per sec: 174684.0526918956, Loss: 2.527329921722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4975 , TFLOPS: 68.323970677972, Tokens per sec: 169314.32584671475, Loss: 2.544916868209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4976 , TFLOPS: 71.49854847271823, Tokens per sec: 177181.2793307092, Loss: 2.5394351482391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4977 , TFLOPS: 71.50071708980474, Tokens per sec: 177186.65340274246, Loss: 2.5334229469299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4978 , TFLOPS: 67.97617631343257, Tokens per sec: 168452.45309867398, Loss: 2.5337984561920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4979 , TFLOPS: 71.54791905491363, Tokens per sec: 177303.62507200803, Loss: 2.5256059169769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4980 , TFLOPS: 71.49887090507796, Tokens per sec: 177182.07835361548, Loss: 2.5392770767211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4981 , TFLOPS: 68.54707744947933, Tokens per sec: 169867.20900374968, Loss: 2.538410186767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4982 , TFLOPS: 70.88134578718018, Tokens per sec: 175651.78308545734, Loss: 2.5369434356689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4983 , TFLOPS: 71.52729604250227, Tokens per sec: 177252.5189754403, Loss: 2.5511362552642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4984 , TFLOPS: 70.2026071412614, Tokens per sec: 173969.79395163635, Loss: 2.5315439701080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4985 , TFLOPS: 68.63422858748204, Tokens per sec: 170083.1791242108, Loss: 2.5357611179351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4986 , TFLOPS: 71.53993509690127, Tokens per sec: 177283.8399445478, Loss: 2.5076851844787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4987 , TFLOPS: 71.55263253042484, Tokens per sec: 177315.305583557, Loss: 2.5226337909698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4988 , TFLOPS: 68.75530325471897, Tokens per sec: 170383.21548709954, Loss: 2.5255112648010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4989 , TFLOPS: 70.45683878113648, Tokens per sec: 174599.80796117394, Loss: 2.5214366912841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4990 , TFLOPS: 71.51864075267268, Tokens per sec: 177231.07021378368, Loss: 2.5282838344573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4991 , TFLOPS: 70.56843787583715, Tokens per sec: 174876.3628114973, Loss: 2.524928569793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4992 , TFLOPS: 68.77064587440358, Tokens per sec: 170421.23618880712, Loss: 2.551889181137085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4993 , TFLOPS: 70.53687425471595, Tokens per sec: 174798.1446813391, Loss: 2.551029920578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4994 , TFLOPS: 71.54370474825187, Tokens per sec: 177293.18155585596, Loss: 2.5731871128082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4995 , TFLOPS: 68.79410969913334, Tokens per sec: 170479.3821312411, Loss: 2.53611421585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4996 , TFLOPS: 70.48272210244707, Tokens per sec: 174663.9496826648, Loss: 2.5320444107055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4997 , TFLOPS: 71.55936776321103, Tokens per sec: 177331.99623234832, Loss: 2.5249876976013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4998 , TFLOPS: 68.83516765507356, Tokens per sec: 170581.1282689682, Loss: 2.5344905853271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4999 , TFLOPS: 70.93747319896009, Tokens per sec: 175790.8729947928, Loss: 2.5510454177856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5000 , TFLOPS: 70.60520736929377, Tokens per sec: 174967.48166677693, Loss: 2.544072389602661 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/model.pt +[rank0]:[2024-08-26 12:34:27,154] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006379457001457922, 'preprocessing_with_comm': 0.0012739010053337552, 'state_converting': 0.9521405390041764, : 0.9611485350033036}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000, took 4.57s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003000 + eval ppl=9.903579711914062, eval loss=2.292896270751953 +------------------------------------------------------------------ +iteration: 5001 , TFLOPS: 65.05689467507649, Tokens per sec: 161218.15161340754, Loss: 2.5474910736083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5002 , TFLOPS: 70.89090789110725, Tokens per sec: 175675.4790323413, Loss: 2.525272846221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5003 , TFLOPS: 71.52167395520934, Tokens per sec: 177238.58682380436, Loss: 2.53195858001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5004 , TFLOPS: 69.24849931105462, Tokens per sec: 171605.40964472995, Loss: 2.5108988285064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5005 , TFLOPS: 69.66623792014882, Tokens per sec: 172640.61193577203, Loss: 2.5254905223846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5006 , TFLOPS: 71.47575958504069, Tokens per sec: 177124.80595664107, Loss: 2.5434327125549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5007 , TFLOPS: 70.44165355585349, Tokens per sec: 174562.17730013203, Loss: 2.559534788131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5008 , TFLOPS: 68.76680372275742, Tokens per sec: 170411.71491377993, Loss: 2.5295705795288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5009 , TFLOPS: 71.50615887608028, Tokens per sec: 177200.1387485955, Loss: 2.5330593585968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5010 , TFLOPS: 71.53921334999836, Tokens per sec: 177282.05137621542, Loss: 2.5314528942108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5011 , TFLOPS: 69.80308241390553, Tokens per sec: 172979.72766596658, Loss: 2.5487680435180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5012 , TFLOPS: 69.82933243690711, Tokens per sec: 173044.77811464327, Loss: 2.5380561351776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5013 , TFLOPS: 71.47476089613107, Tokens per sec: 177122.33109550836, Loss: 2.528536319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5014 , TFLOPS: 70.49064747386923, Tokens per sec: 174683.58962610064, Loss: 2.529175043106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5015 , TFLOPS: 68.91202418608427, Tokens per sec: 170771.5872192585, Loss: 2.552056074142456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5016 , TFLOPS: 71.52101456626141, Tokens per sec: 177236.9527853537, Loss: 2.5338351726531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5017 , TFLOPS: 71.51114239333648, Tokens per sec: 177212.48845333615, Loss: 2.5394415855407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5018 , TFLOPS: 68.60109818375227, Tokens per sec: 170001.07833415316, Loss: 2.551748752593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5019 , TFLOPS: 71.53393427098423, Tokens per sec: 177268.9692368798, Loss: 2.514044761657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5020 , TFLOPS: 71.54111330896126, Tokens per sec: 177286.7596838232, Loss: 2.539064645767212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5021 , TFLOPS: 68.66069753615257, Tokens per sec: 170148.77209481172, Loss: 2.5169029235839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5022 , TFLOPS: 70.43157870424757, Tokens per sec: 174537.210708016, Loss: 2.5562827587127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5023 , TFLOPS: 71.49524504351933, Tokens per sec: 177173.09306925157, Loss: 2.512421131134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5024 , TFLOPS: 70.74750042132328, Tokens per sec: 175320.09952457974, Loss: 2.5606770515441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5025 , TFLOPS: 69.57694405717898, Tokens per sec: 172419.33190680176, Loss: 2.552781343460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5026 , TFLOPS: 70.60388051043937, Tokens per sec: 174964.19356437013, Loss: 2.534862995147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5027 , TFLOPS: 71.52880336326588, Tokens per sec: 177256.2542823376, Loss: 2.5362842082977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5028 , TFLOPS: 68.69569612196264, Tokens per sec: 170235.5024458616, Loss: 2.5328731536865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5029 , TFLOPS: 70.47705926532544, Tokens per sec: 174649.91654846436, Loss: 2.5352942943573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5030 , TFLOPS: 71.56886242794404, Tokens per sec: 177355.52505748035, Loss: 2.533191442489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5031 , TFLOPS: 70.70885706645825, Tokens per sec: 175224.3370343072, Loss: 2.5131020545959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5032 , TFLOPS: 69.58409300625058, Tokens per sec: 172437.04779012292, Loss: 2.531400442123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5033 , TFLOPS: 70.53320636470566, Tokens per sec: 174789.05524584214, Loss: 2.525707960128784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5034 , TFLOPS: 71.52018426764428, Tokens per sec: 177234.89521391553, Loss: 2.549010992050171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5035 , TFLOPS: 68.78394277736561, Tokens per sec: 170454.18737912175, Loss: 2.53701114654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5036 , TFLOPS: 70.5320520870839, Tokens per sec: 174786.19481874123, Loss: 2.5253052711486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5037 , TFLOPS: 71.55501887155037, Tokens per sec: 177321.2191997432, Loss: 2.5310771465301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5038 , TFLOPS: 68.7607876325394, Tokens per sec: 170396.80637949266, Loss: 2.528555154800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5039 , TFLOPS: 71.55834058404456, Tokens per sec: 177329.45076922065, Loss: 2.552577495574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5040 , TFLOPS: 70.4695732440859, Tokens per sec: 174631.36536317947, Loss: 2.556304693222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5041 , TFLOPS: 69.77724778890277, Tokens per sec: 172915.70661929087, Loss: 2.54858136177063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5042 , TFLOPS: 70.65285055587366, Tokens per sec: 175085.54673145214, Loss: 2.5465831756591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5043 , TFLOPS: 71.52467057112747, Tokens per sec: 177246.012768154, Loss: 2.541881799697876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5044 , TFLOPS: 69.67919018065395, Tokens per sec: 172672.70906411303, Loss: 2.5095415115356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5045 , TFLOPS: 69.23947390909602, Tokens per sec: 171583.04370444643, Loss: 2.52119779586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5046 , TFLOPS: 71.4460491293136, Tokens per sec: 177051.18017447242, Loss: 2.5133607387542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5047 , TFLOPS: 71.51826164452034, Tokens per sec: 177230.13074201994, Loss: 2.5465033054351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5048 , TFLOPS: 67.86726196137572, Tokens per sec: 168182.55133637076, Loss: 2.537862539291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5049 , TFLOPS: 71.38463628720537, Tokens per sec: 176898.9923865452, Loss: 2.5521864891052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5050 , TFLOPS: 71.53281583111848, Tokens per sec: 177266.19761968622, Loss: 2.5431501865386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5051 , TFLOPS: 70.51433451798042, Tokens per sec: 174742.2887307515, Loss: 2.5192489624023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5052 , TFLOPS: 69.44049524213396, Tokens per sec: 172081.19671204235, Loss: 2.5468242168426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5053 , TFLOPS: 71.4270830228594, Tokens per sec: 177004.18007338984, Loss: 2.4959211349487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5054 , TFLOPS: 71.48956392484882, Tokens per sec: 177159.01463135917, Loss: 2.5440125465393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5055 , TFLOPS: 67.80139409108348, Tokens per sec: 168019.3234978417, Loss: 2.538926839828491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5056 , TFLOPS: 71.47708967607407, Tokens per sec: 177128.1020687434, Loss: 2.524052619934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5057 , TFLOPS: 71.52744169934753, Tokens per sec: 177252.87992914827, Loss: 2.5484468936920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5058 , TFLOPS: 69.83108036618509, Tokens per sec: 173049.1096759444, Loss: 2.512967109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5059 , TFLOPS: 69.69736842041539, Tokens per sec: 172717.75674474027, Loss: 2.5468039512634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5060 , TFLOPS: 71.52530575446441, Tokens per sec: 177247.58682243415, Loss: 2.5265817642211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5061 , TFLOPS: 70.58897158887395, Tokens per sec: 174927.24761437773, Loss: 2.5378527641296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5062 , TFLOPS: 69.87809669312477, Tokens per sec: 173165.6213133769, Loss: 2.5408754348754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5063 , TFLOPS: 70.48348096010432, Tokens per sec: 174665.83021553452, Loss: 2.5178050994873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5064 , TFLOPS: 71.51317199022644, Tokens per sec: 177217.51801800798, Loss: 2.5462818145751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5065 , TFLOPS: 69.05142428519035, Tokens per sec: 171117.03602103202, Loss: 2.538687229156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5066 , TFLOPS: 70.5291178845999, Tokens per sec: 174778.9235417568, Loss: 2.538499355316162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5067 , TFLOPS: 71.53028989211778, Tokens per sec: 177259.9380645873, Loss: 2.5104236602783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5068 , TFLOPS: 68.67910857015276, Tokens per sec: 170194.39666520708, Loss: 2.5286002159118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5069 , TFLOPS: 71.50794600072072, Tokens per sec: 177204.56744032254, Loss: 2.54681396484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5070 , TFLOPS: 70.5069233274617, Tokens per sec: 174723.9229842361, Loss: 2.5130507946014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5071 , TFLOPS: 70.60677658638167, Tokens per sec: 174971.37035958425, Loss: 2.543020009994507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5072 , TFLOPS: 69.54608731785005, Tokens per sec: 172342.8655075942, Loss: 2.5275673866271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5073 , TFLOPS: 70.55503292955913, Tokens per sec: 174843.1438779437, Loss: 2.5243546962738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5074 , TFLOPS: 71.46714395780275, Tokens per sec: 177103.45548325573, Loss: 2.543191909790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5075 , TFLOPS: 68.78390730896405, Tokens per sec: 170454.09948451555, Loss: 2.5506887435913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5076 , TFLOPS: 71.49654320633577, Tokens per sec: 177176.31005971515, Loss: 2.5464491844177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5077 , TFLOPS: 70.44791696949467, Tokens per sec: 174577.6987291068, Loss: 2.5270042419433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5078 , TFLOPS: 70.50920063057004, Tokens per sec: 174729.56639220414, Loss: 2.5260276794433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5079 , TFLOPS: 69.90742648835976, Tokens per sec: 173238.3037769129, Loss: 2.5607917308807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5080 , TFLOPS: 71.53348041527146, Tokens per sec: 177267.84453242723, Loss: 2.5311191082000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5081 , TFLOPS: 70.52553164183428, Tokens per sec: 174770.03643712634, Loss: 2.5321366786956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5082 , TFLOPS: 68.96975807471725, Tokens per sec: 170914.6581552039, Loss: 2.5407426357269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5083 , TFLOPS: 71.4952056810928, Tokens per sec: 177172.99552482247, Loss: 2.5493085384368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5084 , TFLOPS: 71.52304190213496, Tokens per sec: 177241.97674697786, Loss: 2.509558916091919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5085 , TFLOPS: 67.59577497384353, Tokens per sec: 167509.77667450547, Loss: 2.536583423614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5086 , TFLOPS: 71.53868766320302, Tokens per sec: 177280.7486664274, Loss: 2.5347259044647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5087 , TFLOPS: 71.47266116216652, Tokens per sec: 177117.1277234394, Loss: 2.5256786346435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5088 , TFLOPS: 69.51800161253205, Tokens per sec: 172273.26603592036, Loss: 2.5389931201934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5089 , TFLOPS: 70.63344013058419, Tokens per sec: 175037.44555935, Loss: 2.5254931449890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5090 , TFLOPS: 71.50300027191716, Tokens per sec: 177192.31137953032, Loss: 2.5472586154937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5091 , TFLOPS: 70.58802959140158, Tokens per sec: 174924.91324087163, Loss: 2.5440139770507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5092 , TFLOPS: 67.9274489228311, Tokens per sec: 168331.70125699835, Loss: 2.5145528316497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5093 , TFLOPS: 71.51037175470148, Tokens per sec: 177210.57872590574, Loss: 2.5291953086853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5094 , TFLOPS: 71.49948938891993, Tokens per sec: 177183.6110247075, Loss: 2.50388765335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5095 , TFLOPS: 68.6759441605759, Tokens per sec: 170186.55490968708, Loss: 2.5311667919158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5096 , TFLOPS: 69.69251584946899, Tokens per sec: 172705.7315393803, Loss: 2.538735866546631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5097 , TFLOPS: 71.51249644324315, Tokens per sec: 177215.84393816578, Loss: 2.528209924697876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5098 , TFLOPS: 70.58689482533768, Tokens per sec: 174922.10116556048, Loss: 2.5146849155426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5099 , TFLOPS: 69.56866778389417, Tokens per sec: 172398.8223898952, Loss: 2.5272116661071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5100 , TFLOPS: 70.49432980517872, Tokens per sec: 174692.71484872783, Loss: 2.5294594764709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5101 , TFLOPS: 71.4242095979223, Tokens per sec: 176997.05940986183, Loss: 2.523036479949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5102 , TFLOPS: 68.62977350405284, Tokens per sec: 170072.13893670528, Loss: 2.5290462970733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5103 , TFLOPS: 69.48763947211283, Tokens per sec: 172198.02530729567, Loss: 2.532972574234009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5104 , TFLOPS: 71.54585309470059, Tokens per sec: 177298.5053950158, Loss: 2.540182590484619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5105 , TFLOPS: 68.71987657855846, Tokens per sec: 170295.42428100324, Loss: 2.5197973251342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5106 , TFLOPS: 71.5572449986738, Tokens per sec: 177326.7357879832, Loss: 2.5257081985473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5107 , TFLOPS: 70.540487083731, Tokens per sec: 174807.09766962432, Loss: 2.522963523864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5108 , TFLOPS: 70.70054408201639, Tokens per sec: 175203.73654310947, Loss: 2.5248467922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5109 , TFLOPS: 69.73054309410527, Tokens per sec: 172799.96724063583, Loss: 2.511913776397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5110 , TFLOPS: 70.40129971432022, Tokens per sec: 174462.17603546995, Loss: 2.5305089950561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5111 , TFLOPS: 71.43831796101436, Tokens per sec: 177032.02148776842, Loss: 2.5354268550872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5112 , TFLOPS: 68.87014092421246, Tokens per sec: 170667.79588251805, Loss: 2.530961513519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5113 , TFLOPS: 71.47057807874738, Tokens per sec: 177111.96561325618, Loss: 2.527801036834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5114 , TFLOPS: 70.52936024447821, Tokens per sec: 174779.5241362334, Loss: 2.5215795040130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5115 , TFLOPS: 68.75786929139919, Tokens per sec: 170389.5744086654, Loss: 2.548288106918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5116 , TFLOPS: 71.52131374242398, Tokens per sec: 177237.69417684156, Loss: 2.545966148376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5117 , TFLOPS: 71.54545116116107, Tokens per sec: 177297.50935942822, Loss: 2.5242702960968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5118 , TFLOPS: 69.67633536540085, Tokens per sec: 172665.63451743196, Loss: 2.5321245193481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5119 , TFLOPS: 69.51856671168545, Tokens per sec: 172274.66641387326, Loss: 2.527806520462036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5120 , TFLOPS: 71.55033223400176, Tokens per sec: 177309.60519562202, Loss: 2.5407931804656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5121 , TFLOPS: 71.5191351613712, Tokens per sec: 177232.2954130022, Loss: 2.531981945037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5122 , TFLOPS: 68.46316529297167, Tokens per sec: 169659.26543623654, Loss: 2.5363447666168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5123 , TFLOPS: 71.52989839676684, Tokens per sec: 177258.9678959809, Loss: 2.5441908836364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5124 , TFLOPS: 71.50972480195006, Tokens per sec: 177208.97550571995, Loss: 2.5422616004943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5125 , TFLOPS: 68.50793753872172, Tokens per sec: 169770.21599327523, Loss: 2.545341730117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5126 , TFLOPS: 70.58809999737079, Tokens per sec: 174925.08771461918, Loss: 2.5174779891967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5127 , TFLOPS: 71.45277463870839, Tokens per sec: 177067.84672762876, Loss: 2.541116237640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5128 , TFLOPS: 70.73516242705716, Tokens per sec: 175289.52461564602, Loss: 2.542757749557495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5129 , TFLOPS: 68.42999171212988, Tokens per sec: 169577.057648539, Loss: 2.548335313796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5130 , TFLOPS: 71.49022968451435, Tokens per sec: 177160.66445714983, Loss: 2.537907838821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5131 , TFLOPS: 71.52647774467674, Tokens per sec: 177250.4911432857, Loss: 2.5205488204956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5132 , TFLOPS: 68.63354721437612, Tokens per sec: 170081.4906065949, Loss: 2.5341689586639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5133 , TFLOPS: 70.54098334327652, Tokens per sec: 174808.32745544566, Loss: 2.5416510105133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5134 , TFLOPS: 71.4981167715462, Tokens per sec: 177180.20952764907, Loss: 2.52557373046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5135 , TFLOPS: 68.7355718604587, Tokens per sec: 170334.31891853007, Loss: 2.529219627380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5136 , TFLOPS: 71.53033563292396, Tokens per sec: 177260.05141534377, Loss: 2.537520408630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5137 , TFLOPS: 70.44978553982912, Tokens per sec: 174582.32925223562, Loss: 2.5154497623443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5138 , TFLOPS: 70.56743126293358, Tokens per sec: 174873.86831383474, Loss: 2.5535738468170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5139 , TFLOPS: 69.52404541880071, Tokens per sec: 172288.24325363434, Loss: 2.5304348468780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5140 , TFLOPS: 70.5483983683705, Tokens per sec: 174826.702704461, Loss: 2.538128137588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5141 , TFLOPS: 71.56341192605399, Tokens per sec: 177342.01811337445, Loss: 2.5292367935180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5142 , TFLOPS: 68.7041329897572, Tokens per sec: 170256.40993365424, Loss: 2.513704299926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5143 , TFLOPS: 71.51738533778565, Tokens per sec: 177227.95915739826, Loss: 2.5253868103027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5144 , TFLOPS: 70.4692249204465, Tokens per sec: 174630.50217883004, Loss: 2.524181365966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5145 , TFLOPS: 70.54638278009045, Tokens per sec: 174821.70785466765, Loss: 2.5250625610351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5146 , TFLOPS: 69.82062381619313, Tokens per sec: 173023.19719318024, Loss: 2.5188660621643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5147 , TFLOPS: 70.54477194528796, Tokens per sec: 174817.71602857896, Loss: 2.5363359451293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5148 , TFLOPS: 71.52505224021426, Tokens per sec: 177246.9585861954, Loss: 2.550407648086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5149 , TFLOPS: 68.82877354141574, Tokens per sec: 170565.28295095437, Loss: 2.5344297885894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5150 , TFLOPS: 71.49360539535482, Tokens per sec: 177169.02984047774, Loss: 2.5295252799987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5151 , TFLOPS: 70.43012125373886, Tokens per sec: 174533.598985104, Loss: 2.522460699081421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5152 , TFLOPS: 68.63261346125167, Tokens per sec: 170079.17666349106, Loss: 2.5243327617645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5153 , TFLOPS: 71.4895624961703, Tokens per sec: 177159.01109093637, Loss: 2.545220375061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5154 , TFLOPS: 71.49731950428692, Tokens per sec: 177178.23381155438, Loss: 2.5316708087921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5155 , TFLOPS: 67.78784623714299, Tokens per sec: 167985.75042335785, Loss: 2.5186736583709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5156 , TFLOPS: 71.55408862024332, Tokens per sec: 177318.91393452868, Loss: 2.5370984077453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5157 , TFLOPS: 71.49564861438373, Tokens per sec: 177174.09316231057, Loss: 2.555297374725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5158 , TFLOPS: 70.66908373195156, Tokens per sec: 175125.77433000517, Loss: 2.5154995918273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5159 , TFLOPS: 68.68846705881496, Tokens per sec: 170217.58803103463, Loss: 2.5324454307556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5160 , TFLOPS: 71.53623289453738, Tokens per sec: 177274.6654792588, Loss: 2.5363216400146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5161 , TFLOPS: 71.53251495442188, Tokens per sec: 177265.4520140877, Loss: 2.5218443870544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5162 , TFLOPS: 67.78303033731795, Tokens per sec: 167973.81609307625, Loss: 2.5221283435821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5163 , TFLOPS: 71.52169499881032, Tokens per sec: 177238.63897216573, Loss: 2.514427423477173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5164 , TFLOPS: 71.5302558354891, Tokens per sec: 177259.85366850978, Loss: 2.5178918838500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5165 , TFLOPS: 70.43497575539102, Tokens per sec: 174545.6289749656, Loss: 2.517195224761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5166 , TFLOPS: 68.43285476841987, Tokens per sec: 169584.1526174173, Loss: 2.518214225769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5167 , TFLOPS: 71.53359999186864, Tokens per sec: 177268.14085640752, Loss: 2.558211326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5168 , TFLOPS: 71.4642870163394, Tokens per sec: 177096.3756675915, Loss: 2.5302231311798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5169 , TFLOPS: 68.79380130476402, Tokens per sec: 170478.6178960215, Loss: 2.5197458267211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5170 , TFLOPS: 70.44889088109214, Tokens per sec: 174580.11218933057, Loss: 2.5431697368621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5171 , TFLOPS: 71.49285538895954, Tokens per sec: 177167.1712420102, Loss: 2.530383586883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5172 , TFLOPS: 68.5638807786914, Tokens per sec: 169908.8495045767, Loss: 2.525700330734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5173 , TFLOPS: 71.5653944005561, Tokens per sec: 177346.9309035783, Loss: 2.5129873752593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5174 , TFLOPS: 70.5971545404019, Tokens per sec: 174947.52586969055, Loss: 2.534775495529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5175 , TFLOPS: 69.72236997035873, Tokens per sec: 172779.71333964798, Loss: 2.528583526611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5176 , TFLOPS: 70.46306257271975, Tokens per sec: 174615.23120232453, Loss: 2.533189535140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5177 , TFLOPS: 70.4152492143186, Tokens per sec: 174496.74443313025, Loss: 2.528550624847412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5178 , TFLOPS: 70.67217756953329, Tokens per sec: 175133.44120034704, Loss: 2.545093297958374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5179 , TFLOPS: 69.5181633631744, Tokens per sec: 172273.6668718304, Loss: 2.554408073425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5180 , TFLOPS: 71.50674697687441, Tokens per sec: 177201.59612714802, Loss: 2.5329909324645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5181 , TFLOPS: 70.48395608373232, Tokens per sec: 174667.0076242239, Loss: 2.532231330871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5182 , TFLOPS: 68.66319039614793, Tokens per sec: 170154.94967649147, Loss: 2.5450894832611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5183 , TFLOPS: 71.5216199549656, Tokens per sec: 177238.45300525177, Loss: 2.523566246032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5184 , TFLOPS: 70.30098722100311, Tokens per sec: 174213.5906266968, Loss: 2.525191068649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5185 , TFLOPS: 70.75899809319631, Tokens per sec: 175348.59202205422, Loss: 2.547699213027954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5186 , TFLOPS: 69.57660069824423, Tokens per sec: 172418.48102553733, Loss: 2.5408360958099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5187 , TFLOPS: 71.43833532150752, Tokens per sec: 177032.0645089829, Loss: 2.5276544094085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5188 , TFLOPS: 70.58658300313088, Tokens per sec: 174921.328435782, Loss: 2.5293920040130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5189 , TFLOPS: 68.56512142245971, Tokens per sec: 169911.92395650275, Loss: 2.5021910667419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5190 , TFLOPS: 71.50560816930434, Tokens per sec: 177198.77403653934, Loss: 2.5400333404541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5191 , TFLOPS: 71.532660632453, Tokens per sec: 177265.81302029657, Loss: 2.5537075996398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5192 , TFLOPS: 68.95648164825111, Tokens per sec: 170881.7577079017, Loss: 2.5367584228515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5193 , TFLOPS: 70.61422635909528, Tokens per sec: 174989.8317170288, Loss: 2.5059831142425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5194 , TFLOPS: 71.47908849543944, Tokens per sec: 177133.0553633189, Loss: 2.5145862102508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5195 , TFLOPS: 70.51622406392421, Tokens per sec: 174746.97123375087, Loss: 2.5191187858581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5196 , TFLOPS: 68.6696868373479, Tokens per sec: 170171.04857342708, Loss: 2.5302579402923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5197 , TFLOPS: 71.51368033958882, Tokens per sec: 177218.77776372727, Loss: 2.5476861000061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5198 , TFLOPS: 71.53858773118388, Tokens per sec: 177280.50102387552, Loss: 2.525308847427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5199 , TFLOPS: 68.45699394966535, Tokens per sec: 169643.97216769392, Loss: 2.5172934532165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5200 , TFLOPS: 71.52215448290926, Tokens per sec: 177239.77762437926, Loss: 2.549264430999756 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/optimizer.pt +[rank0]:[2024-08-26 12:59:38,472] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006491151994850952, 'preprocessing_with_comm': 0.0012689759969362058, 'state_converting': 0.9849812819957151, : 0.9941097240007366}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200, took 4.62s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003200 + eval ppl=10.763229370117188, eval loss=2.3761355876922607 +------------------------------------------------------------------ +iteration: 5201 , TFLOPS: 67.23244602232802, Tokens per sec: 166609.4075086619, Loss: 2.526507616043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5202 , TFLOPS: 68.60372234613203, Tokens per sec: 170007.5812976056, Loss: 2.5217154026031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5203 , TFLOPS: 69.40846584804584, Tokens per sec: 172001.82434516295, Loss: 2.540635108947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5204 , TFLOPS: 71.54231708188001, Tokens per sec: 177289.74276572242, Loss: 2.539381265640259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5205 , TFLOPS: 70.60697920653227, Tokens per sec: 174971.87247463793, Loss: 2.5369675159454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5206 , TFLOPS: 69.8041810558938, Tokens per sec: 172982.45022183895, Loss: 2.5560646057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5207 , TFLOPS: 70.4808214220866, Tokens per sec: 174659.23958735433, Loss: 2.5531575679779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5208 , TFLOPS: 71.50597125648991, Tokens per sec: 177199.67380658255, Loss: 2.5423812866210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5209 , TFLOPS: 69.17503765870937, Tokens per sec: 171423.36357774935, Loss: 2.533658742904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5210 , TFLOPS: 70.34785395666282, Tokens per sec: 174329.73156045866, Loss: 2.5289151668548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5211 , TFLOPS: 71.53959201325324, Tokens per sec: 177282.9897454742, Loss: 2.521048069000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5212 , TFLOPS: 69.73727937005482, Tokens per sec: 172816.66047450085, Loss: 2.556281328201294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5213 , TFLOPS: 70.59593513670558, Tokens per sec: 174944.50405300266, Loss: 2.5211691856384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5214 , TFLOPS: 70.49887920468652, Tokens per sec: 174703.98876186585, Loss: 2.535400629043579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5215 , TFLOPS: 69.73454101743316, Tokens per sec: 172809.87453504992, Loss: 2.5259814262390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5216 , TFLOPS: 70.40576644378278, Tokens per sec: 174473.24508312903, Loss: 2.5296871662139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5217 , TFLOPS: 71.53030765497023, Tokens per sec: 177259.9820828924, Loss: 2.5172243118286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5218 , TFLOPS: 69.63622507014267, Tokens per sec: 172566.23678726843, Loss: 2.4949254989624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5219 , TFLOPS: 69.43412943494477, Tokens per sec: 172065.42154057598, Loss: 2.522517442703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5220 , TFLOPS: 71.49070836105653, Tokens per sec: 177161.85067035194, Loss: 2.5376763343811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5221 , TFLOPS: 70.4931196108302, Tokens per sec: 174689.71585381828, Loss: 2.529139995574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5222 , TFLOPS: 68.72655392762834, Tokens per sec: 170311.97148756913, Loss: 2.5377628803253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5223 , TFLOPS: 71.55168570875827, Tokens per sec: 177312.9592551662, Loss: 2.51261043548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5224 , TFLOPS: 71.43627805601444, Tokens per sec: 177026.96637846454, Loss: 2.5210719108581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5225 , TFLOPS: 69.69468172550685, Tokens per sec: 172711.09881879305, Loss: 2.5173609256744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5226 , TFLOPS: 69.38799401273862, Tokens per sec: 171951.09288211286, Loss: 2.527007818222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5227 , TFLOPS: 71.50544615459798, Tokens per sec: 177198.37254624962, Loss: 2.5296239852905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5228 , TFLOPS: 71.47032104483718, Tokens per sec: 177111.32865491207, Loss: 2.5320327281951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5229 , TFLOPS: 67.78956591586511, Tokens per sec: 167990.0119766695, Loss: 2.546898126602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5230 , TFLOPS: 71.4486104194946, Tokens per sec: 177057.52733369035, Loss: 2.5148732662200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5231 , TFLOPS: 71.51995658569719, Tokens per sec: 177234.3309929694, Loss: 2.534250497817993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5232 , TFLOPS: 69.80617590085217, Tokens per sec: 172987.3936673961, Loss: 2.528294324874878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5233 , TFLOPS: 69.49559169196898, Tokens per sec: 172217.73178410853, Loss: 2.537842273712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5234 , TFLOPS: 71.57683277455446, Tokens per sec: 177375.27645438717, Loss: 2.5209317207336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5235 , TFLOPS: 69.77791633154489, Tokens per sec: 172917.3633416041, Loss: 2.5402417182922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5236 , TFLOPS: 69.27524081891679, Tokens per sec: 171671.67804703268, Loss: 2.521350383758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5237 , TFLOPS: 71.52923670109628, Tokens per sec: 177257.32814121747, Loss: 2.5555644035339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5238 , TFLOPS: 70.6164494706578, Tokens per sec: 174995.34083237665, Loss: 2.550273895263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5239 , TFLOPS: 69.47324137742775, Tokens per sec: 172162.34524258497, Loss: 2.5370125770568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5240 , TFLOPS: 70.5173169417246, Tokens per sec: 174749.67950533066, Loss: 2.5202555656433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5241 , TFLOPS: 71.5289909674771, Tokens per sec: 177256.71918623932, Loss: 2.519810676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5242 , TFLOPS: 69.86920939086947, Tokens per sec: 173143.5975993711, Loss: 2.5392136573791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5243 , TFLOPS: 70.69691743578494, Tokens per sec: 175194.74931423986, Loss: 2.5273454189300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5244 , TFLOPS: 70.42942579244637, Tokens per sec: 174531.87555540982, Loss: 2.5367090702056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5245 , TFLOPS: 70.72354594107664, Tokens per sec: 175260.7376837247, Loss: 2.5456788539886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5246 , TFLOPS: 69.49363206296678, Tokens per sec: 172212.87560757014, Loss: 2.520498275756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5247 , TFLOPS: 71.55059120710082, Tokens per sec: 177310.24695948974, Loss: 2.551060676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5248 , TFLOPS: 70.4844440694526, Tokens per sec: 174668.21690659484, Loss: 2.540088176727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5249 , TFLOPS: 68.78662837610231, Tokens per sec: 170460.84258863446, Loss: 2.523066759109497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5250 , TFLOPS: 71.5040773838007, Tokens per sec: 177194.9805814314, Loss: 2.5484259128570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5251 , TFLOPS: 70.43794158304648, Tokens per sec: 174552.9786226087, Loss: 2.523273468017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5252 , TFLOPS: 69.66260994904667, Tokens per sec: 172631.6214237269, Loss: 2.5383458137512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5253 , TFLOPS: 70.34520797109408, Tokens per sec: 174323.17451674, Loss: 2.5235788822174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5254 , TFLOPS: 71.5684779596502, Tokens per sec: 177354.5723026955, Loss: 2.522444009780884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5255 , TFLOPS: 69.68410309906575, Tokens per sec: 172684.8838171401, Loss: 2.5387086868286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5256 , TFLOPS: 69.91040863087504, Tokens per sec: 173245.69385457586, Loss: 2.5371499061584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5257 , TFLOPS: 71.53620398406161, Tokens per sec: 177274.59383591512, Loss: 2.528409719467163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5258 , TFLOPS: 70.47730084542737, Tokens per sec: 174650.51521056917, Loss: 2.5026979446411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5259 , TFLOPS: 69.01362598242852, Tokens per sec: 171023.36766296142, Loss: 2.5300705432891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5260 , TFLOPS: 71.54081569749549, Tokens per sec: 177286.02216982644, Loss: 2.5431807041168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5261 , TFLOPS: 71.48613329760862, Tokens per sec: 177150.51315914316, Loss: 2.5302796363830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5262 , TFLOPS: 68.74759914329249, Tokens per sec: 170364.12385030682, Loss: 2.5359437465667725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5263 , TFLOPS: 70.40094159908793, Tokens per sec: 174461.28858647466, Loss: 2.533304214477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5264 , TFLOPS: 71.55151352513764, Tokens per sec: 177312.53256518667, Loss: 2.5417449474334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5265 , TFLOPS: 70.68273781851111, Tokens per sec: 175159.61066062152, Loss: 2.5188252925872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5266 , TFLOPS: 68.65076096735963, Tokens per sec: 170124.14818273945, Loss: 2.5234227180480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5267 , TFLOPS: 71.4486421229121, Tokens per sec: 177057.60589825135, Loss: 2.5593719482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5268 , TFLOPS: 71.52653514410584, Tokens per sec: 177250.63338539403, Loss: 2.549522876739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5269 , TFLOPS: 68.15318972275733, Tokens per sec: 168891.11182661753, Loss: 2.5345661640167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5270 , TFLOPS: 70.49481914542916, Tokens per sec: 174693.9274877737, Loss: 2.5216944217681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5271 , TFLOPS: 71.517976125381, Tokens per sec: 177229.42319414014, Loss: 2.515185832977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5272 , TFLOPS: 70.67176384594227, Tokens per sec: 175132.41594771305, Loss: 2.538506507873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5273 , TFLOPS: 68.87882624549432, Tokens per sec: 170689.3190654216, Loss: 2.5280520915985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5274 , TFLOPS: 71.4574311151783, Tokens per sec: 177079.3859892744, Loss: 2.5385372638702393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5275 , TFLOPS: 71.5098141833886, Tokens per sec: 177209.19700277087, Loss: 2.5495500564575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5276 , TFLOPS: 69.00489575213349, Tokens per sec: 171001.73319057608, Loss: 2.5069386959075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5277 , TFLOPS: 70.44261647839375, Tokens per sec: 174564.56352826147, Loss: 2.523113965988159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5278 , TFLOPS: 71.52559552581515, Tokens per sec: 177248.3049077622, Loss: 2.546703577041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5279 , TFLOPS: 69.86141978793712, Tokens per sec: 173124.29410520915, Loss: 2.5100224018096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5280 , TFLOPS: 70.66770898751321, Tokens per sec: 175122.3675618463, Loss: 2.5367748737335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5281 , TFLOPS: 70.51696497121829, Tokens per sec: 174748.80728364325, Loss: 2.5532326698303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5282 , TFLOPS: 69.82897649747126, Tokens per sec: 173043.89605751124, Loss: 2.5078160762786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5283 , TFLOPS: 70.39372905026292, Tokens per sec: 174443.41509595845, Loss: 2.5433199405670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5284 , TFLOPS: 70.49581727661871, Tokens per sec: 174696.4009668146, Loss: 2.5347256660461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5285 , TFLOPS: 70.59603855243125, Tokens per sec: 174944.76032856308, Loss: 2.544527053833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5286 , TFLOPS: 69.69789243408408, Tokens per sec: 172719.0553083362, Loss: 2.54115891456604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5287 , TFLOPS: 71.58700380021766, Tokens per sec: 177400.4813764115, Loss: 2.514631509780884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5288 , TFLOPS: 70.55386623170186, Tokens per sec: 174840.25267213097, Loss: 2.546948194503784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5289 , TFLOPS: 68.71568564222018, Tokens per sec: 170285.03867908727, Loss: 2.5243136882781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5290 , TFLOPS: 71.50732630813636, Tokens per sec: 177203.03177383373, Loss: 2.5252268314361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5291 , TFLOPS: 71.53448076513568, Tokens per sec: 177270.32350958843, Loss: 2.5261874198913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5292 , TFLOPS: 69.47970458173862, Tokens per sec: 172178.361774848, Loss: 2.5519070625305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5293 , TFLOPS: 69.57045039016967, Tokens per sec: 172403.2398846162, Loss: 2.5160012245178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5294 , TFLOPS: 71.51976681645472, Tokens per sec: 177233.86072388175, Loss: 2.509502410888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5295 , TFLOPS: 70.47191739017698, Tokens per sec: 174637.17441542217, Loss: 2.541489839553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5296 , TFLOPS: 68.6728529862429, Tokens per sec: 170178.89463916898, Loss: 2.5317537784576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5297 , TFLOPS: 71.5153654131304, Tokens per sec: 177222.95356158947, Loss: 2.5245444774627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5298 , TFLOPS: 71.55549866752324, Tokens per sec: 177322.40818701786, Loss: 2.5366435050964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5299 , TFLOPS: 68.75498289532396, Tokens per sec: 170382.4216012282, Loss: 2.520446300506592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5300 , TFLOPS: 69.92568372186106, Tokens per sec: 173283.5472127294, Loss: 2.523364543914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5301 , TFLOPS: 71.47082238851425, Tokens per sec: 177112.57103977146, Loss: 2.5364298820495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5302 , TFLOPS: 69.73331853300954, Tokens per sec: 172806.84508398044, Loss: 2.511688470840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5303 , TFLOPS: 69.44176502611352, Tokens per sec: 172084.34337662195, Loss: 2.5494790077209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5304 , TFLOPS: 71.52703158500849, Tokens per sec: 177251.86362063832, Loss: 2.5334980487823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5305 , TFLOPS: 70.59236648355996, Tokens per sec: 174935.6605373882, Loss: 2.547788619995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5306 , TFLOPS: 69.56623896328627, Tokens per sec: 172392.80350487216, Loss: 2.5328290462493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5307 , TFLOPS: 70.5284368429653, Tokens per sec: 174777.23584556335, Loss: 2.5140442848205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5308 , TFLOPS: 71.55019657895461, Tokens per sec: 177309.26902747154, Loss: 2.564870834350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5309 , TFLOPS: 68.82748590099071, Tokens per sec: 170562.09203613584, Loss: 2.523729085922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5310 , TFLOPS: 70.4004025795234, Tokens per sec: 174459.9528366159, Loss: 2.525423049926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5311 , TFLOPS: 71.48831407714957, Tokens per sec: 177155.9173710783, Loss: 2.523468494415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5312 , TFLOPS: 70.55705668502173, Tokens per sec: 174848.15896691507, Loss: 2.519198417663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5313 , TFLOPS: 69.53399203232966, Tokens per sec: 172312.89205766897, Loss: 2.535961389541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5314 , TFLOPS: 70.42127921200635, Tokens per sec: 174511.68743165955, Loss: 2.506314516067505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5315 , TFLOPS: 71.4910386965528, Tokens per sec: 177162.66927810118, Loss: 2.5341577529907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5316 , TFLOPS: 68.78350568465716, Tokens per sec: 170453.1042152405, Loss: 2.5354301929473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5317 , TFLOPS: 71.50034173490202, Tokens per sec: 177185.72323194478, Loss: 2.512385606765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5318 , TFLOPS: 70.44872258617747, Tokens per sec: 174579.69513599284, Loss: 2.513688325881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5319 , TFLOPS: 69.76426346403905, Tokens per sec: 172883.53003193712, Loss: 2.5147018432617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5320 , TFLOPS: 70.44694918705213, Tokens per sec: 174575.30045760417, Loss: 2.530317783355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5321 , TFLOPS: 71.56853020225148, Tokens per sec: 177354.7017656165, Loss: 2.541565179824829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5322 , TFLOPS: 69.68110176854982, Tokens per sec: 172677.44618949777, Loss: 2.5291454792022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5323 , TFLOPS: 69.74126712432677, Tokens per sec: 172826.54256887414, Loss: 2.5292809009552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5324 , TFLOPS: 71.52559586399285, Tokens per sec: 177248.30574580384, Loss: 2.525193214416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5325 , TFLOPS: 70.43360881106528, Tokens per sec: 174542.24153634504, Loss: 2.527587890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5326 , TFLOPS: 68.97592338734665, Tokens per sec: 170929.93647906484, Loss: 2.5225377082824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5327 , TFLOPS: 71.55009510635986, Tokens per sec: 177309.0175672036, Loss: 2.532870292663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5328 , TFLOPS: 71.47543465925399, Tokens per sec: 177124.0007547481, Loss: 2.5437119007110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5329 , TFLOPS: 68.87399157696898, Tokens per sec: 170677.3382242331, Loss: 2.515882730484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5330 , TFLOPS: 70.71687475892125, Tokens per sec: 175244.2057594523, Loss: 2.5328824520111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5331 , TFLOPS: 71.56183519529759, Tokens per sec: 177338.1108008689, Loss: 2.512769937515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5332 , TFLOPS: 70.5346165513509, Tokens per sec: 174792.5498436939, Loss: 2.531402587890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5333 , TFLOPS: 69.58549373860191, Tokens per sec: 172440.5189591929, Loss: 2.524766445159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5334 , TFLOPS: 71.5011896977779, Tokens per sec: 177187.82457736201, Loss: 2.529752254486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5335 , TFLOPS: 71.50062779883481, Tokens per sec: 177186.4321298828, Loss: 2.5429842472076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5336 , TFLOPS: 67.73871068491586, Tokens per sec: 167863.98711221106, Loss: 2.51246976852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5337 , TFLOPS: 71.55899656052037, Tokens per sec: 177331.0763511895, Loss: 2.528562545776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5338 , TFLOPS: 71.52177301873931, Tokens per sec: 177238.83231414438, Loss: 2.5178840160369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5339 , TFLOPS: 70.5739837582573, Tokens per sec: 174890.10611906383, Loss: 2.5456225872039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5340 , TFLOPS: 68.88865396177295, Tokens per sec: 170713.67322897236, Loss: 2.5344736576080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5341 , TFLOPS: 71.54827003107158, Tokens per sec: 177304.4948295906, Loss: 2.541450023651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5342 , TFLOPS: 71.52989570051238, Tokens per sec: 177258.96121436535, Loss: 2.5411033630371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5343 , TFLOPS: 68.95607060570362, Tokens per sec: 170880.73909918894, Loss: 2.528296947479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5344 , TFLOPS: 70.4616149876736, Tokens per sec: 174611.64392711548, Loss: 2.5555577278137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5345 , TFLOPS: 71.57069332376616, Tokens per sec: 177360.06221901742, Loss: 2.516223669052124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5346 , TFLOPS: 69.80962401057891, Tokens per sec: 172995.9384631396, Loss: 2.538909673690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5347 , TFLOPS: 69.49092899168669, Tokens per sec: 172206.17709917016, Loss: 2.5191900730133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5348 , TFLOPS: 71.54386897632608, Tokens per sec: 177293.58853111506, Loss: 2.5561013221740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5349 , TFLOPS: 69.42618292561122, Tokens per sec: 172045.7292150672, Loss: 2.532383441925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5350 , TFLOPS: 70.48832330980228, Tokens per sec: 174677.83009151492, Loss: 2.5079727172851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5351 , TFLOPS: 70.44140010181982, Tokens per sec: 174561.54921311632, Loss: 2.5496044158935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5352 , TFLOPS: 70.6349611373248, Tokens per sec: 175041.2147816059, Loss: 2.513521194458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5353 , TFLOPS: 69.46676533886252, Tokens per sec: 172146.2969056258, Loss: 2.5397820472717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5354 , TFLOPS: 71.56694759621509, Tokens per sec: 177350.7798935199, Loss: 2.542440891265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5355 , TFLOPS: 70.44322244698735, Tokens per sec: 174566.06518518933, Loss: 2.5235555171966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5356 , TFLOPS: 68.74779816967735, Tokens per sec: 170364.61705961305, Loss: 2.534370183944702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5357 , TFLOPS: 71.47493373340521, Tokens per sec: 177122.75940531326, Loss: 2.496908664703369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5358 , TFLOPS: 70.43617877214808, Tokens per sec: 174548.61018301282, Loss: 2.5306150913238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5359 , TFLOPS: 70.67934728518225, Tokens per sec: 175151.20854553452, Loss: 2.541745185852051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5360 , TFLOPS: 69.62513688502192, Tokens per sec: 172538.7590430748, Loss: 2.533384323120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5361 , TFLOPS: 71.50199084642074, Tokens per sec: 177189.80991195276, Loss: 2.5222342014312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5362 , TFLOPS: 70.47778045467612, Tokens per sec: 174651.70373512094, Loss: 2.5167880058288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5363 , TFLOPS: 68.75607572251224, Tokens per sec: 170385.1297473856, Loss: 2.5225093364715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5364 , TFLOPS: 71.50428440438424, Tokens per sec: 177195.49360124258, Loss: 2.514139175415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5365 , TFLOPS: 71.54581480777841, Tokens per sec: 177298.410515805, Loss: 2.528768539428711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5366 , TFLOPS: 68.65008296005749, Tokens per sec: 170122.4680059561, Loss: 2.526158094406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5367 , TFLOPS: 70.42844316258072, Tokens per sec: 174529.44049035714, Loss: 2.5223047733306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5368 , TFLOPS: 71.53643634131339, Tokens per sec: 177275.1696427813, Loss: 2.482588529586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5369 , TFLOPS: 70.52180780860408, Tokens per sec: 174760.80836816787, Loss: 2.5149784088134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5370 , TFLOPS: 69.80940220170953, Tokens per sec: 172995.38879632702, Loss: 2.5142269134521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5371 , TFLOPS: 71.52698436995026, Tokens per sec: 177251.74661652284, Loss: 2.522585153579712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5372 , TFLOPS: 71.4564422161568, Tokens per sec: 177076.93538856294, Loss: 2.5216002464294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5373 , TFLOPS: 67.93090946944206, Tokens per sec: 168340.27687271097, Loss: 2.5241527557373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5374 , TFLOPS: 71.49830249606984, Tokens per sec: 177180.66977347792, Loss: 2.521347999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5375 , TFLOPS: 71.53202120146294, Tokens per sec: 177264.2284398641, Loss: 2.516646385192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5376 , TFLOPS: 68.78081431608004, Tokens per sec: 170446.43470742766, Loss: 2.5325546264648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5377 , TFLOPS: 70.53379963793932, Tokens per sec: 174790.52544226963, Loss: 2.5309534072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5378 , TFLOPS: 71.47802892752553, Tokens per sec: 177130.42963730718, Loss: 2.53840970993042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5379 , TFLOPS: 70.69658162549621, Tokens per sec: 175193.91713935183, Loss: 2.5331923961639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5380 , TFLOPS: 69.55901920914562, Tokens per sec: 172374.91215879004, Loss: 2.5572595596313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5381 , TFLOPS: 70.54597407195162, Tokens per sec: 174820.6950308767, Loss: 2.5453009605407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5382 , TFLOPS: 71.53633835847445, Tokens per sec: 177274.92683051268, Loss: 2.52933406829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5383 , TFLOPS: 68.67814688585186, Tokens per sec: 170192.01350557114, Loss: 2.5295357704162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5384 , TFLOPS: 70.42862807452751, Tokens per sec: 174529.898722531, Loss: 2.542233467102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5385 , TFLOPS: 71.44638106732987, Tokens per sec: 177052.00275344274, Loss: 2.552279472351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5386 , TFLOPS: 69.39585492904345, Tokens per sec: 171970.5730986676, Loss: 2.537872314453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5387 , TFLOPS: 70.42259900091092, Tokens per sec: 174514.9580139518, Loss: 2.530606985092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5388 , TFLOPS: 70.56637413356066, Tokens per sec: 174871.2486307962, Loss: 2.529296636581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5389 , TFLOPS: 70.60585061998748, Tokens per sec: 174969.07571285582, Loss: 2.5118954181671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5390 , TFLOPS: 69.54825101799568, Tokens per sec: 172348.22739490634, Loss: 2.5422887802124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5391 , TFLOPS: 71.56268944426269, Tokens per sec: 177340.2277239077, Loss: 2.5173020362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5392 , TFLOPS: 70.38821758047764, Tokens per sec: 174429.75706669202, Loss: 2.5229291915893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5393 , TFLOPS: 68.76745030991442, Tokens per sec: 170413.31722798245, Loss: 2.5300076007843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5394 , TFLOPS: 71.51195492140931, Tokens per sec: 177214.50198740794, Loss: 2.5101659297943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5395 , TFLOPS: 71.57539310872971, Tokens per sec: 177371.70880388113, Loss: 2.536177158355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5396 , TFLOPS: 68.75324935078322, Tokens per sec: 170378.12568689056, Loss: 2.510692834854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5397 , TFLOPS: 70.70274453283308, Tokens per sec: 175209.1895026329, Loss: 2.5425102710723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5398 , TFLOPS: 71.45973996757324, Tokens per sec: 177085.10757985016, Loss: 2.532247543334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5399 , TFLOPS: 69.69676411395191, Tokens per sec: 172716.25920675395, Loss: 2.527865171432495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5400 , TFLOPS: 69.55465884976343, Tokens per sec: 172364.10670790775, Loss: 2.5379648208618164 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/optimizer.pt +[rank0]:[2024-08-26 13:24:50,548] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006401863996870816, 'preprocessing_with_comm': 0.0013104130048304796, 'state_converting': 0.9651441280002473, : 0.9742139969966956}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400, took 4.71s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003400 + eval ppl=10.374720573425293, eval loss=2.339372158050537 +------------------------------------------------------------------ +iteration: 5401 , TFLOPS: 66.95789473619539, Tokens per sec: 165929.03917730434, Loss: 2.5205647945404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5402 , TFLOPS: 70.19288780124766, Tokens per sec: 173945.7083564087, Loss: 2.5314993858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5403 , TFLOPS: 69.77653786719922, Tokens per sec: 172913.94735510516, Loss: 2.5479085445404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5404 , TFLOPS: 70.62287379571028, Tokens per sec: 175011.26101755453, Loss: 2.5060455799102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5405 , TFLOPS: 71.55003977709606, Tokens per sec: 177308.88045519288, Loss: 2.51385498046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5406 , TFLOPS: 68.70695061674559, Tokens per sec: 170263.39232371873, Loss: 2.5283491611480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5407 , TFLOPS: 70.42051464933887, Tokens per sec: 174509.792761146, Loss: 2.518857955932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5408 , TFLOPS: 71.51708578877071, Tokens per sec: 177227.2168419412, Loss: 2.528125762939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5409 , TFLOPS: 70.5566023430702, Tokens per sec: 174847.03305750922, Loss: 2.534860610961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5410 , TFLOPS: 68.96800819361962, Tokens per sec: 170910.32175707867, Loss: 2.5236916542053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5411 , TFLOPS: 71.45780615476477, Tokens per sec: 177080.31537868368, Loss: 2.5361227989196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5412 , TFLOPS: 71.4602115131292, Tokens per sec: 177086.27612168287, Loss: 2.5228140354156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5413 , TFLOPS: 70.04335522449725, Tokens per sec: 173575.150158848, Loss: 2.52262544631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5414 , TFLOPS: 69.92844154964185, Tokens per sec: 173290.38141377, Loss: 2.5305607318878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5415 , TFLOPS: 71.52895750145575, Tokens per sec: 177256.63625375184, Loss: 2.5344040393829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5416 , TFLOPS: 69.8921041690151, Tokens per sec: 173200.333381684, Loss: 2.5516934394836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5417 , TFLOPS: 69.5481581043453, Tokens per sec: 172347.9971446454, Loss: 2.5248351097106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5418 , TFLOPS: 71.5112347435297, Tokens per sec: 177212.7173072881, Loss: 2.5494437217712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5419 , TFLOPS: 70.72683874974682, Tokens per sec: 175268.89762634222, Loss: 2.5456106662750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5420 , TFLOPS: 69.591624972415, Tokens per sec: 172455.71283199292, Loss: 2.5015923976898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5421 , TFLOPS: 70.56818772246676, Tokens per sec: 174875.74290388884, Loss: 2.530597448348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5422 , TFLOPS: 71.53816476801164, Tokens per sec: 177279.45287454146, Loss: 2.5197393894195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5423 , TFLOPS: 68.8633418391183, Tokens per sec: 170650.94700067365, Loss: 2.5241334438323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5424 , TFLOPS: 71.52566050921644, Tokens per sec: 177248.46594378914, Loss: 2.5290868282318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5425 , TFLOPS: 70.39852762275562, Tokens per sec: 174455.30648720177, Loss: 2.539094924926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5426 , TFLOPS: 70.47927498688642, Tokens per sec: 174655.4073505744, Loss: 2.527499198913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5427 , TFLOPS: 69.74573184071228, Tokens per sec: 172837.60662790656, Loss: 2.5380401611328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5428 , TFLOPS: 70.62250098773853, Tokens per sec: 175010.3371583324, Loss: 2.5391170978546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5429 , TFLOPS: 71.50442449193562, Tokens per sec: 177195.84075362686, Loss: 2.529228925704956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5430 , TFLOPS: 69.01711849881427, Tokens per sec: 171032.02250329775, Loss: 2.5575389862060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5431 , TFLOPS: 71.5426066044212, Tokens per sec: 177290.46023447302, Loss: 2.5352063179016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5432 , TFLOPS: 70.51610762029061, Tokens per sec: 174746.68267359948, Loss: 2.533184766769409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5433 , TFLOPS: 69.79553863410737, Tokens per sec: 172961.03334861083, Loss: 2.530858278274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5434 , TFLOPS: 70.46619249426551, Tokens per sec: 174622.98749270424, Loss: 2.5173139572143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5435 , TFLOPS: 71.50954357693259, Tokens per sec: 177208.5264101631, Loss: 2.5389881134033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5436 , TFLOPS: 68.99325178051804, Tokens per sec: 170972.8781462236, Loss: 2.501281499862671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5437 , TFLOPS: 70.40245318946596, Tokens per sec: 174465.03447394687, Loss: 2.5542988777160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5438 , TFLOPS: 71.53308863373515, Tokens per sec: 177266.87365462194, Loss: 2.5284695625305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5439 , TFLOPS: 70.50002910166228, Tokens per sec: 174706.8383332434, Loss: 2.5290677547454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5440 , TFLOPS: 69.52738406645702, Tokens per sec: 172296.51679030905, Loss: 2.5251269340515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5441 , TFLOPS: 71.56350043564505, Tokens per sec: 177342.23744989125, Loss: 2.4953932762145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5442 , TFLOPS: 71.45763003436468, Tokens per sec: 177079.87893293102, Loss: 2.516465663909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5443 , TFLOPS: 68.93054051849336, Tokens per sec: 170817.47273071064, Loss: 2.532717227935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5444 , TFLOPS: 70.71067350073565, Tokens per sec: 175228.83835854335, Loss: 2.524359941482544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5445 , TFLOPS: 71.57131451182474, Tokens per sec: 177361.60159145607, Loss: 2.551823377609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5446 , TFLOPS: 70.5664052281802, Tokens per sec: 174871.32568668877, Loss: 2.5202250480651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5447 , TFLOPS: 68.46908900077356, Tokens per sec: 169673.94503671996, Loss: 2.5210323333740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5448 , TFLOPS: 71.54560702446362, Tokens per sec: 177297.8956058617, Loss: 2.5173542499542236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5449 , TFLOPS: 71.4608050603145, Tokens per sec: 177087.74699699265, Loss: 2.538645029067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5450 , TFLOPS: 68.83931740660934, Tokens per sec: 170591.41181040672, Loss: 2.5362744331359863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5451 , TFLOPS: 70.46587905535065, Tokens per sec: 174622.21075654507, Loss: 2.5182735919952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5452 , TFLOPS: 71.49952048096391, Tokens per sec: 177183.68807421756, Loss: 2.520423173904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5453 , TFLOPS: 69.73558867350508, Tokens per sec: 172812.47074220498, Loss: 2.5401506423950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5454 , TFLOPS: 69.48393249382194, Tokens per sec: 172188.83900673344, Loss: 2.512019634246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5455 , TFLOPS: 71.51748299823099, Tokens per sec: 177228.20117074, Loss: 2.516324043273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5456 , TFLOPS: 69.91170346884934, Tokens per sec: 173248.90260571436, Loss: 2.524515390396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5457 , TFLOPS: 70.44105232837988, Tokens per sec: 174560.68739222173, Loss: 2.5130996704101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5458 , TFLOPS: 70.43375086282586, Tokens per sec: 174542.59355625603, Loss: 2.5411221981048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5459 , TFLOPS: 70.62293732819259, Tokens per sec: 175011.41845804438, Loss: 2.515834331512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5460 , TFLOPS: 70.63061584403938, Tokens per sec: 175030.44666617108, Loss: 2.519069194793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5461 , TFLOPS: 70.72022255815155, Tokens per sec: 175252.50197473547, Loss: 2.5067203044891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5462 , TFLOPS: 70.5422653176105, Tokens per sec: 174811.5043290664, Loss: 2.519413471221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5463 , TFLOPS: 69.83403636005755, Tokens per sec: 173056.43495438996, Loss: 2.548143148422241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5464 , TFLOPS: 70.43308974587771, Tokens per sec: 174540.9552356305, Loss: 2.5448973178863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5465 , TFLOPS: 71.54552869154189, Tokens per sec: 177297.70148825255, Loss: 2.5185484886169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5466 , TFLOPS: 69.50049072343789, Tokens per sec: 172229.87212376134, Loss: 2.4986677169799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5467 , TFLOPS: 69.2018341488185, Tokens per sec: 171489.76823211685, Loss: 2.5230891704559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5468 , TFLOPS: 71.5459246410244, Tokens per sec: 177298.68269468768, Loss: 2.5021071434020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5469 , TFLOPS: 70.58072191177357, Tokens per sec: 174906.80400574603, Loss: 2.5316498279571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5470 , TFLOPS: 68.69581317362993, Tokens per sec: 170235.79251278748, Loss: 2.522761106491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5471 , TFLOPS: 71.48005569943878, Tokens per sec: 177135.45220137556, Loss: 2.5259854793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5472 , TFLOPS: 71.49749753903006, Tokens per sec: 177178.67500126004, Loss: 2.51564621925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5473 , TFLOPS: 69.66318661375168, Tokens per sec: 172633.05046239076, Loss: 2.540513038635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5474 , TFLOPS: 70.40826988755131, Tokens per sec: 174479.44889256457, Loss: 2.5235989093780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5475 , TFLOPS: 71.52576836745259, Tokens per sec: 177248.7332283799, Loss: 2.521360397338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5476 , TFLOPS: 69.60687096953689, Tokens per sec: 172493.49409234617, Loss: 2.528477191925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5477 , TFLOPS: 69.84024255705789, Tokens per sec: 173071.81459422584, Loss: 2.5227408409118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5478 , TFLOPS: 71.46671279967815, Tokens per sec: 177102.38702592702, Loss: 2.518386125564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5479 , TFLOPS: 71.52455281210119, Tokens per sec: 177245.72094831467, Loss: 2.5190725326538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5480 , TFLOPS: 69.02065754625282, Tokens per sec: 171040.79265270874, Loss: 2.5380074977874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5481 , TFLOPS: 70.43529082166835, Tokens per sec: 174546.4097439082, Loss: 2.549516439437866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5482 , TFLOPS: 71.52227859175144, Tokens per sec: 177240.08517976193, Loss: 2.5314178466796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5483 , TFLOPS: 69.8078014307378, Tokens per sec: 172991.42190951956, Loss: 2.5415141582489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5484 , TFLOPS: 70.36520401758236, Tokens per sec: 174372.7269226843, Loss: 2.5441291332244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5485 , TFLOPS: 71.54293522519454, Tokens per sec: 177291.27459294992, Loss: 2.542865753173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5486 , TFLOPS: 70.65373996470115, Tokens per sec: 175087.75078450312, Loss: 2.5290660858154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5487 , TFLOPS: 69.6213915925576, Tokens per sec: 172529.4777957702, Loss: 2.5139594078063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5488 , TFLOPS: 70.45918485559118, Tokens per sec: 174605.62179211408, Loss: 2.5249650478363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5489 , TFLOPS: 71.50301993335162, Tokens per sec: 177192.3601027308, Loss: 2.5292561054229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5490 , TFLOPS: 69.89698459325281, Tokens per sec: 173212.42761056815, Loss: 2.5606300830841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5491 , TFLOPS: 69.77040653897595, Tokens per sec: 172898.75324834633, Loss: 2.501755952835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5492 , TFLOPS: 71.5048635509954, Tokens per sec: 177196.9287903433, Loss: 2.5010578632354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5493 , TFLOPS: 70.61303693625523, Tokens per sec: 174986.8841962054, Loss: 2.5237247943878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5494 , TFLOPS: 69.77341632174132, Tokens per sec: 172906.21182159387, Loss: 2.5213358402252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5495 , TFLOPS: 70.47456206279428, Tokens per sec: 174643.7282055026, Loss: 2.5187225341796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5496 , TFLOPS: 71.55633712677036, Tokens per sec: 177324.48598139617, Loss: 2.5246686935424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5497 , TFLOPS: 69.0836288576984, Tokens per sec: 171196.8424993335, Loss: 2.519749879837036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5498 , TFLOPS: 71.57329467623433, Tokens per sec: 177366.50865699613, Loss: 2.5298168659210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5499 , TFLOPS: 70.51875786507682, Tokens per sec: 174753.25027212827, Loss: 2.530428647994995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5500 , TFLOPS: 69.71895490506364, Tokens per sec: 172771.25043164624, Loss: 2.5408947467803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5501 , TFLOPS: 70.49325143734382, Tokens per sec: 174690.0425344409, Loss: 2.542344808578491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5502 , TFLOPS: 70.5017319924513, Tokens per sec: 174711.05828420754, Loss: 2.5329227447509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5503 , TFLOPS: 69.87896454473061, Tokens per sec: 173167.7719452579, Loss: 2.5134971141815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5504 , TFLOPS: 70.42498935390338, Tokens per sec: 174520.88157198628, Loss: 2.514249801635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5505 , TFLOPS: 71.52195359072587, Tokens per sec: 177239.27979141875, Loss: 2.526370048522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5506 , TFLOPS: 69.64186852455518, Tokens per sec: 172580.22188898045, Loss: 2.536109685897827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5507 , TFLOPS: 69.50862555069222, Tokens per sec: 172250.03112182274, Loss: 2.5151352882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5508 , TFLOPS: 71.53356127340801, Tokens per sec: 177268.04490779687, Loss: 2.5385868549346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5509 , TFLOPS: 71.4727165454348, Tokens per sec: 177117.26496927917, Loss: 2.5235705375671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5510 , TFLOPS: 69.75521028885599, Tokens per sec: 172861.09526654394, Loss: 2.518672227859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5511 , TFLOPS: 70.33265847748582, Tokens per sec: 174292.07548913787, Loss: 2.512345790863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5512 , TFLOPS: 71.55244859289651, Tokens per sec: 177314.8497660994, Loss: 2.5022523403167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5513 , TFLOPS: 69.60651038504771, Tokens per sec: 172492.60052426034, Loss: 2.5391831398010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5514 , TFLOPS: 69.49633305489229, Tokens per sec: 172219.56896310035, Loss: 2.503897190093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5515 , TFLOPS: 71.52686869044886, Tokens per sec: 177251.45994997525, Loss: 2.534907817840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5516 , TFLOPS: 71.52383790731987, Tokens per sec: 177243.9493355133, Loss: 2.545823574066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5517 , TFLOPS: 68.76539666271226, Tokens per sec: 170408.2280639876, Loss: 2.527707099914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5518 , TFLOPS: 71.5003439751352, Tokens per sec: 177185.72878348935, Loss: 2.5108635425567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5519 , TFLOPS: 71.52869209425906, Tokens per sec: 177255.97854548143, Loss: 2.524947166442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5520 , TFLOPS: 69.79628998043972, Tokens per sec: 172962.89526759, Loss: 2.544018268585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5521 , TFLOPS: 69.39709085235963, Tokens per sec: 171973.6358527933, Loss: 2.525949478149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5522 , TFLOPS: 71.5445585933971, Tokens per sec: 177295.29747818463, Loss: 2.5176188945770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5523 , TFLOPS: 70.5894598815487, Tokens per sec: 174928.45765741568, Loss: 2.515918731689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5524 , TFLOPS: 69.87534787929378, Tokens per sec: 173158.80944989686, Loss: 2.5253729820251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5525 , TFLOPS: 70.51270132086006, Tokens per sec: 174738.24148837573, Loss: 2.5040979385375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5526 , TFLOPS: 71.46753139956847, Tokens per sec: 177104.4156066314, Loss: 2.5392093658447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5527 , TFLOPS: 70.01691931556408, Tokens per sec: 173509.6390643558, Loss: 2.548678159713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5528 , TFLOPS: 69.61174877175104, Tokens per sec: 172505.5818235671, Loss: 2.5215935707092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5529 , TFLOPS: 71.50283573901692, Tokens per sec: 177191.90364887874, Loss: 2.525282144546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5530 , TFLOPS: 69.69461258595983, Tokens per sec: 172710.9274833792, Loss: 2.5340590476989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5531 , TFLOPS: 70.2560964732782, Tokens per sec: 174102.34640871038, Loss: 2.522960901260376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5532 , TFLOPS: 70.50453313667626, Tokens per sec: 174717.99982816845, Loss: 2.54211163520813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5533 , TFLOPS: 70.71812967653811, Tokens per sec: 175247.31558354743, Loss: 2.553126335144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5534 , TFLOPS: 69.5052960861617, Tokens per sec: 172241.78034194538, Loss: 2.5417635440826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5535 , TFLOPS: 71.46775462839977, Tokens per sec: 177104.96879226607, Loss: 2.5331974029541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5536 , TFLOPS: 70.39701522743316, Tokens per sec: 174451.55860498865, Loss: 2.5210726261138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5537 , TFLOPS: 68.74056773158335, Tokens per sec: 170346.6992375176, Loss: 2.5239295959472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5538 , TFLOPS: 71.56536625003555, Tokens per sec: 177346.86114348733, Loss: 2.508596658706665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5539 , TFLOPS: 71.54349186294816, Tokens per sec: 177292.65400262212, Loss: 2.5303189754486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5540 , TFLOPS: 68.84455304124982, Tokens per sec: 170604.38628979863, Loss: 2.548520088195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5541 , TFLOPS: 70.20865825558138, Tokens per sec: 173984.78927950657, Loss: 2.5225114822387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5542 , TFLOPS: 71.51073344134376, Tokens per sec: 177211.47502524836, Loss: 2.5120701789855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5543 , TFLOPS: 69.53662356418661, Tokens per sec: 172319.41328349768, Loss: 2.5061495304107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5544 , TFLOPS: 69.56338901769242, Tokens per sec: 172385.74102574316, Loss: 2.512052536010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5545 , TFLOPS: 71.56830481481884, Tokens per sec: 177354.14323072997, Loss: 2.5112345218658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5546 , TFLOPS: 71.53469841014739, Tokens per sec: 177270.862857903, Loss: 2.527425527572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5547 , TFLOPS: 68.89054405018128, Tokens per sec: 170718.3570762586, Loss: 2.530190944671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5548 , TFLOPS: 70.4439607619278, Tokens per sec: 174567.8948109439, Loss: 2.49743390083313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5549 , TFLOPS: 71.51971166940545, Tokens per sec: 177233.72406341872, Loss: 2.5318150520324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5550 , TFLOPS: 68.86097994812809, Tokens per sec: 170645.09397461615, Loss: 2.537649154663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5551 , TFLOPS: 70.40152012348, Tokens per sec: 174462.72223364798, Loss: 2.5063815116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5552 , TFLOPS: 71.4952439042444, Tokens per sec: 177173.0902460028, Loss: 2.5233302116394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5553 , TFLOPS: 70.63003303012206, Tokens per sec: 175029.00238908172, Loss: 2.520556926727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5554 , TFLOPS: 69.45422691085271, Tokens per sec: 172115.2252997956, Loss: 2.531181812286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5555 , TFLOPS: 71.5370703996982, Tokens per sec: 177276.74090930613, Loss: 2.5090138912200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5556 , TFLOPS: 71.51573331347632, Tokens per sec: 177223.86525917423, Loss: 2.506159782409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5557 , TFLOPS: 68.78323358585457, Tokens per sec: 170452.42992443353, Loss: 2.5316734313964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5558 , TFLOPS: 70.52219632201728, Tokens per sec: 174761.771147204, Loss: 2.535515308380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5559 , TFLOPS: 71.48608557672907, Tokens per sec: 177150.39490154674, Loss: 2.53619647026062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5560 , TFLOPS: 70.7277645001605, Tokens per sec: 175271.19173784717, Loss: 2.512544631958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5561 , TFLOPS: 69.81939596394159, Tokens per sec: 173020.15444004186, Loss: 2.517346143722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5562 , TFLOPS: 70.49555484393181, Tokens per sec: 174695.75062970715, Loss: 2.5282979011535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5563 , TFLOPS: 71.50127453865524, Tokens per sec: 177188.03482240209, Loss: 2.5423977375030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5564 , TFLOPS: 68.8823096213981, Tokens per sec: 170697.95125463666, Loss: 2.525886297225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5565 , TFLOPS: 70.55899228537864, Tokens per sec: 174852.95559782343, Loss: 2.5213332176208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5566 , TFLOPS: 71.47560053281018, Tokens per sec: 177124.41180769278, Loss: 2.5210037231445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5567 , TFLOPS: 69.89909348489367, Tokens per sec: 173217.65367636777, Loss: 2.5047459602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5568 , TFLOPS: 70.40013880664466, Tokens per sec: 174459.29917836539, Loss: 2.5262880325317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5569 , TFLOPS: 70.42594230289681, Tokens per sec: 174523.24308456798, Loss: 2.528071880340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5570 , TFLOPS: 69.68314151414818, Tokens per sec: 172682.50090378703, Loss: 2.532258987426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5571 , TFLOPS: 70.46316711868566, Tokens per sec: 174615.49027874472, Loss: 2.550137758255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5572 , TFLOPS: 71.5391357657837, Tokens per sec: 177281.85911398486, Loss: 2.514824867248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5573 , TFLOPS: 69.74658565725265, Tokens per sec: 172839.72247934924, Loss: 2.5338234901428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5574 , TFLOPS: 69.62745239205307, Tokens per sec: 172544.49712457214, Loss: 2.50018310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5575 , TFLOPS: 71.49556091946648, Tokens per sec: 177173.87584464523, Loss: 2.5260980129241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5576 , TFLOPS: 70.51673813745573, Tokens per sec: 174748.24516459185, Loss: 2.5353546142578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5577 , TFLOPS: 69.66740245336942, Tokens per sec: 172643.49777737661, Loss: 2.518421173095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5578 , TFLOPS: 70.69561376744565, Tokens per sec: 175191.51868048418, Loss: 2.5162525177001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5579 , TFLOPS: 71.46116304345949, Tokens per sec: 177088.63411866105, Loss: 2.5638234615325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5580 , TFLOPS: 69.74398456148725, Tokens per sec: 172833.27667750823, Loss: 2.5188169479370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5581 , TFLOPS: 69.62929787391718, Tokens per sec: 172549.07043192803, Loss: 2.537834644317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5582 , TFLOPS: 71.53610198086287, Tokens per sec: 177274.34106075205, Loss: 2.521446704864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5583 , TFLOPS: 71.47763156739464, Tokens per sec: 177129.44493513, Loss: 2.5292367935180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5584 , TFLOPS: 67.7343089111691, Tokens per sec: 167853.07903197117, Loss: 2.509641170501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5585 , TFLOPS: 71.52948947182793, Tokens per sec: 177257.95453493547, Loss: 2.546030044555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5586 , TFLOPS: 71.5151468906958, Tokens per sec: 177222.4120389243, Loss: 2.5262928009033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5587 , TFLOPS: 69.48215882308814, Tokens per sec: 172184.44365526916, Loss: 2.524170398712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5588 , TFLOPS: 70.41811986095422, Tokens per sec: 174503.85821172112, Loss: 2.530510902404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5589 , TFLOPS: 71.54860829558838, Tokens per sec: 177305.33308632617, Loss: 2.5367350578308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5590 , TFLOPS: 69.69322260376613, Tokens per sec: 172707.48295438403, Loss: 2.537273406982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5591 , TFLOPS: 69.33744337978762, Tokens per sec: 171825.8228450483, Loss: 2.531834125518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5592 , TFLOPS: 71.55427032251951, Tokens per sec: 177319.36421278527, Loss: 2.51432466506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5593 , TFLOPS: 70.7050161283343, Tokens per sec: 175214.81876652176, Loss: 2.526870012283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5594 , TFLOPS: 70.62981825461897, Tokens per sec: 175028.4701517256, Loss: 2.5025107860565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5595 , TFLOPS: 69.63694722499335, Tokens per sec: 172568.02636654035, Loss: 2.5272750854492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5596 , TFLOPS: 71.53368148810615, Tokens per sec: 177268.34281306164, Loss: 2.5263378620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5597 , TFLOPS: 69.66367870932541, Tokens per sec: 172634.26992943126, Loss: 2.522886276245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5598 , TFLOPS: 70.44623907063698, Tokens per sec: 174573.54071090164, Loss: 2.5303494930267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5599 , TFLOPS: 70.56011481354018, Tokens per sec: 174855.73734626445, Loss: 2.5296239852905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5600 , TFLOPS: 70.73595414696084, Tokens per sec: 175291.48658478245, Loss: 2.534263849258423 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/optimizer.pt +[rank0]:[2024-08-26 13:50:00,171] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006397807002940681, 'preprocessing_with_comm': 0.0013688810067833401, 'state_converting': 0.9495150830043713, : 0.9586514059992624}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600, took 4.27s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003600 + eval ppl=10.252153396606445, eval loss=2.3274877071380615 +------------------------------------------------------------------ +iteration: 5601 , TFLOPS: 64.62980489709055, Tokens per sec: 160159.77609573543, Loss: 2.5188655853271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5602 , TFLOPS: 70.06055224142958, Tokens per sec: 173617.76626121407, Loss: 2.5223889350891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5603 , TFLOPS: 71.53854928774011, Tokens per sec: 177280.40575678696, Loss: 2.5044918060302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5604 , TFLOPS: 69.37945391505117, Tokens per sec: 171929.92957927394, Loss: 2.530163288116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5605 , TFLOPS: 69.84743321410504, Tokens per sec: 173089.63383450493, Loss: 2.5285143852233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5606 , TFLOPS: 70.4641466476073, Tokens per sec: 174617.91765931653, Loss: 2.512585401535034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5607 , TFLOPS: 70.59212143152631, Tokens per sec: 174935.05327145412, Loss: 2.5132791996002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5608 , TFLOPS: 68.99060588843426, Tokens per sec: 170966.32133417096, Loss: 2.5261659622192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5609 , TFLOPS: 71.47239952061383, Tokens per sec: 177116.47934684964, Loss: 2.5427119731903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5610 , TFLOPS: 69.65880603659755, Tokens per sec: 172622.19490964245, Loss: 2.51344895362854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5611 , TFLOPS: 67.90262037721659, Tokens per sec: 168270.17338587547, Loss: 2.527975082397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5612 , TFLOPS: 71.51976777296909, Tokens per sec: 177233.86309422975, Loss: 2.5412330627441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5613 , TFLOPS: 71.49796905191347, Tokens per sec: 177179.84346212653, Loss: 2.5220398902893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5614 , TFLOPS: 66.8411186980345, Tokens per sec: 165639.65529079916, Loss: 2.515486478805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5615 , TFLOPS: 70.69994334591209, Tokens per sec: 175202.24785286645, Loss: 2.546828508377075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5616 , TFLOPS: 71.51394353923543, Tokens per sec: 177219.4300014454, Loss: 2.524904251098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5617 , TFLOPS: 69.60264418837983, Tokens per sec: 172483.01966302062, Loss: 2.5315208435058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5618 , TFLOPS: 68.57033537683468, Tokens per sec: 169924.8447098084, Loss: 2.5041544437408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5619 , TFLOPS: 71.48909614275473, Tokens per sec: 177157.85541579957, Loss: 2.5250842571258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5620 , TFLOPS: 71.52725700237494, Tokens per sec: 177252.42222970416, Loss: 2.533781051635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5621 , TFLOPS: 67.30770334585634, Tokens per sec: 166795.90344664367, Loss: 2.5298283100128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5622 , TFLOPS: 71.51513363702095, Tokens per sec: 177222.37919485805, Loss: 2.5255255699157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5623 , TFLOPS: 71.50404165242104, Tokens per sec: 177194.89203513644, Loss: 2.5227484703063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5624 , TFLOPS: 69.69141674740439, Tokens per sec: 172703.00784338813, Loss: 2.4946250915527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5625 , TFLOPS: 68.57651416918284, Tokens per sec: 169940.15643789736, Loss: 2.531244993209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5626 , TFLOPS: 71.5093486859759, Tokens per sec: 177208.04344890342, Loss: 2.5254886150360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5627 , TFLOPS: 70.61501576970235, Tokens per sec: 174991.78796347388, Loss: 2.5141806602478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5628 , TFLOPS: 69.62121346768215, Tokens per sec: 172529.03638270675, Loss: 2.518864870071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5629 , TFLOPS: 70.47675685637724, Tokens per sec: 174649.16714577764, Loss: 2.5210423469543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5630 , TFLOPS: 71.5443360640451, Tokens per sec: 177294.74602593674, Loss: 2.5486419200897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5631 , TFLOPS: 67.90765352247492, Tokens per sec: 168282.64607427234, Loss: 2.5517375469207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5632 , TFLOPS: 70.44229578872482, Tokens per sec: 174563.76882393495, Loss: 2.53358793258667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5633 , TFLOPS: 71.55374776043901, Tokens per sec: 177318.0692463848, Loss: 2.5068209171295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5634 , TFLOPS: 69.76609223829124, Tokens per sec: 172888.06193599018, Loss: 2.534999370574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5635 , TFLOPS: 69.15802365718878, Tokens per sec: 171381.20100773434, Loss: 2.534379720687866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5636 , TFLOPS: 70.09419881525842, Tokens per sec: 173701.14617763806, Loss: 2.538343667984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5637 , TFLOPS: 71.52634452823723, Tokens per sec: 177250.1610182736, Loss: 2.52167010307312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5638 , TFLOPS: 68.20192259703708, Tokens per sec: 169011.877257452, Loss: 2.52629017829895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5639 , TFLOPS: 70.55449756534885, Tokens per sec: 174841.81718645536, Loss: 2.4984381198883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5640 , TFLOPS: 71.52856957698324, Tokens per sec: 177255.67493417556, Loss: 2.5164589881896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5641 , TFLOPS: 67.93608634017095, Tokens per sec: 168353.10572865655, Loss: 2.528909683227539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5642 , TFLOPS: 71.51461161738587, Tokens per sec: 177221.08557269708, Loss: 2.529223918914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5643 , TFLOPS: 70.4364371833041, Tokens per sec: 174549.25055432375, Loss: 2.526298761367798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5644 , TFLOPS: 70.7111531939179, Tokens per sec: 175230.02709109153, Loss: 2.532747983932495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5645 , TFLOPS: 68.42129638618977, Tokens per sec: 169555.50967299019, Loss: 2.526228904724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5646 , TFLOPS: 71.54187666367571, Tokens per sec: 177288.65136089612, Loss: 2.511293888092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5647 , TFLOPS: 69.66531750145194, Tokens per sec: 172638.33103684866, Loss: 2.5509450435638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5648 , TFLOPS: 69.57788190204667, Tokens per sec: 172421.655989696, Loss: 2.536627769470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5649 , TFLOPS: 71.53364119418066, Tokens per sec: 177268.2429602755, Loss: 2.5298213958740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5650 , TFLOPS: 71.50908920001771, Tokens per sec: 177207.4004141143, Loss: 2.5168967247009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5651 , TFLOPS: 67.96356159135935, Tokens per sec: 168421.19242775018, Loss: 2.5016658306121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5652 , TFLOPS: 70.79185069003721, Tokens per sec: 175430.00437603824, Loss: 2.5158591270446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5653 , TFLOPS: 71.49720596763403, Tokens per sec: 177177.95245522144, Loss: 2.512977123260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5654 , TFLOPS: 69.73932798202911, Tokens per sec: 172821.7371606464, Loss: 2.521320343017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5655 , TFLOPS: 68.96220440316978, Tokens per sec: 170895.93932500368, Loss: 2.5105926990509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5656 , TFLOPS: 71.54827503962755, Tokens per sec: 177304.50724134408, Loss: 2.5343711376190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5657 , TFLOPS: 71.50099232240233, Tokens per sec: 177187.33545943865, Loss: 2.496206045150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5658 , TFLOPS: 66.52425248121476, Tokens per sec: 164854.4258998245, Loss: 2.5278546810150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5659 , TFLOPS: 71.51139620494519, Tokens per sec: 177213.11742646215, Loss: 2.5017218589782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5660 , TFLOPS: 71.50544361470509, Tokens per sec: 177198.36625211523, Loss: 2.5290465354919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5661 , TFLOPS: 67.8848230438801, Tokens per sec: 168226.06963332955, Loss: 2.509387254714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5662 , TFLOPS: 70.50412988002832, Tokens per sec: 174717.00051377248, Loss: 2.503711223602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5663 , TFLOPS: 71.52348027198167, Tokens per sec: 177243.06307574845, Loss: 2.54569673538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5664 , TFLOPS: 70.6861828764649, Tokens per sec: 175168.14789373547, Loss: 2.5124099254608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5665 , TFLOPS: 67.77598442107839, Tokens per sec: 167956.35553646288, Loss: 2.530700206756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5666 , TFLOPS: 71.50572432090112, Tokens per sec: 177199.06187299138, Loss: 2.5020289421081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5667 , TFLOPS: 71.52219508277838, Tokens per sec: 177239.8782353274, Loss: 2.5203638076782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5668 , TFLOPS: 68.89219278220426, Tokens per sec: 170722.4428158344, Loss: 2.5130667686462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5669 , TFLOPS: 70.5502507441718, Tokens per sec: 174831.29309575266, Loss: 2.510976552963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5670 , TFLOPS: 71.51890387284813, Tokens per sec: 177231.72225456353, Loss: 2.5380990505218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5671 , TFLOPS: 69.68063070327003, Tokens per sec: 172676.27883784234, Loss: 2.516246795654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5672 , TFLOPS: 70.04480327269731, Tokens per sec: 173578.7385818034, Loss: 2.5113959312438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5673 , TFLOPS: 70.56966273540706, Tokens per sec: 174879.39814844102, Loss: 2.5092666149139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5674 , TFLOPS: 70.59656788236607, Tokens per sec: 174946.07206645206, Loss: 2.526038408279419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5675 , TFLOPS: 68.68652299155828, Tokens per sec: 170212.7704182159, Loss: 2.537858724594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5676 , TFLOPS: 70.42413917471737, Tokens per sec: 174518.77473430833, Loss: 2.557338237762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5677 , TFLOPS: 71.50524195687743, Tokens per sec: 177197.86652180387, Loss: 2.5353689193725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5678 , TFLOPS: 68.05040239801454, Tokens per sec: 168636.3935129463, Loss: 2.533385753631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5679 , TFLOPS: 71.45549291395793, Tokens per sec: 177074.58291313984, Loss: 2.539797067642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5680 , TFLOPS: 70.5967692673552, Tokens per sec: 174946.57112063948, Loss: 2.5248289108276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5681 , TFLOPS: 68.77485376489287, Tokens per sec: 170431.66380497697, Loss: 2.5144052505493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5682 , TFLOPS: 70.70520711300667, Tokens per sec: 175215.2920475786, Loss: 2.5248467922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5683 , TFLOPS: 71.49707014703847, Tokens per sec: 177177.61587682372, Loss: 2.5303633213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5684 , TFLOPS: 69.56139821789887, Tokens per sec: 172380.80760455006, Loss: 2.512709379196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5685 , TFLOPS: 68.85127713471991, Tokens per sec: 170621.04933413753, Loss: 2.5112645626068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5686 , TFLOPS: 71.51737207358094, Tokens per sec: 177227.92628723782, Loss: 2.5251834392547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5687 , TFLOPS: 71.46665881487185, Tokens per sec: 177102.25324563007, Loss: 2.5179200172424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5688 , TFLOPS: 67.64315040537005, Tokens per sec: 167627.17821266278, Loss: 2.5220887660980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5689 , TFLOPS: 71.48472217013777, Tokens per sec: 177147.01622982186, Loss: 2.5097830295562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5690 , TFLOPS: 71.48731147556383, Tokens per sec: 177153.4328139032, Loss: 2.514749050140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5691 , TFLOPS: 69.44732487948252, Tokens per sec: 172098.12130573878, Loss: 2.526116371154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5692 , TFLOPS: 69.71958513836402, Tokens per sec: 172772.81221919053, Loss: 2.513497829437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5693 , TFLOPS: 71.48457901334223, Tokens per sec: 177146.6614715125, Loss: 2.510531425476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5694 , TFLOPS: 70.74825734297583, Tokens per sec: 175321.97525981654, Loss: 2.5214550495147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5695 , TFLOPS: 67.79705731979621, Tokens per sec: 168008.57650085705, Loss: 2.512876033782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5696 , TFLOPS: 71.51983928653308, Tokens per sec: 177234.04031271918, Loss: 2.5146069526672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5697 , TFLOPS: 71.51724762433746, Tokens per sec: 177227.61788830324, Loss: 2.530243158340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5698 , TFLOPS: 67.91173459910154, Tokens per sec: 168292.7594317208, Loss: 2.494262456893921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5699 , TFLOPS: 70.34624499679795, Tokens per sec: 174325.74438067197, Loss: 2.5390665531158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5700 , TFLOPS: 71.50652454740616, Tokens per sec: 177201.04492242305, Loss: 2.5191800594329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5701 , TFLOPS: 69.78392384882399, Tokens per sec: 172932.25063120434, Loss: 2.504030227661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5702 , TFLOPS: 69.42775190288079, Tokens per sec: 172049.6173135783, Loss: 2.5326035022735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5703 , TFLOPS: 70.4637618495066, Tokens per sec: 174616.964087234, Loss: 2.535066843032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5704 , TFLOPS: 71.52429187261866, Tokens per sec: 177245.07431153208, Loss: 2.524569272994995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5705 , TFLOPS: 68.12091092606738, Tokens per sec: 168811.1214126168, Loss: 2.530552625656128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5706 , TFLOPS: 70.40457648689622, Tokens per sec: 174470.29623887956, Loss: 2.508824586868286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5707 , TFLOPS: 71.55000303280383, Tokens per sec: 177308.78939878903, Loss: 2.5256097316741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5708 , TFLOPS: 68.73553585295232, Tokens per sec: 170334.2296879627, Loss: 2.508359670639038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5709 , TFLOPS: 71.5174265505215, Tokens per sec: 177228.06128709766, Loss: 2.5267608165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5710 , TFLOPS: 70.47105163226173, Tokens per sec: 174635.0289719371, Loss: 2.5212481021881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5711 , TFLOPS: 69.57123987520391, Tokens per sec: 172405.1963155, Loss: 2.5343172550201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5712 , TFLOPS: 69.70559878739734, Tokens per sec: 172738.1525007713, Loss: 2.535583972930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5713 , TFLOPS: 70.36428660221688, Tokens per sec: 174370.45346634698, Loss: 2.5075416564941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5714 , TFLOPS: 70.7226862236292, Tokens per sec: 175258.60720918427, Loss: 2.5120177268981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5715 , TFLOPS: 68.65072772808458, Tokens per sec: 170124.06581215424, Loss: 2.5209667682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5716 , TFLOPS: 71.48590484793661, Tokens per sec: 177149.9470356901, Loss: 2.541297197341919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5717 , TFLOPS: 69.98437625575201, Tokens per sec: 173428.99377722753, Loss: 2.5379679203033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5718 , TFLOPS: 68.75538374016125, Tokens per sec: 170383.4149388917, Loss: 2.53226637840271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5719 , TFLOPS: 70.85596892336005, Tokens per sec: 175588.89642141876, Loss: 2.530068874359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5720 , TFLOPS: 71.4857020551424, Tokens per sec: 177149.44449280656, Loss: 2.528533458709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5721 , TFLOPS: 68.8463529480793, Tokens per sec: 170608.84665719816, Loss: 2.533527374267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5722 , TFLOPS: 68.98488377620367, Tokens per sec: 170952.14130972084, Loss: 2.5238046646118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5723 , TFLOPS: 71.50590128684031, Tokens per sec: 177199.50041408307, Loss: 2.5235753059387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5724 , TFLOPS: 71.54193389452998, Tokens per sec: 177288.7931852575, Loss: 2.533604621887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5725 , TFLOPS: 67.76609497608317, Tokens per sec: 167931.84840235242, Loss: 2.5286266803741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5726 , TFLOPS: 71.55177087256686, Tokens per sec: 177313.17030046546, Loss: 2.538409948348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5727 , TFLOPS: 71.55316643888149, Tokens per sec: 177316.62866752746, Loss: 2.524747610092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5728 , TFLOPS: 67.98949270060048, Tokens per sec: 168485.45257299722, Loss: 2.536008596420288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5729 , TFLOPS: 71.52473273141716, Tokens per sec: 177246.1668081994, Loss: 2.5243020057678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5730 , TFLOPS: 71.52853662358639, Tokens per sec: 177255.59327202803, Loss: 2.5276741981506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5731 , TFLOPS: 70.50764104354754, Tokens per sec: 174725.7015637597, Loss: 2.5169434547424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5732 , TFLOPS: 67.95969004063856, Tokens per sec: 168411.59829857756, Loss: 2.5342509746551514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5733 , TFLOPS: 71.54643381399889, Tokens per sec: 177299.94448140849, Loss: 2.533137321472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5734 , TFLOPS: 70.73013368863053, Tokens per sec: 175277.06284786353, Loss: 2.5271518230438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5735 , TFLOPS: 68.63658051030589, Tokens per sec: 170089.00744805395, Loss: 2.515911817550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5736 , TFLOPS: 70.54032016484777, Tokens per sec: 174806.68402624407, Loss: 2.4808881282806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5737 , TFLOPS: 71.53994195066552, Tokens per sec: 177283.85692893062, Loss: 2.5285305976867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5738 , TFLOPS: 69.59512019289105, Tokens per sec: 172464.37437336287, Loss: 2.519530773162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5739 , TFLOPS: 68.8679183592275, Tokens per sec: 170662.28812164956, Loss: 2.5309524536132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5740 , TFLOPS: 71.48513775488917, Tokens per sec: 177148.04609461623, Loss: 2.527263641357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5741 , TFLOPS: 70.77947531790493, Tokens per sec: 175399.33684063467, Loss: 2.5241076946258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5742 , TFLOPS: 68.73488352098612, Tokens per sec: 170332.61313749017, Loss: 2.5273313522338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5743 , TFLOPS: 70.59595196786562, Tokens per sec: 174944.5457624713, Loss: 2.5400009155273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5744 , TFLOPS: 71.55497137776199, Tokens per sec: 177321.10150490372, Loss: 2.508829116821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5745 , TFLOPS: 67.74105707829733, Tokens per sec: 167869.80173348694, Loss: 2.514376401901245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5746 , TFLOPS: 71.52383636134022, Tokens per sec: 177243.9455044054, Loss: 2.5220630168914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5747 , TFLOPS: 70.44337482894602, Tokens per sec: 174566.44280446917, Loss: 2.5373682975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5748 , TFLOPS: 69.7880989174418, Tokens per sec: 172942.5969111611, Loss: 2.5188965797424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5749 , TFLOPS: 70.76815463483699, Tokens per sec: 175371.28294091613, Loss: 2.5180532932281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5750 , TFLOPS: 70.52922819831508, Tokens per sec: 174779.1969112951, Loss: 2.521223306655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5751 , TFLOPS: 70.57103465551235, Tokens per sec: 174882.79791759135, Loss: 2.5091209411621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5752 , TFLOPS: 68.59307078562931, Tokens per sec: 169981.18555731355, Loss: 2.528172016143799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5753 , TFLOPS: 71.53148607177043, Tokens per sec: 177262.90232953662, Loss: 2.524449110031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5754 , TFLOPS: 70.49149421014606, Tokens per sec: 174685.6879318703, Loss: 2.5245561599731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5755 , TFLOPS: 68.25544610250383, Tokens per sec: 169144.51440009946, Loss: 2.5258116722106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5756 , TFLOPS: 71.5468126406462, Tokens per sec: 177300.8832555717, Loss: 2.5095317363739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5757 , TFLOPS: 71.51742141104336, Tokens per sec: 177228.04855090467, Loss: 2.5234718322753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5758 , TFLOPS: 68.72434500960449, Tokens per sec: 170306.49754537363, Loss: 2.5039708614349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5759 , TFLOPS: 69.72329921532494, Tokens per sec: 172782.01611103903, Loss: 2.5101077556610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5760 , TFLOPS: 71.53172005108806, Tokens per sec: 177263.48215606064, Loss: 2.5325474739074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5761 , TFLOPS: 70.684581722424, Tokens per sec: 175164.18005764042, Loss: 2.5012850761413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5762 , TFLOPS: 68.50531800415757, Tokens per sec: 169763.72449805972, Loss: 2.522813320159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5763 , TFLOPS: 71.49064574827474, Tokens per sec: 177161.69550898127, Loss: 2.5492191314697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5764 , TFLOPS: 71.55000424143594, Tokens per sec: 177308.79239391256, Loss: 2.5202393531799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5765 , TFLOPS: 67.45812972332313, Tokens per sec: 167168.6765808398, Loss: 2.477799654006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5766 , TFLOPS: 71.5262282420748, Tokens per sec: 177249.8728483531, Loss: 2.5221447944641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5767 , TFLOPS: 71.54938262077907, Tokens per sec: 177307.25194944767, Loss: 2.5162627696990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5768 , TFLOPS: 70.62991243184722, Tokens per sec: 175028.70353327165, Loss: 2.5197432041168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5769 , TFLOPS: 68.88625061923531, Tokens per sec: 170707.71748141406, Loss: 2.527620553970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5770 , TFLOPS: 71.54089998477075, Tokens per sec: 177286.23104297955, Loss: 2.5288047790527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5771 , TFLOPS: 71.4985737415644, Tokens per sec: 177181.34194969363, Loss: 2.5349044799804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5772 , TFLOPS: 68.05337611800964, Tokens per sec: 168643.76271867644, Loss: 2.5378670692443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5773 , TFLOPS: 70.5015116031385, Tokens per sec: 174710.5121352125, Loss: 2.5213282108306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5774 , TFLOPS: 71.44677903450147, Tokens per sec: 177052.9889599337, Loss: 2.5253396034240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5775 , TFLOPS: 67.91881522626106, Tokens per sec: 168310.30600581758, Loss: 2.514876365661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5776 , TFLOPS: 71.54674226237644, Tokens per sec: 177300.70885046647, Loss: 2.5122644901275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5777 , TFLOPS: 70.57063022384634, Tokens per sec: 174881.79569137117, Loss: 2.5433127880096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5778 , TFLOPS: 69.51108036630546, Tokens per sec: 172256.11442533266, Loss: 2.5174248218536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5779 , TFLOPS: 69.717887927518, Tokens per sec: 172768.6063437509, Loss: 2.5134470462799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5780 , TFLOPS: 70.40612277292033, Tokens per sec: 174474.12810598477, Loss: 2.5335724353790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5781 , TFLOPS: 70.70364410229506, Tokens per sec: 175211.4187348554, Loss: 2.5258703231811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5782 , TFLOPS: 68.46830590531287, Tokens per sec: 169672.00443990328, Loss: 2.538154363632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5783 , TFLOPS: 71.51183223937767, Tokens per sec: 177214.19796781897, Loss: 2.5181193351745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5784 , TFLOPS: 70.62122017448772, Tokens per sec: 175007.16316200362, Loss: 2.5283074378967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5785 , TFLOPS: 68.83642334500884, Tokens per sec: 170584.2400069535, Loss: 2.526590585708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5786 , TFLOPS: 70.77663703241898, Tokens per sec: 175392.30325653753, Loss: 2.534785032272339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5787 , TFLOPS: 70.28074412494239, Tokens per sec: 174163.4260046972, Loss: 2.5315136909484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5788 , TFLOPS: 70.68019218235882, Tokens per sec: 175153.3022938112, Loss: 2.5255610942840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5789 , TFLOPS: 69.25168058092231, Tokens per sec: 171613.2931819079, Loss: 2.523826837539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5790 , TFLOPS: 71.468213824851, Tokens per sec: 177106.1067316569, Loss: 2.5105977058410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5791 , TFLOPS: 70.48706189519636, Tokens per sec: 174674.70416716515, Loss: 2.5463626384735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5792 , TFLOPS: 67.5225385965325, Tokens per sec: 167328.28886387576, Loss: 2.531966209411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5793 , TFLOPS: 71.5038810063994, Tokens per sec: 177194.4939365982, Loss: 2.540081024169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5794 , TFLOPS: 71.51216103143956, Tokens per sec: 177215.0127507678, Loss: 2.524664878845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5795 , TFLOPS: 67.81374788354154, Tokens per sec: 168049.93755643535, Loss: 2.495485305786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5796 , TFLOPS: 71.5425583150265, Tokens per sec: 177290.34056803345, Loss: 2.5120849609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5797 , TFLOPS: 71.54559812956401, Tokens per sec: 177297.8735633206, Loss: 2.5351710319519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5798 , TFLOPS: 70.63162851949131, Tokens per sec: 175032.95618749643, Loss: 2.529521942138672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5799 , TFLOPS: 67.94994495190159, Tokens per sec: 168387.4488951772, Loss: 2.4934046268463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5800 , TFLOPS: 71.51571013853247, Tokens per sec: 177223.80782911045, Loss: 2.5364367961883545 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/optimizer.pt +[rank0]:[2024-08-26 14:15:15,726] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006437677999201696, 'preprocessing_with_comm': 0.001281686003494542, 'state_converting': 0.9453678319987375, : 0.9544754620001186}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800, took 4.37s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0003800 + eval ppl=10.866003036499023, eval loss=2.385638952255249 +------------------------------------------------------------------ +iteration: 5801 , TFLOPS: 67.53994136382049, Tokens per sec: 167371.41483828847, Loss: 2.5067052841186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5802 , TFLOPS: 67.55553013463171, Tokens per sec: 167410.04552960355, Loss: 2.519258975982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5803 , TFLOPS: 71.53402895144238, Tokens per sec: 177269.20386548503, Loss: 2.504758834838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5804 , TFLOPS: 71.54178632240811, Tokens per sec: 177288.4274852831, Loss: 2.5146806240081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5805 , TFLOPS: 68.79549098223816, Tokens per sec: 170482.80510293573, Loss: 2.5258238315582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5806 , TFLOPS: 70.48975483660993, Tokens per sec: 174681.37757263996, Loss: 2.530799627304077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5807 , TFLOPS: 71.55679900009056, Tokens per sec: 177325.6305543632, Loss: 2.519394874572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5808 , TFLOPS: 70.56462438518649, Tokens per sec: 174866.9125615743, Loss: 2.4877021312713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5809 , TFLOPS: 67.95336276882531, Tokens per sec: 168395.91862201778, Loss: 2.5214481353759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5810 , TFLOPS: 71.50037462368336, Tokens per sec: 177185.80473396793, Loss: 2.5231411457061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5811 , TFLOPS: 70.72739070681396, Tokens per sec: 175270.2654367577, Loss: 2.5491364002227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5812 , TFLOPS: 70.38727773303543, Tokens per sec: 174427.42802119747, Loss: 2.5273849964141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5813 , TFLOPS: 70.55924377460211, Tokens per sec: 174853.57881582304, Loss: 2.521941661834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5814 , TFLOPS: 71.52000721167038, Tokens per sec: 177234.45644970785, Loss: 2.518608331680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5815 , TFLOPS: 69.63625850120141, Tokens per sec: 172566.31963311473, Loss: 2.521801471710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5816 , TFLOPS: 70.03941470188701, Tokens per sec: 173565.38510973542, Loss: 2.525545835494995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5817 , TFLOPS: 70.55270392561378, Tokens per sec: 174837.37234960613, Loss: 2.5302255153656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5818 , TFLOPS: 70.7136721029657, Tokens per sec: 175236.26922519278, Loss: 2.5214169025421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5819 , TFLOPS: 69.54341107654008, Tokens per sec: 172336.23348681576, Loss: 2.528862714767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5820 , TFLOPS: 70.47477425340199, Tokens per sec: 174644.25403720324, Loss: 2.5244224071502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5821 , TFLOPS: 71.49102705432122, Tokens per sec: 177162.6404273688, Loss: 2.5210678577423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5822 , TFLOPS: 67.93917620354614, Tokens per sec: 168360.7627504768, Loss: 2.513532876968384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5823 , TFLOPS: 71.5413223228366, Tokens per sec: 177287.2776432311, Loss: 2.516239881515503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5824 , TFLOPS: 70.6353986965558, Tokens per sec: 175042.29910158148, Loss: 2.519627332687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5825 , TFLOPS: 68.61931347912368, Tokens per sec: 170046.21784266393, Loss: 2.4913604259490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5826 , TFLOPS: 70.83903227662093, Tokens per sec: 175546.92554507355, Loss: 2.514434337615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5827 , TFLOPS: 71.50591699979461, Tokens per sec: 177199.53935251472, Loss: 2.5196797847747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5828 , TFLOPS: 69.69812441698897, Tokens per sec: 172719.63018752952, Loss: 2.503786563873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5829 , TFLOPS: 68.75309655064545, Tokens per sec: 170377.74703131683, Loss: 2.531132936477661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5830 , TFLOPS: 71.48201741176987, Tokens per sec: 177140.3135406325, Loss: 2.5302629470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5831 , TFLOPS: 71.50455640994237, Tokens per sec: 177196.16766097952, Loss: 2.488326072692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5832 , TFLOPS: 68.72202548804002, Tokens per sec: 170300.74951542047, Loss: 2.518676280975342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5833 , TFLOPS: 71.56023043588631, Tokens per sec: 177334.1340302689, Loss: 2.5188188552856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5834 , TFLOPS: 71.48218674214769, Tokens per sec: 177140.733159962, Loss: 2.516190767288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5835 , TFLOPS: 68.69502374254058, Tokens per sec: 170233.83621558506, Loss: 2.5260329246520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5836 , TFLOPS: 68.97345841136082, Tokens per sec: 170923.82799703244, Loss: 2.497124195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5837 , TFLOPS: 71.5245695671619, Tokens per sec: 177245.7624692008, Loss: 2.5276262760162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5838 , TFLOPS: 70.63502954475138, Tokens per sec: 175041.3843027447, Loss: 2.5158939361572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5839 , TFLOPS: 68.65091467493455, Tokens per sec: 170124.5290870424, Loss: 2.5090229511260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5840 , TFLOPS: 71.51209989766181, Tokens per sec: 177214.86125453212, Loss: 2.5221331119537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5841 , TFLOPS: 71.44207636117159, Tokens per sec: 177041.33521737996, Loss: 2.5187082290649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5842 , TFLOPS: 68.1064951852874, Tokens per sec: 168775.39761893908, Loss: 2.5323920249938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5843 , TFLOPS: 70.47017052194879, Tokens per sec: 174632.8454835193, Loss: 2.519232749938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5844 , TFLOPS: 71.472500348435, Tokens per sec: 177116.7292092976, Loss: 2.5115015506744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5845 , TFLOPS: 69.6097056621455, Tokens per sec: 172500.5187728975, Loss: 2.529639959335327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5846 , TFLOPS: 69.85004219784021, Tokens per sec: 173096.09918360398, Loss: 2.4983086585998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5847 , TFLOPS: 70.46396761668284, Tokens per sec: 174617.47400096385, Loss: 2.5252625942230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5848 , TFLOPS: 70.66858122553475, Tokens per sec: 175124.52906374863, Loss: 2.5280561447143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5849 , TFLOPS: 68.7884677979844, Tokens per sec: 170465.4008786873, Loss: 2.525073528289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5850 , TFLOPS: 70.52826572499926, Tokens per sec: 174776.8117963933, Loss: 2.523732900619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5851 , TFLOPS: 71.53899762054738, Tokens per sec: 177281.5167748713, Loss: 2.5129847526550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5852 , TFLOPS: 69.3910236599935, Tokens per sec: 171958.6006817536, Loss: 2.507565498352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5853 , TFLOPS: 71.45985250754953, Tokens per sec: 177085.3864663087, Loss: 2.513256788253784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5854 , TFLOPS: 70.48438555280683, Tokens per sec: 174668.07189590044, Loss: 2.5133819580078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5855 , TFLOPS: 68.85448933313725, Tokens per sec: 170629.00951566856, Loss: 2.528404712677002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5856 , TFLOPS: 69.77155610334663, Tokens per sec: 172901.60199549183, Loss: 2.504060983657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5857 , TFLOPS: 70.49043547856851, Tokens per sec: 174683.06427839224, Loss: 2.535463333129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5858 , TFLOPS: 70.59314373382331, Tokens per sec: 174937.58664916197, Loss: 2.5377197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5859 , TFLOPS: 69.45360777081399, Tokens per sec: 172113.69100257574, Loss: 2.528561592102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5860 , TFLOPS: 71.52689394064637, Tokens per sec: 177251.5225227462, Loss: 2.508681297302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5861 , TFLOPS: 70.46578606527622, Tokens per sec: 174621.9803168969, Loss: 2.5461552143096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5862 , TFLOPS: 68.71602869938987, Tokens per sec: 170285.8888125446, Loss: 2.5186350345611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5863 , TFLOPS: 70.99834704991407, Tokens per sec: 175941.72510326502, Loss: 2.545063018798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5864 , TFLOPS: 71.50782701211129, Tokens per sec: 177204.27257344068, Loss: 2.506545066833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5865 , TFLOPS: 69.66226822808031, Tokens per sec: 172630.7746015286, Loss: 2.5181937217712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5866 , TFLOPS: 68.92134598141867, Tokens per sec: 170794.68765497772, Loss: 2.5048861503601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5867 , TFLOPS: 71.51085951966685, Tokens per sec: 177211.78746122168, Loss: 2.5175368785858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5868 , TFLOPS: 71.53053466829631, Tokens per sec: 177260.544646922, Loss: 2.503164768218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5869 , TFLOPS: 67.3288434831296, Tokens per sec: 166848.2910355852, Loss: 2.5495758056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5870 , TFLOPS: 71.48382260421465, Tokens per sec: 177144.787006369, Loss: 2.5216076374053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5871 , TFLOPS: 71.55689249344668, Tokens per sec: 177325.86224119877, Loss: 2.5104732513427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5872 , TFLOPS: 68.2429328142811, Tokens per sec: 169113.50509343107, Loss: 2.5193657875061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5873 , TFLOPS: 71.52768745143086, Tokens per sec: 177253.48892988262, Loss: 2.5030951499938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5874 , TFLOPS: 71.56354098569317, Tokens per sec: 177342.33793737742, Loss: 2.515934944152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5875 , TFLOPS: 69.71970079327713, Tokens per sec: 172773.09882480564, Loss: 2.5296883583068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5876 , TFLOPS: 68.73926461163671, Tokens per sec: 170343.46996273933, Loss: 2.524958848953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5877 , TFLOPS: 71.5567913403315, Tokens per sec: 177325.6115726364, Loss: 2.5108306407928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5878 , TFLOPS: 70.63924214980182, Tokens per sec: 175051.82360211643, Loss: 2.547133207321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5879 , TFLOPS: 69.59339147496607, Tokens per sec: 172460.09041991076, Loss: 2.5209686756134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5880 , TFLOPS: 70.47647699392343, Tokens per sec: 174648.47361578772, Loss: 2.5242133140563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5881 , TFLOPS: 71.50078480904813, Tokens per sec: 177186.8212184875, Loss: 2.522897720336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5882 , TFLOPS: 69.66799211446722, Tokens per sec: 172644.9590225335, Loss: 2.5235650539398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5883 , TFLOPS: 68.96642020899257, Tokens per sec: 170906.38655624192, Loss: 2.5203661918640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5884 , TFLOPS: 71.5544862506228, Tokens per sec: 177319.89930641168, Loss: 2.5214319229125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5885 , TFLOPS: 70.58040247561387, Tokens per sec: 174906.01240775347, Loss: 2.5092954635620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5886 , TFLOPS: 68.4356007031042, Tokens per sec: 169590.95734605473, Loss: 2.525873899459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5887 , TFLOPS: 70.48959219318739, Tokens per sec: 174680.9745243223, Loss: 2.5259997844696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5888 , TFLOPS: 71.51170074839098, Tokens per sec: 177213.87211866907, Loss: 2.5283329486846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5889 , TFLOPS: 67.97461079055344, Tokens per sec: 168448.57356052293, Loss: 2.510596752166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5890 , TFLOPS: 71.54204072868063, Tokens per sec: 177289.05793205157, Loss: 2.5314714908599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5891 , TFLOPS: 70.50785667295604, Tokens per sec: 174726.2359171875, Loss: 2.5267581939697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5892 , TFLOPS: 69.48274006359036, Tokens per sec: 172185.88403326256, Loss: 2.5404646396636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5893 , TFLOPS: 71.47111163173639, Tokens per sec: 177113.28781633868, Loss: 2.5310416221618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5894 , TFLOPS: 70.49144649396499, Tokens per sec: 174685.56968591726, Loss: 2.4999279975891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5895 , TFLOPS: 70.72497331975404, Tokens per sec: 175264.27488532607, Loss: 2.519080877304077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5896 , TFLOPS: 68.97228356314085, Tokens per sec: 170920.91659372312, Loss: 2.5136778354644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5897 , TFLOPS: 71.47872499611928, Tokens per sec: 177132.1545719608, Loss: 2.508507251739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5898 , TFLOPS: 69.46700515573997, Tokens per sec: 172146.8911982661, Loss: 2.5161287784576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5899 , TFLOPS: 69.58600646801285, Tokens per sec: 172441.78955915503, Loss: 2.5219058990478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5900 , TFLOPS: 71.53577895156613, Tokens per sec: 177273.54055857015, Loss: 2.5067379474639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5901 , TFLOPS: 71.55071292958672, Tokens per sec: 177310.54860121862, Loss: 2.5299830436706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5902 , TFLOPS: 68.72793907467663, Tokens per sec: 170315.40403454134, Loss: 2.51033091545105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5903 , TFLOPS: 69.64875282226417, Tokens per sec: 172597.28193707133, Loss: 2.5186328887939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5904 , TFLOPS: 71.43250135643883, Tokens per sec: 177017.60730087842, Loss: 2.531634569168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5905 , TFLOPS: 70.6114536912815, Tokens per sec: 174982.9607407517, Loss: 2.5115561485290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5906 , TFLOPS: 68.56581449093218, Tokens per sec: 169913.64145652537, Loss: 2.5364274978637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5907 , TFLOPS: 71.54606219430899, Tokens per sec: 177299.0235668796, Loss: 2.5077905654907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5908 , TFLOPS: 71.51416678764224, Tokens per sec: 177219.98323559033, Loss: 2.5436346530914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5909 , TFLOPS: 66.877350650698, Tokens per sec: 165729.44206077838, Loss: 2.5548200607299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5910 , TFLOPS: 71.537812228428, Tokens per sec: 177278.57924261776, Loss: 2.522047758102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5911 , TFLOPS: 71.49817095837571, Tokens per sec: 177180.3438085818, Loss: 2.503526449203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5912 , TFLOPS: 70.60257672157415, Tokens per sec: 174960.96263193796, Loss: 2.529527425765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5913 , TFLOPS: 69.71626957505968, Tokens per sec: 172764.59588808293, Loss: 2.524387836456299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5914 , TFLOPS: 71.4951722950082, Tokens per sec: 177172.9127904271, Loss: 2.52313232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5915 , TFLOPS: 70.62577390776836, Tokens per sec: 175018.4478147082, Loss: 2.53056001663208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5916 , TFLOPS: 68.6236157622729, Tokens per sec: 170056.87937424335, Loss: 2.531214714050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5917 , TFLOPS: 70.4010310292322, Tokens per sec: 174461.51020422374, Loss: 2.5210018157958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5918 , TFLOPS: 71.45322995894556, Tokens per sec: 177068.97506133412, Loss: 2.5558459758758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5919 , TFLOPS: 68.96800974796538, Tokens per sec: 170910.3256089187, Loss: 2.5042495727539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5920 , TFLOPS: 71.57288212976101, Tokens per sec: 177365.48632138953, Loss: 2.50919246673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5921 , TFLOPS: 70.49199635997977, Tokens per sec: 174686.9323144746, Loss: 2.5282821655273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5922 , TFLOPS: 69.69538482629721, Tokens per sec: 172712.84118000453, Loss: 2.5099523067474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5923 , TFLOPS: 69.62698490870982, Tokens per sec: 172543.33864935, Loss: 2.4975314140319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5924 , TFLOPS: 70.45880138695328, Tokens per sec: 174604.67151458684, Loss: 2.5144870281219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5925 , TFLOPS: 70.66288787212626, Tokens per sec: 175110.4203068278, Loss: 2.515268325805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5926 , TFLOPS: 68.72543984281424, Tokens per sec: 170309.2106626732, Loss: 2.5543413162231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5927 , TFLOPS: 71.51913481362422, Tokens per sec: 177232.29455124692, Loss: 2.5534350872039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5928 , TFLOPS: 70.4595998071714, Tokens per sec: 174606.6500878406, Loss: 2.51873517036438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5929 , TFLOPS: 68.63661192957053, Tokens per sec: 170089.08530845286, Loss: 2.527297019958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5930 , TFLOPS: 70.8620122060057, Tokens per sec: 175603.8723415372, Loss: 2.506254196166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5931 , TFLOPS: 70.53036980236102, Tokens per sec: 174782.02593187903, Loss: 2.525604248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5932 , TFLOPS: 70.69889021079793, Tokens per sec: 175199.63806804115, Loss: 2.516993522644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5933 , TFLOPS: 70.41474911249368, Tokens per sec: 174495.5051257173, Loss: 2.5067007541656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5934 , TFLOPS: 71.52542272368181, Tokens per sec: 177247.8766850401, Loss: 2.5223231315612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5935 , TFLOPS: 70.3716536179889, Tokens per sec: 174388.70974297385, Loss: 2.5145890712738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5936 , TFLOPS: 67.98116313335753, Tokens per sec: 168464.81098779157, Loss: 2.518620252609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5937 , TFLOPS: 71.52117690270096, Tokens per sec: 177237.35507293374, Loss: 2.555269718170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5938 , TFLOPS: 71.55096037897843, Tokens per sec: 177311.16180807012, Loss: 2.542649507522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5939 , TFLOPS: 67.8161147234189, Tokens per sec: 168055.8028463804, Loss: 2.4971721172332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5940 , TFLOPS: 71.49686785100069, Tokens per sec: 177177.11456495736, Loss: 2.5285465717315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5941 , TFLOPS: 71.5300285049798, Tokens per sec: 177259.29031846518, Loss: 2.538489818572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5942 , TFLOPS: 69.52764953915995, Tokens per sec: 172297.17466091117, Loss: 2.50730037689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5943 , TFLOPS: 68.63076348641206, Tokens per sec: 170074.592222047, Loss: 2.516288995742798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5944 , TFLOPS: 71.49917118990405, Tokens per sec: 177182.82249249364, Loss: 2.543440580368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5945 , TFLOPS: 70.76233564810886, Tokens per sec: 175356.86285078953, Loss: 2.5096895694732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5946 , TFLOPS: 68.5596335157988, Tokens per sec: 169898.3243192542, Loss: 2.526573419570923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5947 , TFLOPS: 71.46491618843932, Tokens per sec: 177097.93482536412, Loss: 2.541329860687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5948 , TFLOPS: 71.51353905658198, Tokens per sec: 177218.4276488727, Loss: 2.514986038208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5949 , TFLOPS: 69.62676779339226, Tokens per sec: 172542.80061367588, Loss: 2.5121357440948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5950 , TFLOPS: 68.78709935364755, Tokens per sec: 170462.00972287398, Loss: 2.505951404571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5951 , TFLOPS: 71.45326027782565, Tokens per sec: 177069.05019485886, Loss: 2.5184810161590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5952 , TFLOPS: 70.69204205245562, Tokens per sec: 175182.667577282, Loss: 2.51334810256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5953 , TFLOPS: 70.28116423590366, Tokens per sec: 174164.4670859383, Loss: 2.522839307785034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5954 , TFLOPS: 70.39685918962734, Tokens per sec: 174451.1719261168, Loss: 2.543339967727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5955 , TFLOPS: 71.4833815933423, Tokens per sec: 177143.69413284597, Loss: 2.5328335762023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5956 , TFLOPS: 67.76862236006724, Tokens per sec: 167938.11153827922, Loss: 2.511465311050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5957 , TFLOPS: 70.27773855148662, Tokens per sec: 174155.97786258257, Loss: 2.5159285068511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5958 , TFLOPS: 71.54115060478607, Tokens per sec: 177286.85210698558, Loss: 2.5205445289611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5959 , TFLOPS: 68.71348619807462, Tokens per sec: 170279.58821420575, Loss: 2.496140241622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5960 , TFLOPS: 71.5454979365275, Tokens per sec: 177297.625273939, Loss: 2.520630359649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5961 , TFLOPS: 70.55132842174143, Tokens per sec: 174833.9636994862, Loss: 2.5272204875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5962 , TFLOPS: 70.61112734707706, Tokens per sec: 174982.1520238633, Loss: 2.525951385498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5963 , TFLOPS: 68.89487162680527, Tokens per sec: 170729.0812878548, Loss: 2.5286781787872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5964 , TFLOPS: 71.4643775285706, Tokens per sec: 177096.59996687114, Loss: 2.5181965827941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5965 , TFLOPS: 69.63727836577728, Tokens per sec: 172568.84696988107, Loss: 2.538748025894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5966 , TFLOPS: 68.81905220439111, Tokens per sec: 170541.19240691364, Loss: 2.523230791091919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5967 , TFLOPS: 71.51190068472921, Tokens per sec: 177214.36758293997, Loss: 2.5299201011657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5968 , TFLOPS: 70.50353037410773, Tokens per sec: 174715.51487206027, Loss: 2.4935426712036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5969 , TFLOPS: 69.54808342649343, Tokens per sec: 172347.81208470205, Loss: 2.556985378265381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5970 , TFLOPS: 69.66750576087453, Tokens per sec: 172643.75378475455, Loss: 2.507720708847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5971 , TFLOPS: 71.56525852389888, Tokens per sec: 177346.59418625364, Loss: 2.545170307159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5972 , TFLOPS: 70.43725016552041, Tokens per sec: 174551.26521381314, Loss: 2.515840768814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5973 , TFLOPS: 70.38879849735366, Tokens per sec: 174431.1966427041, Loss: 2.521827220916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5974 , TFLOPS: 71.44465364067702, Tokens per sec: 177047.72199990522, Loss: 2.525575876235962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5975 , TFLOPS: 71.51501687527197, Tokens per sec: 177222.08984638166, Loss: 2.524585247039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5976 , TFLOPS: 67.77542598795154, Tokens per sec: 167954.97167765803, Loss: 2.543644428253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5977 , TFLOPS: 71.54145375839751, Tokens per sec: 177287.60335502983, Loss: 2.509265899658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5978 , TFLOPS: 71.53182455465668, Tokens per sec: 177263.7411274157, Loss: 2.4935688972473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5979 , TFLOPS: 69.77133991520792, Tokens per sec: 172901.06625746904, Loss: 2.5241401195526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5980 , TFLOPS: 69.74365895584327, Tokens per sec: 172832.46979085403, Loss: 2.504058361053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5981 , TFLOPS: 71.47813328382297, Tokens per sec: 177130.68824370805, Loss: 2.5175232887268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5982 , TFLOPS: 70.51518572103018, Tokens per sec: 174744.39810567655, Loss: 2.5170483589172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5983 , TFLOPS: 68.63078264974209, Tokens per sec: 170074.6397108898, Loss: 2.5356712341308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5984 , TFLOPS: 71.54361295218337, Tokens per sec: 177292.95407508605, Loss: 2.526602029800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5985 , TFLOPS: 71.51370202573003, Tokens per sec: 177218.8315043741, Loss: 2.524240016937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5986 , TFLOPS: 68.18631691522752, Tokens per sec: 168973.20465882015, Loss: 2.5348691940307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5987 , TFLOPS: 70.6196981297747, Tokens per sec: 175003.39136753773, Loss: 2.522653579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5988 , TFLOPS: 71.47475819434312, Tokens per sec: 177122.3244001802, Loss: 2.5002574920654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5989 , TFLOPS: 69.6503325628731, Tokens per sec: 172601.19670832306, Loss: 2.5570626258850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5990 , TFLOPS: 69.63866368990624, Tokens per sec: 172572.2799556787, Loss: 2.5327374935150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5991 , TFLOPS: 70.40442661349837, Tokens per sec: 174469.9248360898, Loss: 2.5203702449798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5992 , TFLOPS: 70.7050959742613, Tokens per sec: 175215.01663352465, Loss: 2.535109519958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5993 , TFLOPS: 70.38742225569992, Tokens per sec: 174427.78616428058, Loss: 2.4908061027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5994 , TFLOPS: 71.54593683303814, Tokens per sec: 177298.71290784076, Loss: 2.5246849060058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5995 , TFLOPS: 70.54481667874215, Tokens per sec: 174817.82688300626, Loss: 2.5077085494995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5996 , TFLOPS: 68.67780769913152, Tokens per sec: 170191.17296351364, Loss: 2.5004987716674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5997 , TFLOPS: 70.97449018785072, Tokens per sec: 175882.60517102174, Loss: 2.510106325149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5998 , TFLOPS: 70.44872470465882, Tokens per sec: 174579.70038582294, Loss: 2.532001495361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5999 , TFLOPS: 70.67898054224898, Tokens per sec: 175150.2997161458, Loss: 2.5113866329193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6000 , TFLOPS: 69.70748778750001, Tokens per sec: 172742.83365111632, Loss: 2.511371374130249 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/model.pt +[rank0]:[2024-08-26 14:40:29,865] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006318166997516528, 'preprocessing_with_comm': 0.0012631650024559349, 'state_converting': 0.9593806420016335, : 0.9683190630021272}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004000 + eval ppl=10.226719856262207, eval loss=2.3250038623809814 +------------------------------------------------------------------ +iteration: 6001 , TFLOPS: 66.7742641490753, Tokens per sec: 165473.98235384168, Loss: 2.491866111755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6002 , TFLOPS: 68.32358043767769, Tokens per sec: 169313.3587882769, Loss: 2.5259482860565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6003 , TFLOPS: 69.87465719841747, Tokens per sec: 173157.09786660058, Loss: 2.520062208175659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6004 , TFLOPS: 71.54653672651138, Tokens per sec: 177300.19950995117, Loss: 2.522374391555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6005 , TFLOPS: 71.52620813308766, Tokens per sec: 177249.8230160678, Loss: 2.5212228298187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6006 , TFLOPS: 68.40497175892207, Tokens per sec: 169515.0554483147, Loss: 2.530715227127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6007 , TFLOPS: 70.56935430133478, Tokens per sec: 174878.6338148331, Loss: 2.5069446563720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6008 , TFLOPS: 71.51222010784775, Tokens per sec: 177215.15914861512, Loss: 2.5300376415252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6009 , TFLOPS: 68.69425542670957, Tokens per sec: 170231.93224431886, Loss: 2.510222911834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6010 , TFLOPS: 70.11158007939632, Tokens per sec: 173744.21886487605, Loss: 2.510596990585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6011 , TFLOPS: 71.50998804015771, Tokens per sec: 177209.62783899662, Loss: 2.513885021209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6012 , TFLOPS: 70.6737300946596, Tokens per sec: 175137.2885286348, Loss: 2.517613410949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6013 , TFLOPS: 68.41998838392908, Tokens per sec: 169552.26829929987, Loss: 2.5281288623809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6014 , TFLOPS: 71.54564923657634, Tokens per sec: 177298.00021212708, Loss: 2.507387638092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6015 , TFLOPS: 71.5224788093938, Tokens per sec: 177240.5813411355, Loss: 2.508511781692505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6016 , TFLOPS: 67.3945593314403, Tokens per sec: 167011.14214689741, Loss: 2.514498472213745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6017 , TFLOPS: 70.67575834427963, Tokens per sec: 175142.31475462337, Loss: 2.5171632766723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6018 , TFLOPS: 71.54975802662888, Tokens per sec: 177308.18224649786, Loss: 2.508301258087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6019 , TFLOPS: 68.83151628074972, Tokens per sec: 170572.0797611616, Loss: 2.534756898880005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6020 , TFLOPS: 69.56767264371356, Tokens per sec: 172396.35632290394, Loss: 2.544832468032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6021 , TFLOPS: 71.51841128358025, Tokens per sec: 177230.50156409453, Loss: 2.5069687366485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6022 , TFLOPS: 70.58976246794091, Tokens per sec: 174929.2074998282, Loss: 2.5120410919189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6023 , TFLOPS: 69.04964432172112, Tokens per sec: 171112.6250754761, Loss: 2.522582769393921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6024 , TFLOPS: 70.56308021412906, Tokens per sec: 174863.0859355895, Loss: 2.5145106315612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6025 , TFLOPS: 70.6727975222808, Tokens per sec: 175134.97751154882, Loss: 2.547032117843628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6026 , TFLOPS: 69.5282639645971, Tokens per sec: 172298.69727482894, Loss: 2.5253357887268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6027 , TFLOPS: 70.86663224786595, Tokens per sec: 175615.32131420652, Loss: 2.5141189098358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6028 , TFLOPS: 70.44068308701131, Tokens per sec: 174559.77237143513, Loss: 2.5086417198181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6029 , TFLOPS: 69.68504976933419, Tokens per sec: 172687.2297703496, Loss: 2.512038469314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6030 , TFLOPS: 69.76212690516358, Tokens per sec: 172878.2354036823, Loss: 2.5181844234466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6031 , TFLOPS: 70.50792234610083, Tokens per sec: 174726.39866247482, Loss: 2.5302395820617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6032 , TFLOPS: 70.55666311410656, Tokens per sec: 174847.18365483184, Loss: 2.527419328689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6033 , TFLOPS: 68.98117700069353, Tokens per sec: 170942.9555116717, Loss: 2.5148329734802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6034 , TFLOPS: 71.48527613617286, Tokens per sec: 177148.3890186812, Loss: 2.528742551803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6035 , TFLOPS: 70.60550012859683, Tokens per sec: 174968.20715658003, Loss: 2.5091147422790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6036 , TFLOPS: 68.4803409044564, Tokens per sec: 169701.82849354553, Loss: 2.5030956268310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6037 , TFLOPS: 70.70391138195907, Tokens per sec: 175212.08108330617, Loss: 2.5277044773101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6038 , TFLOPS: 71.49640523517078, Tokens per sec: 177175.96815196952, Loss: 2.5212631225585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6039 , TFLOPS: 68.61937963008052, Tokens per sec: 170046.3817720221, Loss: 2.5289885997772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6040 , TFLOPS: 69.89385222505734, Tokens per sec: 173204.66525712094, Loss: 2.519906759262085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6041 , TFLOPS: 71.54225893212424, Tokens per sec: 177289.5986642219, Loss: 2.563040256500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6042 , TFLOPS: 70.72971003564624, Tokens per sec: 175276.01298909922, Loss: 2.5246407985687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6043 , TFLOPS: 69.03667016697356, Tokens per sec: 171080.4736907905, Loss: 2.535536289215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6044 , TFLOPS: 70.95512577593249, Tokens per sec: 175834.61802512227, Loss: 2.5213215351104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6045 , TFLOPS: 71.50553583430838, Tokens per sec: 177198.594782451, Loss: 2.519407033920288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6046 , TFLOPS: 67.85044118089668, Tokens per sec: 168140.86758938179, Loss: 2.5217912197113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6047 , TFLOPS: 70.55975260762304, Tokens per sec: 174854.8397601015, Loss: 2.512146472930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6048 , TFLOPS: 71.5409414468784, Tokens per sec: 177286.33379064972, Loss: 2.505331516265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6049 , TFLOPS: 69.69044182420636, Tokens per sec: 172700.59187630675, Loss: 2.5371875762939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6050 , TFLOPS: 68.7696270016725, Tokens per sec: 170418.7113099414, Loss: 2.5114195346832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6051 , TFLOPS: 71.53307705508776, Tokens per sec: 177266.84496145824, Loss: 2.5220837593078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6052 , TFLOPS: 70.59355033435772, Tokens per sec: 174938.59425007695, Loss: 2.515134811401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6053 , TFLOPS: 69.04707549756884, Tokens per sec: 171106.25924624738, Loss: 2.5082666873931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6054 , TFLOPS: 71.432581774466, Tokens per sec: 177017.80658560846, Loss: 2.5181198120117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6055 , TFLOPS: 71.52712364768554, Tokens per sec: 177252.0917620936, Loss: 2.539250373840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6056 , TFLOPS: 68.77762109455733, Tokens per sec: 170438.521552732, Loss: 2.5155112743377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6057 , TFLOPS: 69.74251969980652, Tokens per sec: 172829.64658889553, Loss: 2.5057482719421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6058 , TFLOPS: 71.55024940400928, Tokens per sec: 177309.39993377618, Loss: 2.50970196723938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6059 , TFLOPS: 70.62901319241006, Tokens per sec: 175026.47511888697, Loss: 2.522106885910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6060 , TFLOPS: 68.96760385057941, Tokens per sec: 170909.31975048294, Loss: 2.4954776763916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6061 , TFLOPS: 70.4519950092235, Tokens per sec: 174587.80456078803, Loss: 2.517157793045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6062 , TFLOPS: 71.53744482578185, Tokens per sec: 177277.66877838783, Loss: 2.5382583141326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6063 , TFLOPS: 69.02584464087907, Tokens per sec: 171053.6468445982, Loss: 2.5181806087493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6064 , TFLOPS: 70.68362168561475, Tokens per sec: 175161.80098067035, Loss: 2.533818483352661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6065 , TFLOPS: 70.52410484303408, Tokens per sec: 174766.5006725246, Loss: 2.515920400619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6066 , TFLOPS: 68.67511128676729, Tokens per sec: 170184.49095664074, Loss: 2.5208444595336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6067 , TFLOPS: 70.6002120506192, Tokens per sec: 174955.10271682154, Loss: 2.5182347297668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6068 , TFLOPS: 70.49858811752478, Tokens per sec: 174703.2674158132, Loss: 2.502254009246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6069 , TFLOPS: 70.69838148375986, Tokens per sec: 175198.37738639992, Loss: 2.5106403827667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6070 , TFLOPS: 69.03350753995335, Tokens per sec: 171072.6363526423, Loss: 2.550825834274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6071 , TFLOPS: 71.54681830654971, Tokens per sec: 177300.89729630479, Loss: 2.5262303352355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6072 , TFLOPS: 69.50932625312073, Tokens per sec: 172251.7675396294, Loss: 2.52752423286438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6073 , TFLOPS: 70.44820769463493, Tokens per sec: 174578.41917803042, Loss: 2.5127816200256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6074 , TFLOPS: 70.80562938110303, Tokens per sec: 175464.14949034748, Loss: 2.503361225128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6075 , TFLOPS: 71.48019626132813, Tokens per sec: 177135.80052922157, Loss: 2.4910640716552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6076 , TFLOPS: 68.8410851016619, Tokens per sec: 170595.79235347465, Loss: 2.5193300247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6077 , TFLOPS: 70.00905560327395, Tokens per sec: 173490.15191904144, Loss: 2.5168352127075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6078 , TFLOPS: 71.54198602094216, Tokens per sec: 177288.92236024907, Loss: 2.502725601196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6079 , TFLOPS: 70.62543201800699, Tokens per sec: 175017.60057421727, Loss: 2.533970594406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6080 , TFLOPS: 67.68805487835914, Tokens per sec: 167738.4564433658, Loss: 2.523214340209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6081 , TFLOPS: 71.47374636261542, Tokens per sec: 177119.8169696965, Loss: 2.518160581588745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6082 , TFLOPS: 71.51552680618346, Tokens per sec: 177223.35351135387, Loss: 2.5120632648468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6083 , TFLOPS: 67.76899610269116, Tokens per sec: 167939.0377136725, Loss: 2.474454879760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6084 , TFLOPS: 70.63497448139395, Tokens per sec: 175041.24784967926, Loss: 2.5010721683502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6085 , TFLOPS: 71.43426748118992, Tokens per sec: 177021.98395256628, Loss: 2.521583318710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6086 , TFLOPS: 68.63902619112395, Tokens per sec: 170095.06811453513, Loss: 2.512411117553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6087 , TFLOPS: 69.50627724239179, Tokens per sec: 172244.2117551668, Loss: 2.499149799346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6088 , TFLOPS: 71.5150484445018, Tokens per sec: 177222.16807841085, Loss: 2.5333876609802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6089 , TFLOPS: 70.68816000741522, Tokens per sec: 175173.04744202932, Loss: 2.511533498764038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6090 , TFLOPS: 68.51796022683025, Tokens per sec: 169795.05331849842, Loss: 2.5119693279266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6091 , TFLOPS: 71.52164623528118, Tokens per sec: 177238.5181307688, Loss: 2.5139384269714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6092 , TFLOPS: 71.54112779037652, Tokens per sec: 177286.7955703655, Loss: 2.522865056991577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6093 , TFLOPS: 69.72458226480131, Tokens per sec: 172785.19564898105, Loss: 2.499437093734741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6094 , TFLOPS: 69.55610236078415, Tokens per sec: 172367.68388723297, Loss: 2.5210142135620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6095 , TFLOPS: 71.46158842431002, Tokens per sec: 177089.68825926824, Loss: 2.526200532913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6096 , TFLOPS: 69.67155828206155, Tokens per sec: 172653.79637867803, Loss: 2.519125461578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6097 , TFLOPS: 69.65103780202487, Tokens per sec: 172602.94436862966, Loss: 2.5181760787963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6098 , TFLOPS: 70.52181818856849, Tokens per sec: 174760.8340908631, Loss: 2.5117347240448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6099 , TFLOPS: 70.6402901815705, Tokens per sec: 175054.4207402895, Loss: 2.5219345092773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6100 , TFLOPS: 68.74436649902998, Tokens per sec: 170356.11300171912, Loss: 2.5419487953186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6101 , TFLOPS: 71.55201572809774, Tokens per sec: 177313.77707944403, Loss: 2.528125524520874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6102 , TFLOPS: 70.55379601639189, Tokens per sec: 174840.078670858, Loss: 2.5227441787719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6103 , TFLOPS: 68.72603832472186, Tokens per sec: 170310.6937667685, Loss: 2.5126407146453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6104 , TFLOPS: 70.64539249025955, Tokens per sec: 175067.06482328664, Loss: 2.502197742462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6105 , TFLOPS: 70.33203811510037, Tokens per sec: 174290.53816280817, Loss: 2.499542474746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6106 , TFLOPS: 69.75715884382137, Tokens per sec: 172865.9240003992, Loss: 2.5355629920959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6107 , TFLOPS: 69.89506693708377, Tokens per sec: 173207.67544733398, Loss: 2.518838882446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6108 , TFLOPS: 71.53563919551455, Tokens per sec: 177273.1942276789, Loss: 2.5215582847595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6109 , TFLOPS: 69.76369937346448, Tokens per sec: 172882.1321533536, Loss: 2.542275905609131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6110 , TFLOPS: 69.51418412303556, Tokens per sec: 172263.80587641685, Loss: 2.5335400104522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6111 , TFLOPS: 70.94916488389067, Tokens per sec: 175819.8462779962, Loss: 2.5296237468719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6112 , TFLOPS: 71.48954957049862, Tokens per sec: 177158.97905969812, Loss: 2.5147275924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6113 , TFLOPS: 69.34843950480635, Tokens per sec: 171853.0724541679, Loss: 2.5405995845794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6114 , TFLOPS: 70.60125748404346, Tokens per sec: 174957.69341601085, Loss: 2.5075058937072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6115 , TFLOPS: 71.51793790694975, Tokens per sec: 177229.32848465742, Loss: 2.507408618927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6116 , TFLOPS: 69.66443823775295, Tokens per sec: 172636.15212454367, Loss: 2.4971749782562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6117 , TFLOPS: 68.72419997417802, Tokens per sec: 170306.13813160989, Loss: 2.5297932624816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6118 , TFLOPS: 71.53407165242845, Tokens per sec: 177269.30968323242, Loss: 2.5326762199401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6119 , TFLOPS: 70.59976995638637, Tokens per sec: 174954.00715861187, Loss: 2.5049452781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6120 , TFLOPS: 67.75664375416, Tokens per sec: 167908.4272332887, Loss: 2.490123748779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6121 , TFLOPS: 71.52333054400762, Tokens per sec: 177242.69203333484, Loss: 2.5224039554595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6122 , TFLOPS: 71.51359749786472, Tokens per sec: 177218.5724728092, Loss: 2.516033411026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6123 , TFLOPS: 68.75742534172787, Tokens per sec: 170388.47425247479, Loss: 2.5276975631713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6124 , TFLOPS: 70.30991752210853, Tokens per sec: 174235.72089658992, Loss: 2.5147109031677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6125 , TFLOPS: 71.48991011983327, Tokens per sec: 177159.87254066713, Loss: 2.529219150543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6126 , TFLOPS: 70.59429161550555, Tokens per sec: 174940.43122642016, Loss: 2.516094923019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6127 , TFLOPS: 67.39194495821573, Tokens per sec: 167004.66344204993, Loss: 2.525451421737671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6128 , TFLOPS: 71.43385130220702, Tokens per sec: 177020.95261520075, Loss: 2.541750192642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6129 , TFLOPS: 71.45731469615401, Tokens per sec: 177079.0974901076, Loss: 2.5110716819763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6130 , TFLOPS: 68.95208193201171, Tokens per sec: 170870.85472639173, Loss: 2.5072269439697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6131 , TFLOPS: 69.06885307321433, Tokens per sec: 171160.22647769493, Loss: 2.5462124347686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6132 , TFLOPS: 71.5191888970698, Tokens per sec: 177232.42857598283, Loss: 2.519888162612915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6133 , TFLOPS: 69.49624557996039, Tokens per sec: 172219.35219058293, Loss: 2.5218701362609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6134 , TFLOPS: 70.62035449330115, Tokens per sec: 175005.0179086607, Loss: 2.5237882137298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6135 , TFLOPS: 70.57403343949935, Tokens per sec: 174890.2292346545, Loss: 2.5068297386169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6136 , TFLOPS: 69.6021705780295, Tokens per sec: 172481.8460044, Loss: 2.5032362937927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6137 , TFLOPS: 69.114412635774, Tokens per sec: 171273.12809251793, Loss: 2.5143885612487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6138 , TFLOPS: 71.48551728404335, Tokens per sec: 177148.98660966894, Loss: 2.510373592376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6139 , TFLOPS: 69.0727020537007, Tokens per sec: 171169.76467534035, Loss: 2.5043158531188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6140 , TFLOPS: 68.80583643016226, Tokens per sec: 170508.44226253682, Loss: 2.500229835510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6141 , TFLOPS: 71.55068319506924, Tokens per sec: 177310.4749158088, Loss: 2.5096166133880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6142 , TFLOPS: 70.55951913222272, Tokens per sec: 174854.26118234007, Loss: 2.5292210578918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6143 , TFLOPS: 69.64618013622491, Tokens per sec: 172590.90653766127, Loss: 2.4989771842956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6144 , TFLOPS: 69.8459984841479, Tokens per sec: 173086.0784156226, Loss: 2.5338568687438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6145 , TFLOPS: 71.52303339943695, Tokens per sec: 177241.95567635554, Loss: 2.534348249435425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6146 , TFLOPS: 69.67427260008219, Tokens per sec: 172660.52275774, Loss: 2.5203440189361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6147 , TFLOPS: 68.77620393380093, Tokens per sec: 170435.0096722645, Loss: 2.502309799194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6148 , TFLOPS: 71.53025991376893, Tokens per sec: 177259.86377493644, Loss: 2.5242769718170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6149 , TFLOPS: 71.57360716864812, Tokens per sec: 177367.2830476214, Loss: 2.5291390419006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6150 , TFLOPS: 68.55677873287947, Tokens per sec: 169891.2498526998, Loss: 2.5022802352905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6151 , TFLOPS: 70.56962109344038, Tokens per sec: 174879.29495506032, Loss: 2.5129263401031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6152 , TFLOPS: 71.51023485058636, Tokens per sec: 177210.2394624272, Loss: 2.5242838859558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6153 , TFLOPS: 69.6768083533777, Tokens per sec: 172666.80663374253, Loss: 2.5260941982269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6154 , TFLOPS: 69.67313630661786, Tokens per sec: 172657.7068973623, Loss: 2.523331642150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6155 , TFLOPS: 71.50756325862913, Tokens per sec: 177203.61896325706, Loss: 2.5269839763641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6156 , TFLOPS: 70.55385252333053, Tokens per sec: 174840.2187012767, Loss: 2.506793975830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6157 , TFLOPS: 67.95436283105433, Tokens per sec: 168398.39688638723, Loss: 2.5270938873291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6158 , TFLOPS: 71.49123800560501, Tokens per sec: 177163.16318788822, Loss: 2.5103163719177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6159 , TFLOPS: 70.65885316592318, Tokens per sec: 175100.42186039692, Loss: 2.524099826812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6160 , TFLOPS: 69.556194619471, Tokens per sec: 172367.91251442212, Loss: 2.4948890209198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6161 , TFLOPS: 69.92442508495338, Tokens per sec: 173280.42817182187, Loss: 2.491804361343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6162 , TFLOPS: 71.51526561380376, Tokens per sec: 177222.70624786426, Loss: 2.517536163330078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6163 , TFLOPS: 69.59167542072916, Tokens per sec: 172455.83784847273, Loss: 2.5237314701080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6164 , TFLOPS: 68.73499178258118, Tokens per sec: 170332.88142164872, Loss: 2.5183496475219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6165 , TFLOPS: 71.54552305482534, Tokens per sec: 177297.687519848, Loss: 2.51939058303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6166 , TFLOPS: 70.70353935141655, Tokens per sec: 175211.15915063923, Loss: 2.5238027572631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6167 , TFLOPS: 68.42086337748644, Tokens per sec: 169554.4366297235, Loss: 2.5179269313812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6168 , TFLOPS: 70.50530045433975, Tokens per sec: 174719.90132586565, Loss: 2.496994972229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6169 , TFLOPS: 71.49939681796636, Tokens per sec: 177183.38162368702, Loss: 2.5087532997131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6170 , TFLOPS: 68.75924569940317, Tokens per sec: 170392.98529932313, Loss: 2.516779899597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6171 , TFLOPS: 70.60760888267738, Tokens per sec: 174973.43288149004, Loss: 2.5185415744781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6172 , TFLOPS: 70.51897695328461, Tokens per sec: 174753.79319684173, Loss: 2.5126755237579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6173 , TFLOPS: 70.63572393642589, Tokens per sec: 175043.1050818077, Loss: 2.5398902893066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6174 , TFLOPS: 69.93570956944092, Tokens per sec: 173308.39236747054, Loss: 2.513512134552002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6175 , TFLOPS: 71.50687445111693, Tokens per sec: 177201.91202236342, Loss: 2.5388481616973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6176 , TFLOPS: 69.59684893259815, Tokens per sec: 172468.6583807935, Loss: 2.4885241985321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6177 , TFLOPS: 69.54234327206795, Tokens per sec: 172333.58734970549, Loss: 2.5192267894744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6178 , TFLOPS: 70.80415054775034, Tokens per sec: 175460.48477839257, Loss: 2.4957964420318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6179 , TFLOPS: 70.26205906303626, Tokens per sec: 174117.12236296432, Loss: 2.5190610885620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6180 , TFLOPS: 69.0242011051297, Tokens per sec: 171049.5739819609, Loss: 2.5406665802001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6181 , TFLOPS: 70.71401399757646, Tokens per sec: 175237.116477701, Loss: 2.5281262397766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6182 , TFLOPS: 71.53112545088385, Tokens per sec: 177262.00867125398, Loss: 2.510547161102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6183 , TFLOPS: 69.6568933989458, Tokens per sec: 172617.45518284696, Loss: 2.527073621749878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6184 , TFLOPS: 69.74696531772892, Tokens per sec: 172840.6633198325, Loss: 2.509972333908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6185 , TFLOPS: 71.55300475016172, Tokens per sec: 177316.22798506837, Loss: 2.555041551589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6186 , TFLOPS: 70.78199687325902, Tokens per sec: 175405.58553257436, Loss: 2.524055242538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6187 , TFLOPS: 67.73095492733187, Tokens per sec: 167844.76749055073, Loss: 2.5255351066589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6188 , TFLOPS: 71.49167036713949, Tokens per sec: 177164.23462739924, Loss: 2.5274195671081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6189 , TFLOPS: 71.51605689592353, Tokens per sec: 177224.66713212398, Loss: 2.5073440074920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6190 , TFLOPS: 68.75016753312578, Tokens per sec: 170370.48860324515, Loss: 2.5665488243103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6191 , TFLOPS: 69.63899623014423, Tokens per sec: 172573.10402702077, Loss: 2.52421498298645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6192 , TFLOPS: 71.50386661735021, Tokens per sec: 177194.45827894926, Loss: 2.5079846382141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6193 , TFLOPS: 70.30844140536918, Tokens per sec: 174232.0629167021, Loss: 2.5152652263641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6194 , TFLOPS: 68.55676000157034, Tokens per sec: 169891.20343445236, Loss: 2.5357584953308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6195 , TFLOPS: 71.42798769493515, Tokens per sec: 177006.4219504514, Loss: 2.5258536338806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6196 , TFLOPS: 71.49860862745038, Tokens per sec: 177181.4284007621, Loss: 2.5360774993896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6197 , TFLOPS: 68.93699707162959, Tokens per sec: 170833.47278063028, Loss: 2.5183169841766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6198 , TFLOPS: 69.76361184942088, Tokens per sec: 172881.91525913202, Loss: 2.5320680141448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6199 , TFLOPS: 71.51606483676369, Tokens per sec: 177224.68681040066, Loss: 2.521256685256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6200 , TFLOPS: 68.7681671882522, Tokens per sec: 170415.09373147416, Loss: 2.51977801322937 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/optimizer.pt +[rank0]:[2024-08-26 15:05:45,158] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006385647000570316, 'preprocessing_with_comm': 0.0012284819968044758, 'state_converting': 0.9868915990009555, : 0.9958786369970767}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200, took 4.63s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004200 + eval ppl=10.600509643554688, eval loss=2.3609020709991455 +------------------------------------------------------------------ +iteration: 6201 , TFLOPS: 64.6599349037242, Tokens per sec: 160234.44157126787, Loss: 2.5086958408355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6202 , TFLOPS: 71.47682404568658, Tokens per sec: 177127.44380738158, Loss: 2.517411470413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6203 , TFLOPS: 69.6232066855579, Tokens per sec: 172533.97579617394, Loss: 2.521885633468628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6204 , TFLOPS: 69.85656507683103, Tokens per sec: 173112.26359629765, Loss: 2.503561496734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6205 , TFLOPS: 70.40971008459607, Tokens per sec: 174483.01785949228, Loss: 2.5414533615112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6206 , TFLOPS: 70.70734071340561, Tokens per sec: 175220.5793444027, Loss: 2.5340094566345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6207 , TFLOPS: 68.59057388834708, Tokens per sec: 169974.99797079226, Loss: 2.519973039627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6208 , TFLOPS: 70.31111086429202, Tokens per sec: 174238.67812997807, Loss: 2.518411636352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6209 , TFLOPS: 71.44167062811256, Tokens per sec: 177040.32976616453, Loss: 2.521972417831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6210 , TFLOPS: 68.71290085681353, Tokens per sec: 170278.13767408012, Loss: 2.4930179119110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6211 , TFLOPS: 70.67538291159383, Tokens per sec: 175141.3843910706, Loss: 2.525104284286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6212 , TFLOPS: 70.28613015454528, Tokens per sec: 174176.7731793733, Loss: 2.5199532508850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6213 , TFLOPS: 68.83408061726048, Tokens per sec: 170578.4344695203, Loss: 2.5191304683685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6214 , TFLOPS: 70.52790044489893, Tokens per sec: 174775.90659206576, Loss: 2.5218076705932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6215 , TFLOPS: 71.4877059702351, Tokens per sec: 177154.41041515575, Loss: 2.5093271732330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6216 , TFLOPS: 69.63776004734615, Tokens per sec: 172570.04062987035, Loss: 2.512845516204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6217 , TFLOPS: 68.97662607970227, Tokens per sec: 170931.67782813002, Loss: 2.523426055908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6218 , TFLOPS: 71.4938966477649, Tokens per sec: 177169.75159603404, Loss: 2.5255210399627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6219 , TFLOPS: 70.64897199359436, Tokens per sec: 175075.9352268651, Loss: 2.5229721069335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6220 , TFLOPS: 68.09746117218538, Tokens per sec: 168753.01033929357, Loss: 2.4826443195343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6221 , TFLOPS: 70.74796142265046, Tokens per sec: 175321.24193665196, Loss: 2.544433832168579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6222 , TFLOPS: 71.53047960125548, Tokens per sec: 177260.4081847287, Loss: 2.5202155113220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6223 , TFLOPS: 69.52784573925005, Tokens per sec: 172297.66086634764, Loss: 2.512843132019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6224 , TFLOPS: 69.67111204535735, Tokens per sec: 172652.690554968, Loss: 2.517164468765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6225 , TFLOPS: 71.50749392336816, Tokens per sec: 177203.44714284246, Loss: 2.5078372955322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6226 , TFLOPS: 70.57730127593327, Tokens per sec: 174898.32729331963, Loss: 2.5032637119293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6227 , TFLOPS: 67.59021857915926, Tokens per sec: 167496.00731639043, Loss: 2.533712863922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6228 , TFLOPS: 71.49381709570186, Tokens per sec: 177169.55445725843, Loss: 2.5119664669036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6229 , TFLOPS: 71.53572368319155, Tokens per sec: 177273.4035974496, Loss: 2.5109570026397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6230 , TFLOPS: 68.74037921614892, Tokens per sec: 170346.2320755044, Loss: 2.4996297359466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6231 , TFLOPS: 68.8502507590176, Tokens per sec: 170618.5058620826, Loss: 2.518254041671753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6232 , TFLOPS: 71.53260129998343, Tokens per sec: 177265.66598790095, Loss: 2.514878988265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6233 , TFLOPS: 70.51882527739777, Tokens per sec: 174753.41732728566, Loss: 2.5174930095672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6234 , TFLOPS: 69.75667287129764, Tokens per sec: 172864.7197069511, Loss: 2.53105092048645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6235 , TFLOPS: 69.97074260550349, Tokens per sec: 173395.20809004112, Loss: 2.5112879276275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6236 , TFLOPS: 70.69770356065014, Tokens per sec: 175196.69741825463, Loss: 2.51809024810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6237 , TFLOPS: 69.48632304925849, Tokens per sec: 172194.76306644664, Loss: 2.5033493041992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6238 , TFLOPS: 69.18535270067392, Tokens per sec: 171448.92538803318, Loss: 2.528676986694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6239 , TFLOPS: 71.55312407288324, Tokens per sec: 177316.52367991678, Loss: 2.4967305660247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6240 , TFLOPS: 69.00547708476645, Tokens per sec: 171003.17379687962, Loss: 2.5242362022399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6241 , TFLOPS: 70.63224515094471, Tokens per sec: 175034.48426816548, Loss: 2.5276520252227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6242 , TFLOPS: 70.38256655926666, Tokens per sec: 174415.75321361984, Loss: 2.523786783218384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6243 , TFLOPS: 69.73414375599921, Tokens per sec: 172808.89007745465, Loss: 2.5175516605377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6244 , TFLOPS: 69.71755850767181, Tokens per sec: 172767.79000508343, Loss: 2.5112648010253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6245 , TFLOPS: 70.52829382652266, Tokens per sec: 174776.88143506393, Loss: 2.516608238220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6246 , TFLOPS: 70.58180627904264, Tokens per sec: 174909.49118729171, Loss: 2.529463768005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6247 , TFLOPS: 68.69305149626399, Tokens per sec: 170228.94877205082, Loss: 2.522965431213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6248 , TFLOPS: 71.55263078006976, Tokens per sec: 177315.30124598424, Loss: 2.509812593460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6249 , TFLOPS: 70.38261254683911, Tokens per sec: 174415.86717589034, Loss: 2.5088610649108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6250 , TFLOPS: 68.7995383395107, Tokens per sec: 170492.83490011032, Loss: 2.501312017440796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6251 , TFLOPS: 70.69090680904398, Tokens per sec: 175179.85431905056, Loss: 2.510784149169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6252 , TFLOPS: 71.52734821529512, Tokens per sec: 177252.64826536828, Loss: 2.5182783603668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6253 , TFLOPS: 69.49736989834379, Tokens per sec: 172222.1383753909, Loss: 2.550114393234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6254 , TFLOPS: 69.47196224673002, Tokens per sec: 172159.1754158081, Loss: 2.5000405311584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6255 , TFLOPS: 71.54351750191911, Tokens per sec: 177292.7175388165, Loss: 2.5288429260253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6256 , TFLOPS: 71.51205787004679, Tokens per sec: 177214.75710547235, Loss: 2.5088768005371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6257 , TFLOPS: 68.91044259033981, Tokens per sec: 170767.66785077663, Loss: 2.530853033065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6258 , TFLOPS: 70.61203000183728, Tokens per sec: 174984.38890179474, Loss: 2.5166642665863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6259 , TFLOPS: 71.47488047911455, Tokens per sec: 177122.62743531456, Loss: 2.4945473670959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6260 , TFLOPS: 68.6000602405182, Tokens per sec: 169998.50619648024, Loss: 2.5131804943084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6261 , TFLOPS: 70.714054393514, Tokens per sec: 175237.2165832842, Loss: 2.4899582862854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6262 , TFLOPS: 71.48672336629761, Tokens per sec: 177151.97541435593, Loss: 2.499431610107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6263 , TFLOPS: 69.61204672272609, Tokens per sec: 172506.3201789053, Loss: 2.513558864593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6264 , TFLOPS: 68.6502156211951, Tokens per sec: 170122.7967548689, Loss: 2.5031073093414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6265 , TFLOPS: 71.50190655433569, Tokens per sec: 177189.6010268805, Loss: 2.550570249557495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6266 , TFLOPS: 70.58220589873788, Tokens per sec: 174910.48148891827, Loss: 2.5068254470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6267 , TFLOPS: 68.86674607356362, Tokens per sec: 170659.38306863653, Loss: 2.5039427280426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6268 , TFLOPS: 69.99582353730433, Tokens per sec: 173457.36140193633, Loss: 2.508509397506714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6269 , TFLOPS: 71.53807722969557, Tokens per sec: 177279.23594495116, Loss: 2.516423463821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6270 , TFLOPS: 69.75918924806625, Tokens per sec: 172870.95556578547, Loss: 2.5427660942077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6271 , TFLOPS: 69.60747789136003, Tokens per sec: 172494.9981114819, Loss: 2.521078586578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6272 , TFLOPS: 71.50412643247164, Tokens per sec: 177195.10212944116, Loss: 2.521040678024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6273 , TFLOPS: 70.59044021423124, Tokens per sec: 174930.88702979553, Loss: 2.5098915100097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6274 , TFLOPS: 69.48161283551393, Tokens per sec: 172183.09063791475, Loss: 2.52901554107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6275 , TFLOPS: 70.45611593883339, Tokens per sec: 174598.01667831923, Loss: 2.5197038650512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6276 , TFLOPS: 71.53505146997408, Tokens per sec: 177271.7377790463, Loss: 2.529642105102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6277 , TFLOPS: 68.74890932828498, Tokens per sec: 170367.370633047, Loss: 2.5347418785095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6278 , TFLOPS: 70.67905680049094, Tokens per sec: 175150.48869246992, Loss: 2.536529779434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6279 , TFLOPS: 70.45895710870738, Tokens per sec: 174605.05741024765, Loss: 2.5132014751434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6280 , TFLOPS: 68.63786134672901, Tokens per sec: 170092.18150180628, Loss: 2.533101797103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6281 , TFLOPS: 70.5363088873116, Tokens per sec: 174796.74363863075, Loss: 2.5136539936065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6282 , TFLOPS: 71.53679106153783, Tokens per sec: 177276.04867857308, Loss: 2.528512716293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6283 , TFLOPS: 69.66622184107182, Tokens per sec: 172640.57209004794, Loss: 2.5204148292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6284 , TFLOPS: 69.01915754376756, Tokens per sec: 171037.07548131133, Loss: 2.5051183700561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6285 , TFLOPS: 71.52290016679245, Tokens per sec: 177241.62551118567, Loss: 2.5118911266326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6286 , TFLOPS: 69.5212766806017, Tokens per sec: 172281.38201537545, Loss: 2.5229263305664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6287 , TFLOPS: 69.49943261179635, Tokens per sec: 172227.250006553, Loss: 2.5197179317474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6288 , TFLOPS: 70.87219304323979, Tokens per sec: 175629.10157771508, Loss: 2.496901035308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6289 , TFLOPS: 71.51905503140718, Tokens per sec: 177232.09684212433, Loss: 2.51706862449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6290 , TFLOPS: 68.45132283700556, Tokens per sec: 169629.91852579833, Loss: 2.5057833194732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6291 , TFLOPS: 69.70696998981663, Tokens per sec: 172741.55049141674, Loss: 2.518488645553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6292 , TFLOPS: 71.53011102721847, Tokens per sec: 177259.49481766322, Loss: 2.5228779315948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6293 , TFLOPS: 69.64216295004974, Tokens per sec: 172580.95150778958, Loss: 2.5153956413269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6294 , TFLOPS: 69.4437879871955, Tokens per sec: 172089.35649703068, Loss: 2.5314292907714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6295 , TFLOPS: 71.4927298375741, Tokens per sec: 177166.86011184647, Loss: 2.506701707839966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6296 , TFLOPS: 71.53988118506192, Tokens per sec: 177283.706345071, Loss: 2.5035691261291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6297 , TFLOPS: 67.78696332642825, Tokens per sec: 167983.56247334607, Loss: 2.5376968383789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6298 , TFLOPS: 70.66110304245133, Tokens per sec: 175105.99730227806, Loss: 2.522944927215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6299 , TFLOPS: 71.54210877754666, Tokens per sec: 177289.22656463776, Loss: 2.5061113834381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6300 , TFLOPS: 69.80917916997689, Tokens per sec: 172994.8360991245, Loss: 2.5096275806427 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6301 , TFLOPS: 68.91983760643312, Tokens per sec: 170790.9497356023, Loss: 2.5186729431152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6302 , TFLOPS: 71.51000263441857, Tokens per sec: 177209.6640051827, Loss: 2.5062663555145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6303 , TFLOPS: 70.52470418222242, Tokens per sec: 174767.98590105597, Loss: 2.518064498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6304 , TFLOPS: 69.5074147521198, Tokens per sec: 172247.03062957985, Loss: 2.5244646072387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6305 , TFLOPS: 69.91535481012552, Tokens per sec: 173257.95103162518, Loss: 2.5204358100891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6306 , TFLOPS: 71.39749702900127, Tokens per sec: 176930.86272144312, Loss: 2.5150465965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6307 , TFLOPS: 69.08635955504437, Tokens per sec: 171203.60946816794, Loss: 2.5094034671783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6308 , TFLOPS: 69.61847528749041, Tokens per sec: 172522.2508705562, Loss: 2.5219783782958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6309 , TFLOPS: 71.52140890153446, Tokens per sec: 177237.92999159996, Loss: 2.534479856491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6310 , TFLOPS: 69.65648057137109, Tokens per sec: 172616.43215064012, Loss: 2.515101909637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6311 , TFLOPS: 69.70326968308396, Tokens per sec: 172732.38072371075, Loss: 2.503105878829956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6312 , TFLOPS: 70.45126989447063, Tokens per sec: 174586.00764655243, Loss: 2.52004337310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6313 , TFLOPS: 70.71264846017488, Tokens per sec: 175233.73252559348, Loss: 2.5239768028259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6314 , TFLOPS: 69.23398838876024, Tokens per sec: 171569.44998077443, Loss: 2.514631748199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6315 , TFLOPS: 71.52336116403755, Tokens per sec: 177242.7679131421, Loss: 2.528822422027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6316 , TFLOPS: 70.53293567583506, Tokens per sec: 174788.38444900198, Loss: 2.505225896835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6317 , TFLOPS: 68.74289020984122, Tokens per sec: 170352.45459448273, Loss: 2.526632308959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6318 , TFLOPS: 70.65118430164785, Tokens per sec: 175081.41756992752, Loss: 2.5127387046813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6319 , TFLOPS: 71.56665866990424, Tokens per sec: 177350.06390229377, Loss: 2.519714593887329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6320 , TFLOPS: 69.71383102494124, Tokens per sec: 172758.55289226092, Loss: 2.5300018787384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6321 , TFLOPS: 69.10164184608246, Tokens per sec: 171241.48066884367, Loss: 2.5091633796691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6322 , TFLOPS: 71.57548071903211, Tokens per sec: 177371.92591186153, Loss: 2.518242597579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6323 , TFLOPS: 70.58204120997303, Tokens per sec: 174910.073372017, Loss: 2.527894973754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6324 , TFLOPS: 68.68700863045959, Tokens per sec: 170213.97388491087, Loss: 2.517615795135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6325 , TFLOPS: 70.57621268161691, Tokens per sec: 174895.62963668513, Loss: 2.520906448364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6326 , TFLOPS: 71.48624082422091, Tokens per sec: 177150.77962193344, Loss: 2.5239779949188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6327 , TFLOPS: 67.8895118248054, Tokens per sec: 168237.6889489743, Loss: 2.523386001586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6328 , TFLOPS: 70.61671016698385, Tokens per sec: 174995.9868665907, Loss: 2.5250303745269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6329 , TFLOPS: 71.52547521258401, Tokens per sec: 177248.00675832186, Loss: 2.5078301429748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6330 , TFLOPS: 69.6322275499868, Tokens per sec: 172556.330491959, Loss: 2.489403486251831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6331 , TFLOPS: 68.97403990634551, Tokens per sec: 170925.26900566145, Loss: 2.503885507583618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6332 , TFLOPS: 71.53068512215293, Tokens per sec: 177260.9174881526, Loss: 2.5244579315185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6333 , TFLOPS: 70.68053282850806, Tokens per sec: 175154.14645249426, Loss: 2.512800931930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6334 , TFLOPS: 69.52924137727776, Tokens per sec: 172301.11941112252, Loss: 2.514155149459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6335 , TFLOPS: 70.99516920112812, Tokens per sec: 175933.85004390447, Loss: 2.4995956420898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6336 , TFLOPS: 71.49628841879756, Tokens per sec: 177175.67866812833, Loss: 2.498377799987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6337 , TFLOPS: 69.63033867044716, Tokens per sec: 172551.6496403825, Loss: 2.5296549797058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6338 , TFLOPS: 68.8849723085642, Tokens per sec: 170704.5496867535, Loss: 2.5139145851135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6339 , TFLOPS: 71.52798811363695, Tokens per sec: 177254.23400394997, Loss: 2.530829429626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6340 , TFLOPS: 70.68729413779656, Tokens per sec: 175170.9017217309, Loss: 2.5308468341827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6341 , TFLOPS: 68.70075437811838, Tokens per sec: 170248.03736182844, Loss: 2.5127274990081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6342 , TFLOPS: 70.49706643541145, Tokens per sec: 174699.49651990924, Loss: 2.526552200317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6343 , TFLOPS: 71.50741974955749, Tokens per sec: 177203.2633319687, Loss: 2.5128188133239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6344 , TFLOPS: 68.61389455022739, Tokens per sec: 170032.78913991465, Loss: 2.512463092803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6345 , TFLOPS: 69.4630186278941, Tokens per sec: 172137.01214311176, Loss: 2.5244059562683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6346 , TFLOPS: 71.48941866349047, Tokens per sec: 177158.65465771133, Loss: 2.519832134246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6347 , TFLOPS: 68.83607895938582, Tokens per sec: 170583.38658144244, Loss: 2.5362954139709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6348 , TFLOPS: 70.73418008346898, Tokens per sec: 175287.09026002043, Loss: 2.5185210704803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6349 , TFLOPS: 70.5089173270299, Tokens per sec: 174728.86433482327, Loss: 2.5101852416992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6350 , TFLOPS: 70.48496208067772, Tokens per sec: 174669.5005954743, Loss: 2.5173287391662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6351 , TFLOPS: 68.63199035947919, Tokens per sec: 170077.63254865856, Loss: 2.5481228828430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6352 , TFLOPS: 71.53503494193653, Tokens per sec: 177271.69682074853, Loss: 2.4946115016937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6353 , TFLOPS: 70.56364945288122, Tokens per sec: 174864.49657192422, Loss: 2.4920308589935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6354 , TFLOPS: 69.37447091826444, Tokens per sec: 171917.58116431398, Loss: 2.511016607284546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6355 , TFLOPS: 70.65962979679065, Tokens per sec: 175102.3464372395, Loss: 2.522822141647339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6356 , TFLOPS: 71.5075897893943, Tokens per sec: 177203.68470941577, Loss: 2.5232155323028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6357 , TFLOPS: 68.52982393497338, Tokens per sec: 169824.4528941724, Loss: 2.505985736846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6358 , TFLOPS: 69.65531364483014, Tokens per sec: 172613.54037812407, Loss: 2.504849672317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6359 , TFLOPS: 71.49106709747207, Tokens per sec: 177162.7396587077, Loss: 2.533397674560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6360 , TFLOPS: 69.73787246514146, Tokens per sec: 172818.13022946037, Loss: 2.5239148139953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6361 , TFLOPS: 68.78479744409874, Tokens per sec: 170456.30533743577, Loss: 2.5133721828460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6362 , TFLOPS: 71.56174865330996, Tokens per sec: 177337.89634029017, Loss: 2.532630443572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6363 , TFLOPS: 71.53338795923096, Tokens per sec: 177267.61541617403, Loss: 2.5103018283843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6364 , TFLOPS: 68.57489939543076, Tokens per sec: 169936.15485065768, Loss: 2.5060300827026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6365 , TFLOPS: 70.6941798142806, Tokens per sec: 175187.96518657822, Loss: 2.517143964767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6366 , TFLOPS: 71.49250294831396, Tokens per sec: 177166.29785526593, Loss: 2.5280117988586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6367 , TFLOPS: 69.63955561275706, Tokens per sec: 172574.49023875644, Loss: 2.5280563831329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6368 , TFLOPS: 69.55358959423198, Tokens per sec: 172361.4569749126, Loss: 2.526289701461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6369 , TFLOPS: 71.53132782494306, Tokens per sec: 177262.51017646608, Loss: 2.5379536151885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6370 , TFLOPS: 70.56775650735871, Tokens per sec: 174874.6743053488, Loss: 2.5047597885131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6371 , TFLOPS: 67.82277120486091, Tokens per sec: 168072.29834066564, Loss: 2.511665105819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6372 , TFLOPS: 71.01169414653228, Tokens per sec: 175974.800678989, Loss: 2.5275821685791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6373 , TFLOPS: 71.54654600834958, Tokens per sec: 177300.22251136872, Loss: 2.513410806655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6374 , TFLOPS: 69.71101319759583, Tokens per sec: 172751.57000568972, Loss: 2.5107309818267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6375 , TFLOPS: 69.54135904226881, Tokens per sec: 172331.14831984133, Loss: 2.4966626167297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6376 , TFLOPS: 71.54639918111019, Tokens per sec: 177299.85865729515, Loss: 2.507227897644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6377 , TFLOPS: 69.53927506051063, Tokens per sec: 172325.98398347502, Loss: 2.5250704288482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6378 , TFLOPS: 69.7233168174751, Tokens per sec: 172782.05973110613, Loss: 2.5113983154296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6379 , TFLOPS: 70.43538766691213, Tokens per sec: 174546.64973709072, Loss: 2.540759801864624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6380 , TFLOPS: 70.67945079439487, Tokens per sec: 175151.46505276588, Loss: 2.4985618591308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6381 , TFLOPS: 68.69046581366027, Tokens per sec: 170222.54116572233, Loss: 2.524982452392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6382 , TFLOPS: 70.5654127811819, Tokens per sec: 174868.86629369998, Loss: 2.5208253860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6383 , TFLOPS: 71.51943248619465, Tokens per sec: 177233.03221666778, Loss: 2.524803876876831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6384 , TFLOPS: 68.64766301619446, Tokens per sec: 170116.47111848465, Loss: 2.497065544128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6385 , TFLOPS: 70.66607217732583, Tokens per sec: 175118.31136589247, Loss: 2.5110092163085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6386 , TFLOPS: 70.50909198522358, Tokens per sec: 174729.29715706734, Loss: 2.5361416339874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6387 , TFLOPS: 70.65572992487367, Tokens per sec: 175092.68212502287, Loss: 2.5208916664123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6388 , TFLOPS: 68.41181518954677, Tokens per sec: 169532.01422326884, Loss: 2.5168800354003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6389 , TFLOPS: 71.51052226866297, Tokens per sec: 177210.95171608278, Loss: 2.527462959289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6390 , TFLOPS: 70.51235920080282, Tokens per sec: 174737.39367718622, Loss: 2.521199941635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6391 , TFLOPS: 68.99721862991142, Tokens per sec: 170982.70843600493, Loss: 2.52561616897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6392 , TFLOPS: 70.69878113114436, Tokens per sec: 175199.36775664348, Loss: 2.5278797149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6393 , TFLOPS: 71.55507136526933, Tokens per sec: 177321.34928496144, Loss: 2.530681610107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6394 , TFLOPS: 68.485949072214, Tokens per sec: 169715.72615103953, Loss: 2.5391862392425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6395 , TFLOPS: 70.45051987679086, Tokens per sec: 174584.1490201207, Loss: 2.5345027446746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6396 , TFLOPS: 71.53622919253516, Tokens per sec: 177274.6563052895, Loss: 2.5223588943481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6397 , TFLOPS: 68.75412561217279, Tokens per sec: 170380.2971591421, Loss: 2.510776996612549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6398 , TFLOPS: 69.76321100253485, Tokens per sec: 172880.92191639147, Loss: 2.541630744934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6399 , TFLOPS: 71.54566306229763, Tokens per sec: 177298.03447378747, Loss: 2.508275270462036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6400 , TFLOPS: 70.66453952166226, Tokens per sec: 175114.51327632216, Loss: 2.5354325771331787 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/optimizer.pt +[rank0]:[2024-08-26 15:31:00,528] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006351411000650842, 'preprocessing_with_comm': 0.0016941330031841062, 'state_converting': 0.9572087679989636, : 0.9666145959999994}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400, took 4.27s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004400 + eval ppl=10.34994888305664, eval loss=2.3369815349578857 +------------------------------------------------------------------ +iteration: 6401 , TFLOPS: 63.609649577336974, Tokens per sec: 157631.71883400044, Loss: 2.516583204269409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6402 , TFLOPS: 71.52533930389757, Tokens per sec: 177247.6699616253, Loss: 2.499060869216919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6403 , TFLOPS: 71.48511461901266, Tokens per sec: 177147.98876136565, Loss: 2.5019617080688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6404 , TFLOPS: 67.9185286748321, Tokens per sec: 168309.59589981037, Loss: 2.5161080360412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6405 , TFLOPS: 70.69310715092574, Tokens per sec: 175185.30700862125, Loss: 2.5400896072387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6406 , TFLOPS: 71.51274386992999, Tokens per sec: 177216.45708875204, Loss: 2.515645742416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6407 , TFLOPS: 70.67599005162253, Tokens per sec: 175142.88895094377, Loss: 2.54095458984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6408 , TFLOPS: 68.97977718778098, Tokens per sec: 170939.48662107228, Loss: 2.5049657821655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6409 , TFLOPS: 70.45034265880446, Tokens per sec: 174583.70985442825, Loss: 2.508439302444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6410 , TFLOPS: 70.69098489583719, Tokens per sec: 175180.04782672614, Loss: 2.5100114345550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6411 , TFLOPS: 68.72064155169132, Tokens per sec: 170297.3199686952, Loss: 2.4939122200012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6412 , TFLOPS: 69.70776560511257, Tokens per sec: 172743.52211376434, Loss: 2.511803388595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6413 , TFLOPS: 71.51294142608585, Tokens per sec: 177216.94665466886, Loss: 2.5220353603363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6414 , TFLOPS: 68.89050091497838, Tokens per sec: 170718.2501824741, Loss: 2.5337777137756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6415 , TFLOPS: 70.66319091844703, Tokens per sec: 175111.1712889939, Loss: 2.515895128250122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6416 , TFLOPS: 70.43080745302231, Tokens per sec: 174535.29946251953, Loss: 2.52007794380188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6417 , TFLOPS: 68.5197156684671, Tokens per sec: 169799.4034962524, Loss: 2.50191330909729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6418 , TFLOPS: 69.08486090541525, Tokens per sec: 171199.8956492971, Loss: 2.5316762924194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6419 , TFLOPS: 70.43409757095957, Tokens per sec: 174543.4527372044, Loss: 2.5366296768188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6420 , TFLOPS: 70.605642074214, Tokens per sec: 174968.55891345599, Loss: 2.528134346008301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6421 , TFLOPS: 68.7623974104133, Tokens per sec: 170400.79558639583, Loss: 2.498725652694702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6422 , TFLOPS: 71.49379486169836, Tokens per sec: 177169.4993589485, Loss: 2.500749111175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6423 , TFLOPS: 70.54154120072711, Tokens per sec: 174809.70988766127, Loss: 2.5015361309051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6424 , TFLOPS: 68.84236698490604, Tokens per sec: 170598.9690013647, Loss: 2.5163936614990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6425 , TFLOPS: 69.97695064129536, Tokens per sec: 173410.59228660498, Loss: 2.5184874534606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6426 , TFLOPS: 71.54846695206817, Tokens per sec: 177304.98282151282, Loss: 2.5239081382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6427 , TFLOPS: 69.72256680263362, Tokens per sec: 172780.20111170804, Loss: 2.5213050842285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6428 , TFLOPS: 68.90089516207772, Tokens per sec: 170744.0082718071, Loss: 2.504303455352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6429 , TFLOPS: 71.56235097874274, Tokens per sec: 177339.3889690642, Loss: 2.4946396350860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6430 , TFLOPS: 70.4903492185407, Tokens per sec: 174682.85051654102, Loss: 2.502887010574341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6431 , TFLOPS: 68.27835584654764, Tokens per sec: 169201.2872695567, Loss: 2.505256175994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6432 , TFLOPS: 70.57853264362706, Tokens per sec: 174901.3787581112, Loss: 2.494903087615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6433 , TFLOPS: 71.49496538506789, Tokens per sec: 177172.40004480197, Loss: 2.5119290351867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6434 , TFLOPS: 68.68368215303585, Tokens per sec: 170205.7305074131, Loss: 2.499048948287964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6435 , TFLOPS: 70.67499181221663, Tokens per sec: 175140.41520373078, Loss: 2.501681327819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6436 , TFLOPS: 71.54103027790454, Tokens per sec: 177286.55392371802, Loss: 2.512627363204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6437 , TFLOPS: 69.79417722169403, Tokens per sec: 172957.6596186786, Loss: 2.511507034301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6438 , TFLOPS: 68.04202915692086, Tokens per sec: 168615.64369912754, Loss: 2.530245065689087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6439 , TFLOPS: 71.55029485121185, Tokens per sec: 177309.51255695056, Loss: 2.514620542526245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6440 , TFLOPS: 70.61474222037985, Tokens per sec: 174991.11007811877, Loss: 2.5129904747009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6441 , TFLOPS: 69.56564089557423, Tokens per sec: 172391.32142719915, Loss: 2.505699634552002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6442 , TFLOPS: 69.93276312285924, Tokens per sec: 173301.0907482633, Loss: 2.5000152587890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6443 , TFLOPS: 71.5357551546656, Tokens per sec: 177273.48158722915, Loss: 2.5115890502929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6444 , TFLOPS: 68.83364050652247, Tokens per sec: 170577.3438266293, Loss: 2.527292251586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6445 , TFLOPS: 68.9075946525149, Tokens per sec: 170760.61034712024, Loss: 2.514573812484741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6446 , TFLOPS: 71.56891262008321, Tokens per sec: 177355.64943913036, Loss: 2.4982047080993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6447 , TFLOPS: 70.59617480250711, Tokens per sec: 174945.09797126017, Loss: 2.5266942977905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6448 , TFLOPS: 68.52451301837179, Tokens per sec: 169811.29185778793, Loss: 2.516082286834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6449 , TFLOPS: 70.44020871529919, Tokens per sec: 174558.59682607616, Loss: 2.510319232940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6450 , TFLOPS: 71.53436246134972, Tokens per sec: 177270.03033977456, Loss: 2.529648780822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6451 , TFLOPS: 68.15570061507059, Tokens per sec: 168897.33409436737, Loss: 2.534546375274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6452 , TFLOPS: 70.598110441045, Tokens per sec: 174949.89469678528, Loss: 2.486588716506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6453 , TFLOPS: 70.49338997532827, Tokens per sec: 174690.3858468276, Loss: 2.5040223598480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6454 , TFLOPS: 69.68381118049014, Tokens per sec: 172684.16041075226, Loss: 2.5239739418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6455 , TFLOPS: 69.9340092010838, Tokens per sec: 173304.17866737058, Loss: 2.533632516860962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6456 , TFLOPS: 71.56398776483069, Tokens per sec: 177343.445105297, Loss: 2.5239551067352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6457 , TFLOPS: 69.5966931100256, Tokens per sec: 172468.2722352934, Loss: 2.5206449031829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6458 , TFLOPS: 69.08515171268374, Tokens per sec: 171200.6163017435, Loss: 2.52707839012146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6459 , TFLOPS: 70.85540519981895, Tokens per sec: 175587.49945238463, Loss: 2.511164426803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6460 , TFLOPS: 69.54061766829497, Tokens per sec: 172329.31111346508, Loss: 2.517712354660034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6461 , TFLOPS: 69.58315666762658, Tokens per sec: 172434.7274398664, Loss: 2.4979867935180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6462 , TFLOPS: 70.68370084158333, Tokens per sec: 175161.99713788021, Loss: 2.5000851154327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6463 , TFLOPS: 71.51289120622602, Tokens per sec: 177216.822204324, Loss: 2.5029706954956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6464 , TFLOPS: 68.00652752246694, Tokens per sec: 168527.66673812282, Loss: 2.5227603912353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6465 , TFLOPS: 69.10322720680469, Tokens per sec: 171245.40936735526, Loss: 2.5065078735351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6466 , TFLOPS: 71.53559902505766, Tokens per sec: 177273.0946808616, Loss: 2.5387799739837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6467 , TFLOPS: 69.72316001480604, Tokens per sec: 172781.67115681898, Loss: 2.528245210647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6468 , TFLOPS: 68.65400635838753, Tokens per sec: 170132.19061921592, Loss: 2.5263891220092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6469 , TFLOPS: 71.49622991897314, Tokens per sec: 177175.5336991191, Loss: 2.519185781478882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6470 , TFLOPS: 71.55820183262038, Tokens per sec: 177329.1069279068, Loss: 2.5167624950408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6471 , TFLOPS: 67.03607009875883, Tokens per sec: 166122.76633746366, Loss: 2.5322518348693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6472 , TFLOPS: 70.59348544288436, Tokens per sec: 174938.4334418576, Loss: 2.5433311462402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6473 , TFLOPS: 71.52434777083711, Tokens per sec: 177245.2128334751, Loss: 2.527108669281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6474 , TFLOPS: 70.72882243599405, Tokens per sec: 175273.81341938395, Loss: 2.51741886138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6475 , TFLOPS: 67.65571302500393, Tokens per sec: 167658.30976800233, Loss: 2.5060691833496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6476 , TFLOPS: 71.50313876101096, Tokens per sec: 177192.65457076058, Loss: 2.526019334793091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6477 , TFLOPS: 70.58775421835165, Tokens per sec: 174924.23083611904, Loss: 2.523512125015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6478 , TFLOPS: 68.8557226688642, Tokens per sec: 170632.0658574631, Loss: 2.5221986770629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6479 , TFLOPS: 69.60430363686001, Tokens per sec: 172487.13195915785, Loss: 2.501016139984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6480 , TFLOPS: 70.71547983998798, Tokens per sec: 175240.74899667263, Loss: 2.4842300415039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6481 , TFLOPS: 69.51500738962945, Tokens per sec: 172265.84602172681, Loss: 2.5296072959899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6482 , TFLOPS: 69.65675613135178, Tokens per sec: 172617.1150186277, Loss: 2.494194984436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6483 , TFLOPS: 71.5047733610814, Tokens per sec: 177196.7052898012, Loss: 2.503298759460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6484 , TFLOPS: 69.82111813853753, Tokens per sec: 173024.42217840377, Loss: 2.503525972366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6485 , TFLOPS: 68.94461250403708, Tokens per sec: 170852.3446610452, Loss: 2.5217061042785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6486 , TFLOPS: 70.43335936907916, Tokens per sec: 174541.62339162512, Loss: 2.531581163406372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6487 , TFLOPS: 70.66251702758915, Tokens per sec: 175109.50131321288, Loss: 2.523981809616089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6488 , TFLOPS: 69.57515578766296, Tokens per sec: 172414.9003779469, Loss: 2.519256353378296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6489 , TFLOPS: 70.9442374516467, Tokens per sec: 175807.63555809416, Loss: 2.5213794708251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6490 , TFLOPS: 70.62082299539988, Tokens per sec: 175006.17890847108, Loss: 2.5175793170928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6491 , TFLOPS: 68.84824749335282, Tokens per sec: 170613.54154908733, Loss: 2.523348093032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6492 , TFLOPS: 70.03818023566069, Tokens per sec: 173562.32596643892, Loss: 2.5229310989379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6493 , TFLOPS: 71.52377772953116, Tokens per sec: 177243.80020832372, Loss: 2.50937819480896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6494 , TFLOPS: 69.6645112040903, Tokens per sec: 172636.33294316556, Loss: 2.5050954818725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6495 , TFLOPS: 68.63686949556983, Tokens per sec: 170089.72358537235, Loss: 2.4974827766418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6496 , TFLOPS: 71.50212746785874, Tokens per sec: 177190.14847492627, Loss: 2.521982192993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6497 , TFLOPS: 69.53691724645671, Tokens per sec: 172320.14106051475, Loss: 2.523329257965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6498 , TFLOPS: 68.9037793399621, Tokens per sec: 170751.155582323, Loss: 2.5267586708068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6499 , TFLOPS: 70.61972164145585, Tokens per sec: 175003.44963207364, Loss: 2.5251264572143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6500 , TFLOPS: 71.55772833535029, Tokens per sec: 177327.93354951148, Loss: 2.52095365524292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6501 , TFLOPS: 67.96999358232135, Tokens per sec: 168437.1316098963, Loss: 2.5309693813323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6502 , TFLOPS: 70.57207383487844, Tokens per sec: 174885.37311853562, Loss: 2.5244648456573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6503 , TFLOPS: 71.5171464287064, Tokens per sec: 177227.3671143819, Loss: 2.52347469329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6504 , TFLOPS: 69.6856288784481, Tokens per sec: 172688.66486652804, Loss: 2.5169100761413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6505 , TFLOPS: 68.31603278980396, Tokens per sec: 169294.65488540375, Loss: 2.5138349533081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6506 , TFLOPS: 71.44137811401438, Tokens per sec: 177039.60488400617, Loss: 2.5238473415374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6507 , TFLOPS: 70.70369322037271, Tokens per sec: 175211.54045486267, Loss: 2.5234148502349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6508 , TFLOPS: 68.13903584870754, Tokens per sec: 168856.03696754418, Loss: 2.521733283996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6509 , TFLOPS: 70.69441335104595, Tokens per sec: 175188.54391640896, Loss: 2.510244607925415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6510 , TFLOPS: 71.5103875565536, Tokens per sec: 177210.61788463598, Loss: 2.5270509719848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6511 , TFLOPS: 68.92010429157453, Tokens per sec: 170791.6106107605, Loss: 2.5008440017700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6512 , TFLOPS: 68.591176404028, Tokens per sec: 169976.49107102217, Loss: 2.5269315242767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6513 , TFLOPS: 71.51821584190283, Tokens per sec: 177230.0172380882, Loss: 2.541332721710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6514 , TFLOPS: 70.68921198287298, Tokens per sec: 175175.65435309833, Loss: 2.5257320404052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6515 , TFLOPS: 68.66069859575734, Tokens per sec: 170148.77472062912, Loss: 2.5008153915405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6516 , TFLOPS: 70.4543819109742, Tokens per sec: 174593.7195662652, Loss: 2.5193395614624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6517 , TFLOPS: 71.49956543890708, Tokens per sec: 177183.79948495326, Loss: 2.4866833686828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6518 , TFLOPS: 68.27129316698094, Tokens per sec: 169183.785171572, Loss: 2.543519973754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6519 , TFLOPS: 69.55999200325833, Tokens per sec: 172377.3228497635, Loss: 2.541700601577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6520 , TFLOPS: 71.50140713063345, Tokens per sec: 177188.36339993027, Loss: 2.511070728302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6521 , TFLOPS: 69.753436287938, Tokens per sec: 172856.6990968468, Loss: 2.5102779865264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6522 , TFLOPS: 69.92703934345151, Tokens per sec: 173286.90659236457, Loss: 2.5101513862609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6523 , TFLOPS: 70.51933115121443, Tokens per sec: 174754.67093832992, Loss: 2.5050857067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6524 , TFLOPS: 70.5676821820849, Tokens per sec: 174874.4901191327, Loss: 2.5022382736206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6525 , TFLOPS: 68.87402557036046, Tokens per sec: 170677.4224636019, Loss: 2.519834041595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6526 , TFLOPS: 70.89037573270875, Tokens per sec: 175674.16028520773, Loss: 2.5166385173797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6527 , TFLOPS: 69.66144926987447, Tokens per sec: 172628.7451328788, Loss: 2.486325979232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6528 , TFLOPS: 69.5332777735481, Tokens per sec: 172311.12204572617, Loss: 2.5294723510742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6529 , TFLOPS: 70.72162163189083, Tokens per sec: 175255.96903358214, Loss: 2.5237903594970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6530 , TFLOPS: 71.53802428729455, Tokens per sec: 177279.10474784928, Loss: 2.5224180221557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6531 , TFLOPS: 67.9341343660729, Tokens per sec: 168348.26852181385, Loss: 2.51492977142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6532 , TFLOPS: 69.64805068035277, Tokens per sec: 172595.54195206755, Loss: 2.5060439109802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6533 , TFLOPS: 71.4898785052956, Tokens per sec: 177159.7941963602, Loss: 2.5253000259399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6534 , TFLOPS: 69.67075426980652, Tokens per sec: 172651.8039477408, Loss: 2.508363962173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6535 , TFLOPS: 68.7757972533094, Tokens per sec: 170434.00187320696, Loss: 2.509793519973755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6536 , TFLOPS: 71.48175196672402, Tokens per sec: 177139.65573856767, Loss: 2.5008764266967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6537 , TFLOPS: 71.4813091284302, Tokens per sec: 177138.55833649292, Loss: 2.520555019378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6538 , TFLOPS: 67.7420785598543, Tokens per sec: 167872.3330773106, Loss: 2.5191783905029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6539 , TFLOPS: 70.59319032647409, Tokens per sec: 174937.7021108832, Loss: 2.5064730644226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6540 , TFLOPS: 71.51495112871211, Tokens per sec: 177221.9269191638, Loss: 2.511608839035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6541 , TFLOPS: 69.63631518756901, Tokens per sec: 172566.46010817814, Loss: 2.513864040374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6542 , TFLOPS: 69.06972400787234, Tokens per sec: 171162.38474971883, Loss: 2.5116021633148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6543 , TFLOPS: 71.53733653608658, Tokens per sec: 177277.4004245939, Loss: 2.511897325515747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6544 , TFLOPS: 70.59203249408071, Tokens per sec: 174934.83287466664, Loss: 2.5289227962493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6545 , TFLOPS: 67.94512605728076, Tokens per sec: 168375.50714346173, Loss: 2.506895065307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6546 , TFLOPS: 70.60340768547712, Tokens per sec: 174963.0218520277, Loss: 2.514004945755005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6547 , TFLOPS: 70.68690049997154, Tokens per sec: 175169.9262438377, Loss: 2.505403995513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6548 , TFLOPS: 69.66868090693555, Tokens per sec: 172646.66592614676, Loss: 2.5167858600616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6549 , TFLOPS: 70.39019512914716, Tokens per sec: 174434.65765014009, Loss: 2.502790689468384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6550 , TFLOPS: 71.53163221928023, Tokens per sec: 177263.26449916535, Loss: 2.5290164947509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6551 , TFLOPS: 68.81741775001818, Tokens per sec: 170537.1420489275, Loss: 2.5130481719970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6552 , TFLOPS: 69.77043166757666, Tokens per sec: 172898.81551978702, Loss: 2.5148324966430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6553 , TFLOPS: 70.49841417785977, Tokens per sec: 174702.8363741623, Loss: 2.532958745956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6554 , TFLOPS: 70.68340497040083, Tokens per sec: 175161.26393649713, Loss: 2.525080680847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6555 , TFLOPS: 68.70548651739568, Tokens per sec: 170259.7641242458, Loss: 2.528183937072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6556 , TFLOPS: 70.44088221523467, Tokens per sec: 174560.26583310837, Loss: 2.513857841491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6557 , TFLOPS: 71.51251566544012, Tokens per sec: 177215.89157288734, Loss: 2.5041720867156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6558 , TFLOPS: 68.8295557681544, Tokens per sec: 170567.22139498216, Loss: 2.517634630203247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6559 , TFLOPS: 70.13465397466759, Tokens per sec: 173801.39851915766, Loss: 2.5316522121429443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6560 , TFLOPS: 70.57687819101466, Tokens per sec: 174897.27884228466, Loss: 2.4985735416412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6561 , TFLOPS: 70.7530342053978, Tokens per sec: 175333.81285111303, Loss: 2.517585277557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6562 , TFLOPS: 68.092682898644, Tokens per sec: 168741.16925108773, Loss: 2.506406307220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6563 , TFLOPS: 71.50459899852717, Tokens per sec: 177196.2732001842, Loss: 2.4977307319641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6564 , TFLOPS: 69.57308625484669, Tokens per sec: 172409.7718476503, Loss: 2.5181758403778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6565 , TFLOPS: 68.85686400587994, Tokens per sec: 170634.89421631675, Loss: 2.5112597942352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6566 , TFLOPS: 70.7215241301794, Tokens per sec: 175255.7274136005, Loss: 2.5162436962127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6567 , TFLOPS: 71.52952410768448, Tokens per sec: 177258.04036640347, Loss: 2.5111937522888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6568 , TFLOPS: 68.1872786274389, Tokens per sec: 168975.58788762128, Loss: 2.515155076980591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6569 , TFLOPS: 70.56647484789083, Tokens per sec: 174871.49821200097, Loss: 2.514103889465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6570 , TFLOPS: 71.4720145152836, Tokens per sec: 177115.5252612295, Loss: 2.5157878398895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6571 , TFLOPS: 68.88717733998311, Tokens per sec: 170710.01399751383, Loss: 2.5120770931243896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6572 , TFLOPS: 69.04254160113312, Tokens per sec: 171095.0237514298, Loss: 2.5120463371276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6573 , TFLOPS: 71.54983635924376, Tokens per sec: 177308.37636334656, Loss: 2.514927387237549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6574 , TFLOPS: 70.65635734022815, Tokens per sec: 175094.23692938677, Loss: 2.49757719039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6575 , TFLOPS: 68.46026610483224, Tokens per sec: 169652.08092865514, Loss: 2.5456042289733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6576 , TFLOPS: 70.81853264041561, Tokens per sec: 175496.1251883354, Loss: 2.512497663497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6577 , TFLOPS: 71.54103920071782, Tokens per sec: 177286.57603543226, Loss: 2.499208927154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6578 , TFLOPS: 67.9742494994867, Tokens per sec: 168447.67824146003, Loss: 2.518500328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6579 , TFLOPS: 69.94074387983183, Tokens per sec: 173320.86794319432, Loss: 2.5319056510925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6580 , TFLOPS: 71.53127489246116, Tokens per sec: 177262.37900394492, Loss: 2.516178607940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6581 , TFLOPS: 70.69494926422145, Tokens per sec: 175189.8719682935, Loss: 2.5053141117095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6582 , TFLOPS: 67.76802599394341, Tokens per sec: 167936.63367732926, Loss: 2.498924970626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6583 , TFLOPS: 71.52917473597503, Tokens per sec: 177257.17458482095, Loss: 2.515019416809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6584 , TFLOPS: 71.49381704737347, Tokens per sec: 177169.55433749536, Loss: 2.5028932094573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6585 , TFLOPS: 68.26455965997273, Tokens per sec: 169167.09879944863, Loss: 2.5145368576049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6586 , TFLOPS: 70.5102126632835, Tokens per sec: 174732.07432075273, Loss: 2.5089330673217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6587 , TFLOPS: 71.48878316190921, Tokens per sec: 177157.07981478664, Loss: 2.5319721698760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6588 , TFLOPS: 69.81798946354802, Tokens per sec: 173016.66897712776, Loss: 2.5210816860198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6589 , TFLOPS: 69.85334541050672, Tokens per sec: 173104.2849084705, Loss: 2.5078787803649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6590 , TFLOPS: 70.48897275693801, Tokens per sec: 174679.43949305976, Loss: 2.4965016841888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6591 , TFLOPS: 70.7050877403633, Tokens per sec: 175214.99622901838, Loss: 2.512775421142578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6592 , TFLOPS: 69.09119537710181, Tokens per sec: 171215.59316793605, Loss: 2.5368235111236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6593 , TFLOPS: 69.843623851437, Tokens per sec: 173080.1938141776, Loss: 2.4951260089874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6594 , TFLOPS: 70.59639073843202, Tokens per sec: 174945.63308426947, Loss: 2.545983076095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6595 , TFLOPS: 69.30696361261319, Tokens per sec: 171750.29062436655, Loss: 2.524280071258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6596 , TFLOPS: 70.74736723565184, Tokens per sec: 175319.76947581428, Loss: 2.5032410621643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6597 , TFLOPS: 70.41980524368965, Tokens per sec: 174508.03477579975, Loss: 2.509633779525757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6598 , TFLOPS: 68.7944971088914, Tokens per sec: 170480.34217529828, Loss: 2.52490496635437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6599 , TFLOPS: 69.79387854692138, Tokens per sec: 172956.91946969024, Loss: 2.5424718856811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6600 , TFLOPS: 71.51194100244876, Tokens per sec: 177214.4674946904, Loss: 2.5255141258239746 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/optimizer.pt +[rank0]:[2024-08-26 15:56:18,806] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006414302006305661, 'preprocessing_with_comm': 0.001261872996110469, 'state_converting': 0.957224320001842, : 0.9662589479994494}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600, took 4.25s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004600 + eval ppl=10.5043306350708, eval loss=2.351787567138672 +------------------------------------------------------------------ +iteration: 6601 , TFLOPS: 65.30248730031481, Tokens per sec: 161826.75719300914, Loss: 2.498263120651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6602 , TFLOPS: 67.94769639284867, Tokens per sec: 168381.87671814483, Loss: 2.5021181106567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6603 , TFLOPS: 71.45148453524483, Tokens per sec: 177064.64970912941, Loss: 2.506531238555908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6604 , TFLOPS: 70.16117110702754, Tokens per sec: 173867.1108942489, Loss: 2.512019157409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6605 , TFLOPS: 69.03779690398657, Tokens per sec: 171083.2658692291, Loss: 2.5266380310058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6606 , TFLOPS: 69.95991205112755, Tokens per sec: 173368.3687832135, Loss: 2.5303189754486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6607 , TFLOPS: 71.52025896064804, Tokens per sec: 177235.08031140696, Loss: 2.5152199268341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6608 , TFLOPS: 70.34049593198301, Tokens per sec: 174311.4975647488, Loss: 2.497879981994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6609 , TFLOPS: 68.66750594036274, Tokens per sec: 170165.64407045222, Loss: 2.496166229248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6610 , TFLOPS: 71.53364803710475, Tokens per sec: 177268.25991779522, Loss: 2.522453546524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6611 , TFLOPS: 70.62179777486632, Tokens per sec: 175008.59451936977, Loss: 2.4953932762145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6612 , TFLOPS: 68.54061500162416, Tokens per sec: 169851.19434606718, Loss: 2.508882522583008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6613 , TFLOPS: 70.79802931048347, Tokens per sec: 175445.31567813532, Loss: 2.528661012649536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6614 , TFLOPS: 70.61532528120908, Tokens per sec: 174992.5549670831, Loss: 2.495673894882202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6615 , TFLOPS: 69.03576630848998, Tokens per sec: 171078.23382989998, Loss: 2.500447988510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6616 , TFLOPS: 70.68749872096211, Tokens per sec: 175171.4087013518, Loss: 2.5086445808410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6617 , TFLOPS: 71.53072300634967, Tokens per sec: 177261.0113693654, Loss: 2.503227472305298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6618 , TFLOPS: 69.87692622147568, Tokens per sec: 173162.72075569237, Loss: 2.5028233528137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6619 , TFLOPS: 67.59564881639743, Tokens per sec: 167509.46404245662, Loss: 2.506721019744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6620 , TFLOPS: 71.50652697150106, Tokens per sec: 177201.05092959726, Loss: 2.4970297813415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6621 , TFLOPS: 70.59783583655249, Tokens per sec: 174949.21419660264, Loss: 2.5097696781158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6622 , TFLOPS: 68.64707082063634, Tokens per sec: 170115.0035926559, Loss: 2.509364604949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6623 , TFLOPS: 70.36904533012074, Tokens per sec: 174382.2461183099, Loss: 2.5015993118286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6624 , TFLOPS: 71.48965712677372, Tokens per sec: 177159.2455959961, Loss: 2.508405923843384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6625 , TFLOPS: 68.80720016529492, Tokens per sec: 170511.82174842336, Loss: 2.490433931350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6626 , TFLOPS: 69.62586478910934, Tokens per sec: 172540.56286958867, Loss: 2.512298107147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6627 , TFLOPS: 70.52231284214803, Tokens per sec: 174762.05989692378, Loss: 2.5068564414978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6628 , TFLOPS: 70.61883593353137, Tokens per sec: 175001.25475026798, Loss: 2.5113272666931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6629 , TFLOPS: 68.70655885146226, Tokens per sec: 170262.4214861901, Loss: 2.5032336711883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6630 , TFLOPS: 70.51543983779372, Tokens per sec: 174745.02783500988, Loss: 2.516641616821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6631 , TFLOPS: 70.64838502087174, Tokens per sec: 175074.4806437981, Loss: 2.4823405742645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6632 , TFLOPS: 68.69605948962607, Tokens per sec: 170236.40291095982, Loss: 2.524183511734009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6633 , TFLOPS: 70.72201098973615, Tokens per sec: 175256.93390521413, Loss: 2.5235371589660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6634 , TFLOPS: 69.50884190241152, Tokens per sec: 172250.5672652163, Loss: 2.534008264541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6635 , TFLOPS: 69.93002472979984, Tokens per sec: 173294.3047086032, Loss: 2.542335271835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6636 , TFLOPS: 69.74611851159534, Tokens per sec: 172838.5648409501, Loss: 2.511598587036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6637 , TFLOPS: 71.49367781242476, Tokens per sec: 177169.20929795448, Loss: 2.4942407608032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6638 , TFLOPS: 69.54164383118439, Tokens per sec: 172331.85405814621, Loss: 2.505772113800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6639 , TFLOPS: 68.34499337708883, Tokens per sec: 169366.4224109677, Loss: 2.5181543827056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6640 , TFLOPS: 71.51579472400475, Tokens per sec: 177224.01744122864, Loss: 2.5365309715270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6641 , TFLOPS: 69.15602172800737, Tokens per sec: 171376.24000669213, Loss: 2.4998507499694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6642 , TFLOPS: 69.54349085076096, Tokens per sec: 172336.43117612298, Loss: 2.5066540241241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6643 , TFLOPS: 70.5647132535152, Tokens per sec: 174867.13278708252, Loss: 2.500406503677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6644 , TFLOPS: 71.48873363238796, Tokens per sec: 177156.9570751769, Loss: 2.528057336807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6645 , TFLOPS: 67.83003668622483, Tokens per sec: 168090.3030038443, Loss: 2.5166854858398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6646 , TFLOPS: 69.76026516539177, Tokens per sec: 172873.6218074402, Loss: 2.5086138248443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6647 , TFLOPS: 71.48903513514635, Tokens per sec: 177157.70423222546, Loss: 2.5142996311187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6648 , TFLOPS: 70.68473394231194, Tokens per sec: 175164.55727529115, Loss: 2.508012056350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6649 , TFLOPS: 68.0543817767816, Tokens per sec: 168646.25485189582, Loss: 2.5099234580993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6650 , TFLOPS: 71.53768639341067, Tokens per sec: 177278.26740958297, Loss: 2.504788398742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6651 , TFLOPS: 70.58331800531859, Tokens per sec: 174913.23741153392, Loss: 2.5141234397888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6652 , TFLOPS: 67.77496648032258, Tokens per sec: 167953.8329671345, Loss: 2.5160627365112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6653 , TFLOPS: 70.10093075545487, Tokens per sec: 173717.82866702927, Loss: 2.521517038345337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6654 , TFLOPS: 71.52971334881981, Tokens per sec: 177258.50932678345, Loss: 2.511157751083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6655 , TFLOPS: 70.9543069192158, Tokens per sec: 175832.58880797378, Loss: 2.542156457901001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6656 , TFLOPS: 67.36338500824212, Tokens per sec: 166933.88873987712, Loss: 2.5021584033966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6657 , TFLOPS: 71.51227262572762, Tokens per sec: 177215.2892937067, Loss: 2.5259110927581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6658 , TFLOPS: 70.55412389932752, Tokens per sec: 174840.8912008918, Loss: 2.526547431945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6659 , TFLOPS: 67.83896071950578, Tokens per sec: 168112.41774137787, Loss: 2.5068793296813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6660 , TFLOPS: 70.56220572583672, Tokens per sec: 174860.91885726826, Loss: 2.5229499340057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6661 , TFLOPS: 70.71871372799737, Tokens per sec: 175248.76292740204, Loss: 2.5018656253814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6662 , TFLOPS: 69.53651139274393, Tokens per sec: 172319.13531030598, Loss: 2.5082714557647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6663 , TFLOPS: 70.68548801146724, Tokens per sec: 175166.42594172558, Loss: 2.507955312728882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6664 , TFLOPS: 70.4589427715123, Tokens per sec: 174605.02188109886, Loss: 2.4971630573272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6665 , TFLOPS: 68.55562315870934, Tokens per sec: 169888.38621260924, Loss: 2.520385980606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6666 , TFLOPS: 69.65141342666816, Tokens per sec: 172603.8752078742, Loss: 2.512605905532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6667 , TFLOPS: 70.53512270809404, Tokens per sec: 174793.8041558612, Loss: 2.498943328857422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6668 , TFLOPS: 70.74402586512971, Tokens per sec: 175311.48919157812, Loss: 2.4931352138519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6669 , TFLOPS: 68.74662006112035, Tokens per sec: 170361.69757682944, Loss: 2.519649028778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6670 , TFLOPS: 71.51473792185851, Tokens per sec: 177221.39856909387, Loss: 2.505753517150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6671 , TFLOPS: 70.47117744795561, Tokens per sec: 174635.34075708626, Loss: 2.5101566314697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6672 , TFLOPS: 68.82255772694329, Tokens per sec: 170549.87947796308, Loss: 2.5107221603393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6673 , TFLOPS: 69.95778841372669, Tokens per sec: 173363.1061757962, Loss: 2.5146164894104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6674 , TFLOPS: 71.51154785331518, Tokens per sec: 177213.4932278284, Loss: 2.5125105381011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6675 , TFLOPS: 69.76164064283894, Tokens per sec: 172877.03039207563, Loss: 2.5100414752960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6676 , TFLOPS: 68.84332097397707, Tokens per sec: 170601.33309137332, Loss: 2.530972957611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6677 , TFLOPS: 71.53616322334052, Tokens per sec: 177274.4928263581, Loss: 2.533393144607544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6678 , TFLOPS: 70.56395639822242, Tokens per sec: 174865.2572162925, Loss: 2.521986961364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6679 , TFLOPS: 67.71410489055961, Tokens per sec: 167803.01124324428, Loss: 2.5139198303222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6680 , TFLOPS: 71.03807799700988, Tokens per sec: 176040.18276689356, Loss: 2.515185832977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6681 , TFLOPS: 70.71711396187253, Tokens per sec: 175244.79853071593, Loss: 2.527195930480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6682 , TFLOPS: 68.64312277936615, Tokens per sec: 170105.21991147572, Loss: 2.4923019409179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6683 , TFLOPS: 70.5891979934651, Tokens per sec: 174927.80866989514, Loss: 2.5206730365753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6684 , TFLOPS: 71.47003471203615, Tokens per sec: 177110.61909068897, Loss: 2.5123679637908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6685 , TFLOPS: 69.6798287191794, Tokens per sec: 172674.2914329199, Loss: 2.5265889167785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6686 , TFLOPS: 68.14764152350291, Tokens per sec: 168877.36277768045, Loss: 2.512986421585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6687 , TFLOPS: 71.50112933593469, Tokens per sec: 177187.67499406516, Loss: 2.496204137802124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6688 , TFLOPS: 70.72040602048033, Tokens per sec: 175252.95661459622, Loss: 2.5177993774414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6689 , TFLOPS: 68.36760327216335, Tokens per sec: 169422.45222164792, Loss: 2.486668586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6690 , TFLOPS: 71.5623426213197, Tokens per sec: 177339.36825844925, Loss: 2.5283732414245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6691 , TFLOPS: 71.53428050926566, Tokens per sec: 177269.82725348248, Loss: 2.5261728763580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6692 , TFLOPS: 68.65239437841052, Tokens per sec: 170128.19595525862, Loss: 2.516669511795044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6693 , TFLOPS: 68.75492532222421, Tokens per sec: 170382.2789287448, Loss: 2.510486602783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6694 , TFLOPS: 71.51993589094423, Tokens per sec: 177234.27970909182, Loss: 2.528146982192993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6695 , TFLOPS: 70.69641987658476, Tokens per sec: 175193.51630773113, Loss: 2.512897253036499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6696 , TFLOPS: 68.72898403185847, Tokens per sec: 170317.9935535495, Loss: 2.505232334136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6697 , TFLOPS: 70.53657823940239, Tokens per sec: 174797.41112278332, Loss: 2.5153751373291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6698 , TFLOPS: 70.59621571343317, Tokens per sec: 174945.19935304168, Loss: 2.515249013900757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6699 , TFLOPS: 68.87857909201755, Tokens per sec: 170688.70659187992, Loss: 2.481133222579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6700 , TFLOPS: 70.7142465320539, Tokens per sec: 175237.69272375188, Loss: 2.5032050609588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6701 , TFLOPS: 69.28297895052593, Tokens per sec: 171690.8539895857, Loss: 2.5204973220825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6702 , TFLOPS: 70.5487560017447, Tokens per sec: 174827.58895935884, Loss: 2.5197768211364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6703 , TFLOPS: 69.73709299994258, Tokens per sec: 172816.19862883232, Loss: 2.5045673847198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6704 , TFLOPS: 70.42350258423417, Tokens per sec: 174517.19719296476, Loss: 2.5229506492614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6705 , TFLOPS: 70.58348261534776, Tokens per sec: 174913.6453333195, Loss: 2.5281829833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6706 , TFLOPS: 68.310605078108, Tokens per sec: 169281.20441790938, Loss: 2.520292043685913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6707 , TFLOPS: 71.49389488096493, Tokens per sec: 177169.74721770908, Loss: 2.490415573120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6708 , TFLOPS: 69.77609114913538, Tokens per sec: 172912.8403385329, Loss: 2.517857313156128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6709 , TFLOPS: 68.59409993242436, Tokens per sec: 169983.73589644156, Loss: 2.5285661220550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6710 , TFLOPS: 71.52657981710604, Tokens per sec: 177250.74409000968, Loss: 2.521721124649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6711 , TFLOPS: 71.48967967679701, Tokens per sec: 177159.3014774379, Loss: 2.512033462524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6712 , TFLOPS: 67.80763985701569, Tokens per sec: 168034.8011938496, Loss: 2.5062246322631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6713 , TFLOPS: 69.84677594434768, Tokens per sec: 173088.005047642, Loss: 2.52850604057312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6714 , TFLOPS: 71.525932145903, Tokens per sec: 177249.1390894217, Loss: 2.5096781253814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6715 , TFLOPS: 69.77897020245022, Tokens per sec: 172919.974949801, Loss: 2.5132365226745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6716 , TFLOPS: 68.62482951750934, Tokens per sec: 170059.88719342538, Loss: 2.508366584777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6717 , TFLOPS: 71.54476831383003, Tokens per sec: 177295.81718852007, Loss: 2.5133392810821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6718 , TFLOPS: 70.59474161408585, Tokens per sec: 174941.54637247356, Loss: 2.5301578044891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6719 , TFLOPS: 68.46529777880632, Tokens per sec: 169664.54997104753, Loss: 2.504401445388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6720 , TFLOPS: 70.70067894195337, Tokens per sec: 175204.07074088935, Loss: 2.5058541297912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6721 , TFLOPS: 71.5252276205244, Tokens per sec: 177247.39319792364, Loss: 2.5003230571746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6722 , TFLOPS: 70.93957569992644, Tokens per sec: 175796.083223797, Loss: 2.481771469116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6723 , TFLOPS: 68.66654955051949, Tokens per sec: 170163.2740310656, Loss: 2.5037527084350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6724 , TFLOPS: 71.50247466336867, Tokens per sec: 177191.0088636466, Loss: 2.5100693702697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6725 , TFLOPS: 70.61186736815816, Tokens per sec: 174983.9858776223, Loss: 2.521963596343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6726 , TFLOPS: 67.04516298395158, Tokens per sec: 166145.2995086353, Loss: 2.508129596710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6727 , TFLOPS: 71.54955479414411, Tokens per sec: 177307.6786140127, Loss: 2.509270191192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6728 , TFLOPS: 70.70511206426275, Tokens per sec: 175215.05650632072, Loss: 2.5190517902374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6729 , TFLOPS: 69.50284573019817, Tokens per sec: 172235.70808993874, Loss: 2.5141689777374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6730 , TFLOPS: 69.87693051918023, Tokens per sec: 173162.73140587768, Loss: 2.481213092803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6731 , TFLOPS: 71.50376057319309, Tokens per sec: 177194.19548984617, Loss: 2.5084619522094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6732 , TFLOPS: 69.85752589590972, Tokens per sec: 173114.6446118176, Loss: 2.494870662689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6733 , TFLOPS: 69.13906232190844, Tokens per sec: 171334.21273014616, Loss: 2.4918718338012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6734 , TFLOPS: 70.5934466529342, Tokens per sec: 174938.33731608806, Loss: 2.5180788040161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6735 , TFLOPS: 70.65351625740226, Tokens per sec: 175087.1964131731, Loss: 2.519883155822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6736 , TFLOPS: 68.58634754755802, Tokens per sec: 169964.52463274708, Loss: 2.5215697288513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6737 , TFLOPS: 71.48204498519287, Tokens per sec: 177140.38187061215, Loss: 2.5303897857666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6738 , TFLOPS: 70.46859355493872, Tokens per sec: 174628.9375855511, Loss: 2.536287546157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6739 , TFLOPS: 69.06872244756313, Tokens per sec: 171159.90277294166, Loss: 2.503784418106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6740 , TFLOPS: 70.16573744466221, Tokens per sec: 173878.42678193122, Loss: 2.511070966720581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6741 , TFLOPS: 70.56827038247233, Tokens per sec: 174875.9477444884, Loss: 2.50852108001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6742 , TFLOPS: 70.66242539457677, Tokens per sec: 175109.27423651403, Loss: 2.5082645416259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6743 , TFLOPS: 68.5466760655975, Tokens per sec: 169866.21433027444, Loss: 2.522836208343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6744 , TFLOPS: 71.5013165728476, Tokens per sec: 177188.1389877612, Loss: 2.514960765838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6745 , TFLOPS: 69.58880105436025, Tokens per sec: 172448.714851973, Loss: 2.5013017654418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6746 , TFLOPS: 68.86143850964388, Tokens per sec: 170646.23034056695, Loss: 2.5099587440490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6747 , TFLOPS: 70.92386490568097, Tokens per sec: 175757.15014496987, Loss: 2.523454189300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6748 , TFLOPS: 70.65772533449216, Tokens per sec: 175097.62696987, Loss: 2.4965124130249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6749 , TFLOPS: 68.25075502285932, Tokens per sec: 169132.8893879753, Loss: 2.5059216022491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6750 , TFLOPS: 70.67373132109479, Tokens per sec: 175137.2915678763, Loss: 2.509854555130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6751 , TFLOPS: 71.5504338910391, Tokens per sec: 177309.85711295903, Loss: 2.5238423347473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6752 , TFLOPS: 70.5999766692293, Tokens per sec: 174954.519415808, Loss: 2.5040974617004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6753 , TFLOPS: 67.4642563106094, Tokens per sec: 167183.85893903452, Loss: 2.5052907466888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6754 , TFLOPS: 71.48767586336542, Tokens per sec: 177154.33580701615, Loss: 2.4835474491119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6755 , TFLOPS: 70.71074523760377, Tokens per sec: 175229.01613040504, Loss: 2.5141117572784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6756 , TFLOPS: 67.38717034061207, Tokens per sec: 166992.83141366052, Loss: 2.5021486282348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6757 , TFLOPS: 71.41233123669285, Tokens per sec: 176967.6235222257, Loss: 2.491459369659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6758 , TFLOPS: 71.54891679177116, Tokens per sec: 177306.09757385077, Loss: 2.5222315788269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6759 , TFLOPS: 68.89815161612714, Tokens per sec: 170737.20946271505, Loss: 2.5293331146240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6760 , TFLOPS: 68.81255391044397, Tokens per sec: 170525.08891866644, Loss: 2.5187556743621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6761 , TFLOPS: 71.48572093482848, Tokens per sec: 177149.49127874846, Loss: 2.505795955657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6762 , TFLOPS: 70.56882945227694, Tokens per sec: 174877.33318105084, Loss: 2.5019564628601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6763 , TFLOPS: 67.71939477298176, Tokens per sec: 167816.12015461503, Loss: 2.499199390411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6764 , TFLOPS: 71.52478204992933, Tokens per sec: 177246.28902490542, Loss: 2.4977593421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6765 , TFLOPS: 70.75120366213088, Tokens per sec: 175329.27656324662, Loss: 2.504298210144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6766 , TFLOPS: 68.82978666877696, Tokens per sec: 170567.79359216065, Loss: 2.5349926948547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6767 , TFLOPS: 69.67582498100633, Tokens per sec: 172664.3697286789, Loss: 2.5265212059020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6768 , TFLOPS: 71.42314468032956, Tokens per sec: 176994.42042675664, Loss: 2.5297060012817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6769 , TFLOPS: 69.90602831844927, Tokens per sec: 173234.83895785356, Loss: 2.50523042678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6770 , TFLOPS: 69.86595007571931, Tokens per sec: 173135.52065738582, Loss: 2.5387208461761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6771 , TFLOPS: 70.44512442754426, Tokens per sec: 174570.7785025298, Loss: 2.535541296005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6772 , TFLOPS: 70.70906824326767, Tokens per sec: 175224.860353704, Loss: 2.5376439094543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6773 , TFLOPS: 67.94396414595273, Tokens per sec: 168372.62779919576, Loss: 2.520937919616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6774 , TFLOPS: 71.49249006014192, Tokens per sec: 177166.2659169559, Loss: 2.513378620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6775 , TFLOPS: 69.62705646885443, Tokens per sec: 172543.51598327138, Loss: 2.4984679222106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6776 , TFLOPS: 68.7143373846561, Tokens per sec: 170281.69754832066, Loss: 2.5202150344848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6777 , TFLOPS: 71.53826380956974, Tokens per sec: 177279.69831043275, Loss: 2.5077669620513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6778 , TFLOPS: 70.57223537265175, Tokens per sec: 174885.77342693272, Loss: 2.511107921600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6779 , TFLOPS: 68.87247673572058, Tokens per sec: 170673.58428074606, Loss: 2.496198892593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6780 , TFLOPS: 69.91187501466348, Tokens per sec: 173249.3277151391, Loss: 2.5021743774414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6781 , TFLOPS: 71.53287209054173, Tokens per sec: 177266.33703673448, Loss: 2.5128936767578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6782 , TFLOPS: 69.67767723707875, Tokens per sec: 172668.95982326922, Loss: 2.5163192749023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6783 , TFLOPS: 68.50316473912962, Tokens per sec: 169758.3884701187, Loss: 2.497427225112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6784 , TFLOPS: 71.55162344370655, Tokens per sec: 177312.80495550897, Loss: 2.5061378479003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6785 , TFLOPS: 70.66103416958104, Tokens per sec: 175105.82662771852, Loss: 2.514732837677002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6786 , TFLOPS: 67.73736714391897, Tokens per sec: 167860.65766961756, Loss: 2.4951610565185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6787 , TFLOPS: 70.69839680611261, Tokens per sec: 175198.41535687793, Loss: 2.5256571769714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6788 , TFLOPS: 71.52286408800758, Tokens per sec: 177241.53610398236, Loss: 2.5214550495147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6789 , TFLOPS: 69.62760878279252, Tokens per sec: 172544.88467805236, Loss: 2.5076708793640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6790 , TFLOPS: 68.84901295462598, Tokens per sec: 170615.43844644472, Loss: 2.5139005184173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6791 , TFLOPS: 71.52419322731228, Tokens per sec: 177244.8298575962, Loss: 2.5227808952331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6792 , TFLOPS: 70.7449627949378, Tokens per sec: 175313.81100685388, Loss: 2.4989664554595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6793 , TFLOPS: 66.91552163531874, Tokens per sec: 165824.03396554964, Loss: 2.518228769302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6794 , TFLOPS: 71.53118526413188, Tokens per sec: 177262.1568950716, Loss: 2.5106253623962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6795 , TFLOPS: 70.58681712835848, Tokens per sec: 174921.908623887, Loss: 2.5152745246887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6796 , TFLOPS: 69.556887005926, Tokens per sec: 172369.62832433026, Loss: 2.508239269256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6797 , TFLOPS: 69.8137157479737, Tokens per sec: 173006.07823914598, Loss: 2.526463031768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6798 , TFLOPS: 71.49973517104529, Tokens per sec: 177184.22009988927, Loss: 2.5169637203216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6799 , TFLOPS: 68.82333976768393, Tokens per sec: 170551.81746106732, Loss: 2.52459979057312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6800 , TFLOPS: 68.77143240829821, Tokens per sec: 170423.18530644185, Loss: 2.5227537155151367 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/optimizer.pt +[rank0]:[2024-08-26 16:21:37,864] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006414260002202354, 'preprocessing_with_comm': 0.0012201369972899556, 'state_converting': 0.9956968199985567, : 1.0047064360041986}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800, took 4.52s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0004800 + eval ppl=11.117547988891602, eval loss=2.408524751663208 +------------------------------------------------------------------ +iteration: 6801 , TFLOPS: 66.80442318985442, Tokens per sec: 165548.71977918496, Loss: 2.511504888534546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6802 , TFLOPS: 70.57573050864762, Tokens per sec: 174894.4347589515, Loss: 2.529505491256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6803 , TFLOPS: 69.59135248884873, Tokens per sec: 172455.03758769916, Loss: 2.5132553577423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6804 , TFLOPS: 69.26305648607169, Tokens per sec: 171641.48392802748, Loss: 2.5021955966949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6805 , TFLOPS: 70.63499409237751, Tokens per sec: 175041.29644785685, Loss: 2.5195891857147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6806 , TFLOPS: 70.41507634130294, Tokens per sec: 174496.31603475387, Loss: 2.5193164348602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6807 , TFLOPS: 70.155800929279, Tokens per sec: 173853.8030022141, Loss: 2.505951166152954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6808 , TFLOPS: 70.34675351974163, Tokens per sec: 174327.0045565448, Loss: 2.5252151489257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6809 , TFLOPS: 70.61513060003595, Tokens per sec: 174992.07252569014, Loss: 2.518543004989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6810 , TFLOPS: 68.22806217013185, Tokens per sec: 169076.6539990925, Loss: 2.498504400253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6811 , TFLOPS: 70.39796727914312, Tokens per sec: 174453.91789400307, Loss: 2.503882884979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6812 , TFLOPS: 70.54207517456706, Tokens per sec: 174811.03313365878, Loss: 2.5115013122558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6813 , TFLOPS: 68.64541134444842, Tokens per sec: 170110.89122785584, Loss: 2.5180368423461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6814 , TFLOPS: 70.65402308481258, Tokens per sec: 175088.4523873275, Loss: 2.5311214923858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6815 , TFLOPS: 70.45428206945594, Tokens per sec: 174593.47214798452, Loss: 2.496129035949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6816 , TFLOPS: 69.55042534164117, Tokens per sec: 172353.6156084214, Loss: 2.5064268112182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6817 , TFLOPS: 69.71930368260384, Tokens per sec: 172772.11474081187, Loss: 2.498900890350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6818 , TFLOPS: 71.52434495899824, Tokens per sec: 177245.20586542864, Loss: 2.523160457611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6819 , TFLOPS: 69.6391144482901, Tokens per sec: 172573.39698460908, Loss: 2.5009193420410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6820 , TFLOPS: 68.58938706918208, Tokens per sec: 169972.05690216247, Loss: 2.498823881149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6821 , TFLOPS: 71.50052219868896, Tokens per sec: 177186.17044108853, Loss: 2.5192108154296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6822 , TFLOPS: 70.5987683991396, Tokens per sec: 174951.52518942376, Loss: 2.53285551071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6823 , TFLOPS: 67.73267972742278, Tokens per sec: 167849.04173517844, Loss: 2.500324010848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6824 , TFLOPS: 70.71916930995349, Tokens per sec: 175249.89190967582, Loss: 2.5146894454956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6825 , TFLOPS: 71.52698556817238, Tokens per sec: 177251.74958584926, Loss: 2.5120770931243896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6826 , TFLOPS: 67.96409511425507, Tokens per sec: 168422.5145562582, Loss: 2.5295770168304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6827 , TFLOPS: 70.58006643522407, Tokens per sec: 174905.1796626497, Loss: 2.522711753845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6828 , TFLOPS: 71.55999486758743, Tokens per sec: 177333.55026607428, Loss: 2.4897921085357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6829 , TFLOPS: 69.65570681020948, Tokens per sec: 172614.51468524482, Loss: 2.522606372833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6830 , TFLOPS: 68.63182772400368, Tokens per sec: 170077.22952003448, Loss: 2.507747173309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6831 , TFLOPS: 71.0209816317134, Tokens per sec: 175997.81609036872, Loss: 2.533012628555298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6832 , TFLOPS: 70.58538138361196, Tokens per sec: 174918.35069024478, Loss: 2.5362703800201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6833 , TFLOPS: 67.94750963738335, Tokens per sec: 168381.41391752893, Loss: 2.524311065673828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6834 , TFLOPS: 70.56076341189508, Tokens per sec: 174857.34464443687, Loss: 2.5219438076019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6835 , TFLOPS: 71.53765046407018, Tokens per sec: 177278.17837271927, Loss: 2.5103976726531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6836 , TFLOPS: 69.64697219304547, Tokens per sec: 172592.8693417149, Loss: 2.512244701385498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6837 , TFLOPS: 68.67170041667005, Tokens per sec: 170176.03844480144, Loss: 2.5184013843536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6838 , TFLOPS: 71.50719607006832, Tokens per sec: 177202.70902955424, Loss: 2.5226221084594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6839 , TFLOPS: 70.5477627632528, Tokens per sec: 174825.12760496166, Loss: 2.5174174308776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6840 , TFLOPS: 68.72951561412833, Tokens per sec: 170319.31087297283, Loss: 2.503518581390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6841 , TFLOPS: 70.48700557951904, Tokens per sec: 174674.56461071305, Loss: 2.521359920501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6842 , TFLOPS: 70.73483751828837, Tokens per sec: 175288.71945592528, Loss: 2.507054328918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6843 , TFLOPS: 68.78628400599112, Tokens per sec: 170459.9892015634, Loss: 2.513596773147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6844 , TFLOPS: 70.67671362915912, Tokens per sec: 175144.68205578806, Loss: 2.512018918991089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6845 , TFLOPS: 70.44212526648299, Tokens per sec: 174563.34625103514, Loss: 2.5014808177948 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6846 , TFLOPS: 69.5613042637914, Tokens per sec: 172380.57477592185, Loss: 2.508626937866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6847 , TFLOPS: 69.9052337090663, Tokens per sec: 173232.8698282691, Loss: 2.5039446353912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6848 , TFLOPS: 70.44332925860482, Tokens per sec: 174566.32987614372, Loss: 2.5197255611419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6849 , TFLOPS: 70.67233473585863, Tokens per sec: 175133.83067581453, Loss: 2.508420467376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6850 , TFLOPS: 69.02188431646239, Tokens per sec: 171043.83272442795, Loss: 2.531419038772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6851 , TFLOPS: 70.70179390697905, Tokens per sec: 175206.8337470465, Loss: 2.5073180198669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6852 , TFLOPS: 70.47039492694684, Tokens per sec: 174633.40158382474, Loss: 2.5204882621765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6853 , TFLOPS: 68.78375232607742, Tokens per sec: 170453.7154198497, Loss: 2.5087263584136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6854 , TFLOPS: 70.59879843971923, Tokens per sec: 174951.59963328936, Loss: 2.504863739013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6855 , TFLOPS: 71.51487339426743, Tokens per sec: 177221.73428464666, Loss: 2.5376029014587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6856 , TFLOPS: 68.61700149539952, Tokens per sec: 170040.48849230947, Loss: 2.5145468711853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6857 , TFLOPS: 69.81970126555242, Tokens per sec: 173020.91101106524, Loss: 2.5170578956604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6858 , TFLOPS: 71.53366262175608, Tokens per sec: 177268.29606016784, Loss: 2.5133159160614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6859 , TFLOPS: 70.54028374541699, Tokens per sec: 174806.59377488264, Loss: 2.495173931121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6860 , TFLOPS: 67.779710557336, Tokens per sec: 167965.589312577, Loss: 2.5168404579162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6861 , TFLOPS: 71.55972826271855, Tokens per sec: 177332.88958984023, Loss: 2.5187108516693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6862 , TFLOPS: 71.5520839313123, Tokens per sec: 177313.9460945229, Loss: 2.509544849395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6863 , TFLOPS: 69.02912304588654, Tokens per sec: 171061.77109335284, Loss: 2.514259099960327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6864 , TFLOPS: 70.08834371457928, Tokens per sec: 173686.63659316252, Loss: 2.497262716293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6865 , TFLOPS: 71.52434492996238, Tokens per sec: 177245.20579347457, Loss: 2.5037729740142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6866 , TFLOPS: 70.56074953163288, Tokens per sec: 174857.31024761812, Loss: 2.495333671569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6867 , TFLOPS: 67.66690728688714, Tokens per sec: 167686.05038209868, Loss: 2.4864981174468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6868 , TFLOPS: 71.50474296952005, Tokens per sec: 177196.6299761643, Loss: 2.5237858295440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6869 , TFLOPS: 70.70188435803614, Tokens per sec: 175207.05789472998, Loss: 2.4982926845550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6870 , TFLOPS: 67.90615045006189, Tokens per sec: 168278.92129525574, Loss: 2.492699146270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6871 , TFLOPS: 70.7332443180424, Tokens per sec: 175284.77133021026, Loss: 2.509321689605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6872 , TFLOPS: 70.40115447940566, Tokens per sec: 174461.8161273527, Loss: 2.5369648933410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6873 , TFLOPS: 68.49115936501933, Tokens per sec: 169728.63783057226, Loss: 2.5198347568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6874 , TFLOPS: 69.59949014680443, Tokens per sec: 172475.2036005504, Loss: 2.514643430709839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6875 , TFLOPS: 71.5249314940391, Tokens per sec: 177246.65936387202, Loss: 2.516721487045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6876 , TFLOPS: 69.73138991699909, Tokens per sec: 172802.06576105193, Loss: 2.514716625213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6877 , TFLOPS: 69.85744644113021, Tokens per sec: 173114.44771412134, Loss: 2.499969244003296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6878 , TFLOPS: 70.5811095314411, Tokens per sec: 174907.764569982, Loss: 2.531132221221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6879 , TFLOPS: 70.56760830858005, Tokens per sec: 174874.30705244988, Loss: 2.5143463611602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6880 , TFLOPS: 69.56746877800347, Tokens per sec: 172395.8511212167, Loss: 2.505803346633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6881 , TFLOPS: 70.97775235786925, Tokens per sec: 175890.68918768532, Loss: 2.5035879611968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6882 , TFLOPS: 70.40402613958163, Tokens per sec: 174468.93241760827, Loss: 2.492119789123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6883 , TFLOPS: 69.05546389370303, Tokens per sec: 171127.04661592605, Loss: 2.5106000900268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6884 , TFLOPS: 69.76359730648043, Tokens per sec: 172881.87922012355, Loss: 2.5218231678009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6885 , TFLOPS: 70.4687713149692, Tokens per sec: 174629.37809448852, Loss: 2.5093142986297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6886 , TFLOPS: 70.53857070260875, Tokens per sec: 174802.3486660966, Loss: 2.5296318531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6887 , TFLOPS: 68.71715187976739, Tokens per sec: 170288.6721772486, Loss: 2.5261926651000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6888 , TFLOPS: 71.57614720903884, Tokens per sec: 177373.57754751813, Loss: 2.5139870643615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6889 , TFLOPS: 69.61871564956964, Tokens per sec: 172522.84651426657, Loss: 2.5314323902130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6890 , TFLOPS: 69.00909099972037, Tokens per sec: 171012.129476241, Loss: 2.4994864463806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6891 , TFLOPS: 70.62293271203885, Tokens per sec: 175011.40701870693, Loss: 2.490464925765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6892 , TFLOPS: 70.62615171544681, Tokens per sec: 175019.38406375438, Loss: 2.4896531105041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6893 , TFLOPS: 68.73688966338952, Tokens per sec: 170337.5845793606, Loss: 2.5180766582489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6894 , TFLOPS: 70.04189704338201, Tokens per sec: 173571.53662541247, Loss: 2.520376443862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6895 , TFLOPS: 71.55580728117992, Tokens per sec: 177323.1729656557, Loss: 2.511425733566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6896 , TFLOPS: 70.67552619500303, Tokens per sec: 175141.7394631426, Loss: 2.50703763961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6897 , TFLOPS: 68.84075989649733, Tokens per sec: 170594.98645925257, Loss: 2.515012264251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6898 , TFLOPS: 70.79826053265022, Tokens per sec: 175445.88867213574, Loss: 2.5077643394470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6899 , TFLOPS: 70.55207709934704, Tokens per sec: 174835.81900506656, Loss: 2.5164926052093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6900 , TFLOPS: 68.69038638924363, Tokens per sec: 170222.34434326857, Loss: 2.5208163261413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6901 , TFLOPS: 70.5750795748019, Tokens per sec: 174892.82167317552, Loss: 2.5117027759552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6902 , TFLOPS: 71.50757042494493, Tokens per sec: 177203.63672217706, Loss: 2.488147020339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6903 , TFLOPS: 68.99660079517108, Tokens per sec: 170981.17737345825, Loss: 2.5181117057800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6904 , TFLOPS: 68.88180004008726, Tokens per sec: 170696.68845601342, Loss: 2.5248146057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6905 , TFLOPS: 71.5102311416465, Tokens per sec: 177210.2302712657, Loss: 2.5268683433532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6906 , TFLOPS: 70.59199320695978, Tokens per sec: 174934.73551685316, Loss: 2.5161304473876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6907 , TFLOPS: 67.65148999380605, Tokens per sec: 167647.84463148867, Loss: 2.5041275024414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6908 , TFLOPS: 71.50739374047309, Tokens per sec: 177203.19887859243, Loss: 2.5415127277374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6909 , TFLOPS: 70.67624086599994, Tokens per sec: 175143.51049660047, Loss: 2.523313045501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6910 , TFLOPS: 68.99033079999198, Tokens per sec: 170965.63963470748, Loss: 2.498741626739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6911 , TFLOPS: 69.66262591648062, Tokens per sec: 172631.66099278722, Loss: 2.4916272163391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6912 , TFLOPS: 71.47444571540653, Tokens per sec: 177121.5500429529, Loss: 2.4945225715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6913 , TFLOPS: 70.08007140310178, Tokens per sec: 173666.136894051, Loss: 2.513761520385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6914 , TFLOPS: 69.05091539580619, Tokens per sec: 171115.77493707926, Loss: 2.538811206817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6915 , TFLOPS: 70.44690814898983, Tokens per sec: 174575.1987607651, Loss: 2.502424478530884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6916 , TFLOPS: 70.67820826963397, Tokens per sec: 175148.38593953298, Loss: 2.5070457458496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6917 , TFLOPS: 69.60107255775894, Tokens per sec: 172479.12498921264, Loss: 2.5149147510528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6918 , TFLOPS: 70.66250066450684, Tokens per sec: 175109.4607636924, Loss: 2.512087106704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6919 , TFLOPS: 69.50733905751662, Tokens per sec: 172246.84305001472, Loss: 2.5067830085754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6920 , TFLOPS: 69.46492416052943, Tokens per sec: 172141.73426289385, Loss: 2.503919839859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6921 , TFLOPS: 70.5755376262863, Tokens per sec: 174893.9567752124, Loss: 2.4938368797302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6922 , TFLOPS: 70.4442587372891, Tokens per sec: 174568.6332267139, Loss: 2.5262582302093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6923 , TFLOPS: 68.88673651413838, Tokens per sec: 170708.92158250962, Loss: 2.526215076446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6924 , TFLOPS: 69.82449178893289, Tokens per sec: 173032.78245572175, Loss: 2.5177197456359863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6925 , TFLOPS: 71.55892686432861, Tokens per sec: 177330.90363634867, Loss: 2.535215377807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6926 , TFLOPS: 69.63697251412904, Tokens per sec: 172568.0890358044, Loss: 2.504331588745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6927 , TFLOPS: 68.55526710752076, Tokens per sec: 169887.50387854164, Loss: 2.510472297668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6928 , TFLOPS: 71.50197754935262, Tokens per sec: 177189.7769603532, Loss: 2.5145678520202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6929 , TFLOPS: 71.45147983911917, Tokens per sec: 177064.63807161272, Loss: 2.5157718658447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6930 , TFLOPS: 68.12929331018437, Tokens per sec: 168831.89388385456, Loss: 2.5052525997161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6931 , TFLOPS: 70.08771053597138, Tokens per sec: 173685.06750682183, Loss: 2.5208070278167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6932 , TFLOPS: 71.54544957555649, Tokens per sec: 177297.50543012543, Loss: 2.506530284881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6933 , TFLOPS: 70.7695316819071, Tokens per sec: 175374.6954152501, Loss: 2.5199151039123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6934 , TFLOPS: 67.83628036861957, Tokens per sec: 168105.77553661683, Loss: 2.51584529876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6935 , TFLOPS: 71.53150777834566, Tokens per sec: 177262.95612082118, Loss: 2.5046894550323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6936 , TFLOPS: 70.59792126782624, Tokens per sec: 174949.42590471002, Loss: 2.5084753036499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6937 , TFLOPS: 68.09842743054901, Tokens per sec: 168755.4048339608, Loss: 2.5059714317321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6938 , TFLOPS: 70.53914684842515, Tokens per sec: 174803.77641889732, Loss: 2.4954450130462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6939 , TFLOPS: 70.66676310470369, Tokens per sec: 175120.02356004674, Loss: 2.5040624141693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6940 , TFLOPS: 69.58829561932704, Tokens per sec: 172447.46232828245, Loss: 2.5071887969970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6941 , TFLOPS: 69.68379241523908, Tokens per sec: 172684.11390839296, Loss: 2.5165140628814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6942 , TFLOPS: 71.4957473644582, Tokens per sec: 177174.33787587332, Loss: 2.507357597351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6943 , TFLOPS: 68.83327325676876, Tokens per sec: 170576.43374128372, Loss: 2.518930435180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6944 , TFLOPS: 69.49691585529516, Tokens per sec: 172221.01320669943, Loss: 2.516340732574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6945 , TFLOPS: 71.48963179673241, Tokens per sec: 177159.1828253634, Loss: 2.5059332847595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6946 , TFLOPS: 70.70531947008334, Tokens per sec: 175215.5704807918, Loss: 2.5255138874053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6947 , TFLOPS: 69.32997976746513, Tokens per sec: 171807.3271915285, Loss: 2.550752639770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6948 , TFLOPS: 69.91048939706347, Tokens per sec: 173245.8940020879, Loss: 2.506392478942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6949 , TFLOPS: 71.53948305964053, Tokens per sec: 177282.71974641967, Loss: 2.5230250358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6950 , TFLOPS: 68.74399533648851, Tokens per sec: 170355.19322005447, Loss: 2.507887125015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6951 , TFLOPS: 69.64312477849525, Tokens per sec: 172583.33502463176, Loss: 2.491537094116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6952 , TFLOPS: 70.40746574822721, Tokens per sec: 174477.4561467362, Loss: 2.5172922611236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6953 , TFLOPS: 70.69835081084258, Tokens per sec: 175198.301375532, Loss: 2.4987313747406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6954 , TFLOPS: 68.02291510067096, Tokens per sec: 168568.27696215315, Loss: 2.4900312423706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6955 , TFLOPS: 70.49790732740622, Tokens per sec: 174701.5803429042, Loss: 2.533045768737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6956 , TFLOPS: 70.52663688999048, Tokens per sec: 174772.77536381056, Loss: 2.5026824474334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6957 , TFLOPS: 69.61640057705598, Tokens per sec: 172517.10950955143, Loss: 2.5147101879119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6958 , TFLOPS: 70.65338438703144, Tokens per sec: 175086.86962386747, Loss: 2.5005338191986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6959 , TFLOPS: 70.4531429365861, Tokens per sec: 174590.6492512472, Loss: 2.4977266788482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6960 , TFLOPS: 69.46754250067735, Tokens per sec: 172148.22279821438, Loss: 2.501598358154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6961 , TFLOPS: 70.60715313614479, Tokens per sec: 174972.30349137765, Loss: 2.511580467224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6962 , TFLOPS: 71.50404402807564, Tokens per sec: 177194.89792227026, Loss: 2.528597354888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6963 , TFLOPS: 68.71056433546526, Tokens per sec: 170272.34751679006, Loss: 2.490877389907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6964 , TFLOPS: 70.08126716929065, Tokens per sec: 173669.1001343915, Loss: 2.5113444328308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6965 , TFLOPS: 70.93209880543657, Tokens per sec: 175777.55465560316, Loss: 2.508307933807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6966 , TFLOPS: 70.73521481153456, Tokens per sec: 175289.65443015166, Loss: 2.5000314712524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6967 , TFLOPS: 68.63252984654657, Tokens per sec: 170078.96945704095, Loss: 2.528304100036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6968 , TFLOPS: 70.54387106561138, Tokens per sec: 174815.48354950035, Loss: 2.5002329349517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6969 , TFLOPS: 71.48753754042461, Tokens per sec: 177153.99302753128, Loss: 2.50563383102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6970 , TFLOPS: 67.77154308352536, Tokens per sec: 167945.34941275272, Loss: 2.5017149448394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6971 , TFLOPS: 69.84930192250822, Tokens per sec: 173094.26469978323, Loss: 2.5176303386688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6972 , TFLOPS: 71.522478886725, Tokens per sec: 177240.5815327707, Loss: 2.5056777000427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6973 , TFLOPS: 70.7015680447276, Tokens per sec: 175206.27403550662, Loss: 2.505732297897339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6974 , TFLOPS: 67.11347978837985, Tokens per sec: 166314.59607572583, Loss: 2.5105302333831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6975 , TFLOPS: 71.46362035080148, Tokens per sec: 177094.72359694925, Loss: 2.5084314346313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6976 , TFLOPS: 70.59016358931738, Tokens per sec: 174930.20152278626, Loss: 2.5052130222320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6977 , TFLOPS: 69.4600284102908, Tokens per sec: 172129.60205449112, Loss: 2.523331642150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6978 , TFLOPS: 69.49269934298935, Tokens per sec: 172210.56422471846, Loss: 2.5083744525909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6979 , TFLOPS: 71.57623072794718, Tokens per sec: 177373.7845165734, Loss: 2.4974207878112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6980 , TFLOPS: 69.96770431854685, Tokens per sec: 173387.67888026763, Loss: 2.492518663406372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6981 , TFLOPS: 70.08824497274169, Tokens per sec: 173686.3919000117, Loss: 2.5129566192626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6982 , TFLOPS: 70.51930266235799, Tokens per sec: 174754.6003398054, Loss: 2.5045230388641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6983 , TFLOPS: 70.59792781988158, Tokens per sec: 174949.44214142492, Loss: 2.4877920150756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6984 , TFLOPS: 68.52495912725567, Tokens per sec: 169812.39736474515, Loss: 2.5274698734283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6985 , TFLOPS: 69.58785493031966, Tokens per sec: 172446.37025237622, Loss: 2.5095560550689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6986 , TFLOPS: 70.70518023945168, Tokens per sec: 175215.225451949, Loss: 2.5193099975585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6987 , TFLOPS: 69.57137453870249, Tokens per sec: 172405.53002648387, Loss: 2.4974429607391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6988 , TFLOPS: 70.71455770737174, Tokens per sec: 175238.46385046828, Loss: 2.506953716278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6989 , TFLOPS: 69.54355858555581, Tokens per sec: 172336.59903040624, Loss: 2.5083236694335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6990 , TFLOPS: 68.75720563269685, Tokens per sec: 170387.92978929298, Loss: 2.5274486541748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6991 , TFLOPS: 69.77757238397949, Tokens per sec: 172916.5110016481, Loss: 2.4925429821014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6992 , TFLOPS: 70.52848988948121, Tokens per sec: 174777.36730067333, Loss: 2.5114328861236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6993 , TFLOPS: 70.73440675894919, Tokens per sec: 175287.65198683075, Loss: 2.5023488998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6994 , TFLOPS: 68.80191401574686, Tokens per sec: 170498.72208752585, Loss: 2.5109848976135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6995 , TFLOPS: 70.93643347656202, Tokens per sec: 175788.29644815548, Loss: 2.5082859992980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6996 , TFLOPS: 70.44655391898013, Tokens per sec: 174574.3209397794, Loss: 2.5246307849884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6997 , TFLOPS: 69.79495954611042, Tokens per sec: 172959.59830476256, Loss: 2.5247690677642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6998 , TFLOPS: 69.92507146005484, Tokens per sec: 173282.02996052738, Loss: 2.5281033515930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6999 , TFLOPS: 71.56657174868481, Tokens per sec: 177349.84850193685, Loss: 2.502249240875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7000 , TFLOPS: 69.779955366483, Tokens per sec: 172922.41629479898, Loss: 2.515423059463501 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/optimizer.pt +[rank0]:[2024-08-26 16:46:56,238] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006441999998060055, 'preprocessing_with_comm': 0.001321354000538122, 'state_converting': 0.9853197340053157, : 0.9944528570049442}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005000 + eval ppl=10.280977249145508, eval loss=2.3302953243255615 +------------------------------------------------------------------ +iteration: 7001 , TFLOPS: 62.1485664458258, Tokens per sec: 154010.9938825219, Loss: 2.5346577167510986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7002 , TFLOPS: 71.49934506258198, Tokens per sec: 177183.2533681432, Loss: 2.5035195350646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7003 , TFLOPS: 70.6479813991342, Tokens per sec: 175073.48042466986, Loss: 2.5062386989593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7004 , TFLOPS: 68.13461884555842, Tokens per sec: 168845.0911471676, Loss: 2.5114431381225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7005 , TFLOPS: 70.65516208483847, Tokens per sec: 175091.27495486286, Loss: 2.51446795463562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7006 , TFLOPS: 71.49646055871973, Tokens per sec: 177176.1052498182, Loss: 2.5274667739868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7007 , TFLOPS: 67.85040086880788, Tokens per sec: 168140.76769158497, Loss: 2.530440330505371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7008 , TFLOPS: 70.66166895035886, Tokens per sec: 175107.39968441278, Loss: 2.5213379859924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7009 , TFLOPS: 71.5092641929834, Tokens per sec: 177207.8340659603, Loss: 2.5084495544433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7010 , TFLOPS: 69.68590147706644, Tokens per sec: 172689.3403959335, Loss: 2.514906883239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7011 , TFLOPS: 68.57753059646345, Tokens per sec: 169942.6752566674, Loss: 2.511871337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7012 , TFLOPS: 71.44066414802293, Tokens per sec: 177037.83559762948, Loss: 2.499202251434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7013 , TFLOPS: 70.57943889688096, Tokens per sec: 174903.62455350638, Loss: 2.503749370574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7014 , TFLOPS: 68.7114549096615, Tokens per sec: 170274.55445775314, Loss: 2.48584246635437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7015 , TFLOPS: 70.3637304506575, Tokens per sec: 174369.07526151772, Loss: 2.505758285522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7016 , TFLOPS: 71.50852180680808, Tokens per sec: 177205.99435123728, Loss: 2.5213918685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7017 , TFLOPS: 68.96230636648131, Tokens per sec: 170896.19200132188, Loss: 2.5069737434387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7018 , TFLOPS: 69.01813072740565, Tokens per sec: 171034.53091725335, Loss: 2.493191719055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7019 , TFLOPS: 71.48531547804528, Tokens per sec: 177148.48651217495, Loss: 2.5138795375823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7020 , TFLOPS: 70.60138592119509, Tokens per sec: 174958.01169742114, Loss: 2.5005130767822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7021 , TFLOPS: 68.91278171945774, Tokens per sec: 170773.4644704072, Loss: 2.5141985416412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7022 , TFLOPS: 70.44415450984725, Tokens per sec: 174568.37493963138, Loss: 2.491577625274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7023 , TFLOPS: 70.65216036247726, Tokens per sec: 175083.8363561848, Loss: 2.5226659774780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7024 , TFLOPS: 68.91235313262271, Tokens per sec: 170772.40238501725, Loss: 2.4981613159179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7025 , TFLOPS: 70.7391262323131, Tokens per sec: 175299.3473617194, Loss: 2.5085368156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7026 , TFLOPS: 70.43089529131024, Tokens per sec: 174535.51713547317, Loss: 2.4883642196655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7027 , TFLOPS: 68.17578435500242, Tokens per sec: 168947.10381432503, Loss: 2.5096540451049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7028 , TFLOPS: 70.54407896466606, Tokens per sec: 174815.99874625978, Loss: 2.522146701812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7029 , TFLOPS: 70.4544252359921, Tokens per sec: 174593.82693043223, Loss: 2.515613079071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7030 , TFLOPS: 69.73524220384107, Tokens per sec: 172811.61215221082, Loss: 2.511676549911499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7031 , TFLOPS: 69.7108349350706, Tokens per sec: 172751.12825151495, Loss: 2.5151243209838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7032 , TFLOPS: 71.53986101044097, Tokens per sec: 177283.6563501379, Loss: 2.505990505218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7033 , TFLOPS: 69.56773424609658, Tokens per sec: 172396.5089803951, Loss: 2.483563184738159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7034 , TFLOPS: 69.54241793772617, Tokens per sec: 172333.7723794316, Loss: 2.5102312564849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7035 , TFLOPS: 70.84846060660372, Tokens per sec: 175570.2899853872, Loss: 2.482039213180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7036 , TFLOPS: 71.55934896072044, Tokens per sec: 177331.94963770534, Loss: 2.5115625858306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7037 , TFLOPS: 68.00567293888982, Tokens per sec: 168525.54898587856, Loss: 2.513169765472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7038 , TFLOPS: 69.7158282643453, Tokens per sec: 172763.50227151831, Loss: 2.4891531467437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7039 , TFLOPS: 71.56512491140788, Tokens per sec: 177346.2630797823, Loss: 2.514322280883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7040 , TFLOPS: 70.70878155598793, Tokens per sec: 175224.14991104358, Loss: 2.532804250717163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7041 , TFLOPS: 68.46045331721872, Tokens per sec: 169652.54486157198, Loss: 2.5012989044189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7042 , TFLOPS: 71.53568458547893, Tokens per sec: 177273.30670901077, Loss: 2.522433280944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7043 , TFLOPS: 71.53906173893392, Tokens per sec: 177281.6756672964, Loss: 2.4923224449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7044 , TFLOPS: 67.3268296452981, Tokens per sec: 166843.30052359618, Loss: 2.494907855987549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7045 , TFLOPS: 70.74623021462504, Tokens per sec: 175316.95181245668, Loss: 2.512042999267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7046 , TFLOPS: 71.52427781358911, Tokens per sec: 177245.03947170812, Loss: 2.51357364654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7047 , TFLOPS: 68.84490050173346, Tokens per sec: 170605.24733515302, Loss: 2.494014263153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7048 , TFLOPS: 69.90315631646811, Tokens per sec: 173227.72182056712, Loss: 2.5241141319274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7049 , TFLOPS: 71.5008192239007, Tokens per sec: 177186.90650228327, Loss: 2.495126247406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7050 , TFLOPS: 70.72486343210473, Tokens per sec: 175264.00257162593, Loss: 2.507741689682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7051 , TFLOPS: 69.88883922092856, Tokens per sec: 173192.24248066184, Loss: 2.5069401264190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7052 , TFLOPS: 70.39443430833306, Tokens per sec: 174445.16280314684, Loss: 2.505176305770874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7053 , TFLOPS: 70.56940400432487, Tokens per sec: 174878.7569843177, Loss: 2.4854958057403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7054 , TFLOPS: 69.49328770141277, Tokens per sec: 172212.02224170472, Loss: 2.511948823928833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7055 , TFLOPS: 70.60844381644361, Tokens per sec: 174975.5019393383, Loss: 2.5018503665924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7056 , TFLOPS: 70.5387827431435, Tokens per sec: 174802.8741258999, Loss: 2.5197956562042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7057 , TFLOPS: 68.95199368458083, Tokens per sec: 170870.63603953674, Loss: 2.502408742904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7058 , TFLOPS: 69.84766881059844, Tokens per sec: 173090.21766856866, Loss: 2.5100109577178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7059 , TFLOPS: 70.45594122427075, Tokens per sec: 174597.58371638658, Loss: 2.503354549407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7060 , TFLOPS: 70.62415117377257, Tokens per sec: 175014.42650110775, Loss: 2.5169124603271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7061 , TFLOPS: 68.73925371450427, Tokens per sec: 170343.4429584447, Loss: 2.510128974914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7062 , TFLOPS: 71.53357320016985, Tokens per sec: 177268.07446362657, Loss: 2.525803565979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7063 , TFLOPS: 70.59617211967911, Tokens per sec: 174945.09132291688, Loss: 2.520909547805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7064 , TFLOPS: 68.03238133644048, Tokens per sec: 168591.73533718195, Loss: 2.506675958633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7065 , TFLOPS: 70.55542914313793, Tokens per sec: 174844.12573883845, Loss: 2.5059986114501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7066 , TFLOPS: 70.4364830632768, Tokens per sec: 174549.3642499502, Loss: 2.5182814598083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7067 , TFLOPS: 69.69755710149853, Tokens per sec: 172718.22431724926, Loss: 2.5063889026641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7068 , TFLOPS: 69.18382880620427, Tokens per sec: 171445.14900966658, Loss: 2.495889186859131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7069 , TFLOPS: 71.48990814097388, Tokens per sec: 177159.8676368356, Loss: 2.5055718421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7070 , TFLOPS: 70.49910917175049, Tokens per sec: 174704.55864558337, Loss: 2.5050745010375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7071 , TFLOPS: 69.48760926618313, Tokens per sec: 172197.9504536744, Loss: 2.4886786937713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7072 , TFLOPS: 70.63138228075128, Tokens per sec: 175032.34598077316, Loss: 2.5175023078918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7073 , TFLOPS: 71.53632134654612, Tokens per sec: 177274.88467308026, Loss: 2.51759934425354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7074 , TFLOPS: 68.06933543209821, Tokens per sec: 168683.31165705156, Loss: 2.512723445892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7075 , TFLOPS: 70.73276444795815, Tokens per sec: 175283.58215927944, Loss: 2.497321128845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7076 , TFLOPS: 71.50787048348873, Tokens per sec: 177204.38030030276, Loss: 2.5302140712738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7077 , TFLOPS: 68.68936025076509, Tokens per sec: 170219.80145908025, Loss: 2.5272369384765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7078 , TFLOPS: 68.76266844239642, Tokens per sec: 170401.46723350667, Loss: 2.5134432315826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7079 , TFLOPS: 71.5247649662311, Tokens per sec: 177246.24668961926, Loss: 2.510957717895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7080 , TFLOPS: 70.69937924172383, Tokens per sec: 175200.84994054673, Loss: 2.5037195682525635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7081 , TFLOPS: 67.722832073945, Tokens per sec: 167824.63816505187, Loss: 2.517634391784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7082 , TFLOPS: 71.53302991896572, Tokens per sec: 177266.7281529554, Loss: 2.5286612510681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7083 , TFLOPS: 71.525529141382, Tokens per sec: 177248.14039982404, Loss: 2.5198466777801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7084 , TFLOPS: 68.69684155508162, Tokens per sec: 170238.3409553103, Loss: 2.505045175552368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7085 , TFLOPS: 69.05729550910443, Tokens per sec: 171131.58556065982, Loss: 2.51956844329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7086 , TFLOPS: 71.50412159423382, Tokens per sec: 177195.09013975487, Loss: 2.5152339935302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7087 , TFLOPS: 70.67782669684247, Tokens per sec: 175147.4403601219, Loss: 2.5172719955444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7088 , TFLOPS: 68.25679147751696, Tokens per sec: 169147.8483875873, Loss: 2.509536027908325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7089 , TFLOPS: 70.42856766447487, Tokens per sec: 174529.74901976585, Loss: 2.5151925086975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7090 , TFLOPS: 70.53203486359482, Tokens per sec: 174786.152137038, Loss: 2.506989002227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7091 , TFLOPS: 69.50500250661032, Tokens per sec: 172241.05281947658, Loss: 2.5041751861572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7092 , TFLOPS: 69.64074024968515, Tokens per sec: 172577.42589956278, Loss: 2.5157246589660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7093 , TFLOPS: 71.54840322282303, Tokens per sec: 177304.82489342298, Loss: 2.500422716140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7094 , TFLOPS: 68.67233436745848, Tokens per sec: 170177.60944469058, Loss: 2.4928555488586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7095 , TFLOPS: 70.67311575756776, Tokens per sec: 175135.76613364636, Loss: 2.534247398376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7096 , TFLOPS: 70.45362446725251, Tokens per sec: 174591.84253728355, Loss: 2.5190744400024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7097 , TFLOPS: 68.72630247727035, Tokens per sec: 170311.3483658825, Loss: 2.4968810081481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7098 , TFLOPS: 69.86376327174966, Tokens per sec: 173130.1015162527, Loss: 2.507282257080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7099 , TFLOPS: 71.54682089771248, Tokens per sec: 177300.90371749154, Loss: 2.5023531913757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7100 , TFLOPS: 69.72823398620214, Tokens per sec: 172794.24501688263, Loss: 2.517455577850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7101 , TFLOPS: 69.3872906709656, Tokens per sec: 171949.34992371977, Loss: 2.500183582305908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7102 , TFLOPS: 70.88356735791763, Tokens per sec: 175657.28838247096, Loss: 2.5055313110351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7103 , TFLOPS: 70.58243019919558, Tokens per sec: 174911.0373301612, Loss: 2.4996681213378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7104 , TFLOPS: 68.70190441063771, Tokens per sec: 170250.88726909788, Loss: 2.4845383167266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7105 , TFLOPS: 69.81907471773987, Tokens per sec: 173019.3583565658, Loss: 2.491133689880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7106 , TFLOPS: 71.50898352659827, Tokens per sec: 177207.13854374003, Loss: 2.520683526992798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7107 , TFLOPS: 69.69633862929126, Tokens per sec: 172715.20480889382, Loss: 2.5140223503112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7108 , TFLOPS: 67.90805176894052, Tokens per sec: 168283.63297288457, Loss: 2.513298273086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7109 , TFLOPS: 71.50547098415805, Tokens per sec: 177198.4340766347, Loss: 2.5160672664642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7110 , TFLOPS: 71.44205046707722, Tokens per sec: 177041.27104896167, Loss: 2.514704704284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7111 , TFLOPS: 68.21870526031209, Tokens per sec: 169053.46654581043, Loss: 2.525541305541992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7112 , TFLOPS: 70.66007664908953, Tokens per sec: 175103.45378646097, Loss: 2.497403144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7113 , TFLOPS: 71.54345724268516, Tokens per sec: 177292.56820979668, Loss: 2.5274784564971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7114 , TFLOPS: 68.72454895059113, Tokens per sec: 170307.00293360447, Loss: 2.493709087371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7115 , TFLOPS: 69.56231974962452, Tokens per sec: 172383.09126168131, Loss: 2.497964382171631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7116 , TFLOPS: 71.51777644502322, Tokens per sec: 177228.928364217, Loss: 2.5141756534576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7117 , TFLOPS: 69.36440253430824, Tokens per sec: 171892.63059974645, Loss: 2.5234453678131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7118 , TFLOPS: 69.065142196818, Tokens per sec: 171151.03051719803, Loss: 2.519174337387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7119 , TFLOPS: 70.89758547185765, Tokens per sec: 175692.02681303708, Loss: 2.5033020973205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7120 , TFLOPS: 70.73385174390204, Tokens per sec: 175286.27659840396, Loss: 2.535370111465454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7121 , TFLOPS: 69.55466122556973, Tokens per sec: 172364.11259541748, Loss: 2.5087990760803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7122 , TFLOPS: 69.4760808230863, Tokens per sec: 172169.3817017176, Loss: 2.5101613998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7123 , TFLOPS: 71.52770702006322, Tokens per sec: 177253.53742310923, Loss: 2.5061042308807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7124 , TFLOPS: 68.86847129058823, Tokens per sec: 170663.65834647187, Loss: 2.5294299125671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7125 , TFLOPS: 69.75347819462701, Tokens per sec: 172856.80294623857, Loss: 2.5047850608825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7126 , TFLOPS: 70.58898073691068, Tokens per sec: 174927.27028422052, Loss: 2.517017364501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7127 , TFLOPS: 70.7218740172564, Tokens per sec: 175256.59447232052, Loss: 2.51303768157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7128 , TFLOPS: 67.95984784863664, Tokens per sec: 168411.98936418077, Loss: 2.500626802444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7129 , TFLOPS: 71.54225856473853, Tokens per sec: 177289.59775379967, Loss: 2.5069334506988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7130 , TFLOPS: 70.42426115090414, Tokens per sec: 174519.07700473594, Loss: 2.515568733215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7131 , TFLOPS: 68.68665749547027, Tokens per sec: 170213.10373372663, Loss: 2.51072359085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7132 , TFLOPS: 70.58850861965846, Tokens per sec: 174926.10032566148, Loss: 2.5051989555358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7133 , TFLOPS: 70.55943068127385, Tokens per sec: 174854.04199114506, Loss: 2.4932544231414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7134 , TFLOPS: 69.31837772843416, Tokens per sec: 171778.57606073958, Loss: 2.4984965324401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7135 , TFLOPS: 69.96841022948792, Tokens per sec: 173389.4282053421, Loss: 2.5245704650878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7136 , TFLOPS: 71.50529476391742, Tokens per sec: 177197.9973834661, Loss: 2.5120060443878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7137 , TFLOPS: 69.5992503292032, Tokens per sec: 172474.60930611647, Loss: 2.5234060287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7138 , TFLOPS: 69.22342597743118, Tokens per sec: 171543.27516195533, Loss: 2.516908645629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7139 , TFLOPS: 70.5686817852079, Tokens per sec: 174876.9672457869, Loss: 2.528731346130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7140 , TFLOPS: 69.7690897883618, Tokens per sec: 172895.49019527237, Loss: 2.5041685104370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7141 , TFLOPS: 69.46867568031733, Tokens per sec: 172151.0309421923, Loss: 2.5062882900238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7142 , TFLOPS: 70.64458012577258, Tokens per sec: 175065.05169459857, Loss: 2.4886765480041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7143 , TFLOPS: 71.45175611026734, Tokens per sec: 177065.32270195163, Loss: 2.5276193618774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7144 , TFLOPS: 67.89139509626906, Tokens per sec: 168242.35590312074, Loss: 2.5119216442108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7145 , TFLOPS: 69.9275262610854, Tokens per sec: 173288.11322789977, Loss: 2.5270206928253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7146 , TFLOPS: 71.51362888219016, Tokens per sec: 177218.65024662492, Loss: 2.5263876914978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7147 , TFLOPS: 70.6908947558164, Tokens per sec: 175179.82444982484, Loss: 2.502767324447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7148 , TFLOPS: 67.73836811261725, Tokens per sec: 167863.1381803177, Loss: 2.4788737297058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7149 , TFLOPS: 70.81236406054751, Tokens per sec: 175480.83876789708, Loss: 2.5087571144104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7150 , TFLOPS: 71.52891869140946, Tokens per sec: 177256.54007818186, Loss: 2.5171871185302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7151 , TFLOPS: 69.64966802030189, Tokens per sec: 172599.5498986262, Loss: 2.517484664916992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7152 , TFLOPS: 69.22291660417896, Tokens per sec: 171542.01287892426, Loss: 2.5081984996795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7153 , TFLOPS: 71.532090400691, Tokens per sec: 177264.39992317415, Loss: 2.5282843112945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7154 , TFLOPS: 70.03753627993179, Tokens per sec: 173560.7301732051, Loss: 2.514953136444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7155 , TFLOPS: 68.31210192279346, Tokens per sec: 169284.913763931, Loss: 2.539168119430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7156 , TFLOPS: 71.50659512660174, Tokens per sec: 177201.21982544463, Loss: 2.4948530197143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7157 , TFLOPS: 70.57366764751947, Tokens per sec: 174889.32276183303, Loss: 2.4956231117248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7158 , TFLOPS: 68.90543632832359, Tokens per sec: 170755.26178201515, Loss: 2.507939338684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7159 , TFLOPS: 69.16071400028676, Tokens per sec: 171387.8679742969, Loss: 2.4946320056915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7160 , TFLOPS: 71.55527134098156, Tokens per sec: 177321.84484680553, Loss: 2.5092039108276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7161 , TFLOPS: 69.0376253987434, Tokens per sec: 171082.84086034365, Loss: 2.501810073852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7162 , TFLOPS: 70.6074055244459, Tokens per sec: 174972.92893739068, Loss: 2.508612632751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7163 , TFLOPS: 70.42057634768189, Tokens per sec: 174509.9456564366, Loss: 2.5289623737335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7164 , TFLOPS: 69.59529032577632, Tokens per sec: 172464.79598139424, Loss: 2.507378101348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7165 , TFLOPS: 69.34789321866977, Tokens per sec: 171851.71869694308, Loss: 2.52288556098938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7166 , TFLOPS: 69.45112205555544, Tokens per sec: 172107.53112634094, Loss: 2.4939441680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7167 , TFLOPS: 70.70361080938169, Tokens per sec: 175211.33623134857, Loss: 2.5323750972747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7168 , TFLOPS: 68.82850725837498, Tokens per sec: 170564.62307224583, Loss: 2.520047426223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7169 , TFLOPS: 70.52391495761724, Tokens per sec: 174766.03011554407, Loss: 2.499941110610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7170 , TFLOPS: 70.22849350078792, Tokens per sec: 174033.94320216103, Loss: 2.5165772438049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7171 , TFLOPS: 69.55657432092087, Tokens per sec: 172368.85345644245, Loss: 2.538259267807007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7172 , TFLOPS: 69.68713388453739, Tokens per sec: 172692.3944374074, Loss: 2.5188851356506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7173 , TFLOPS: 71.53733721266408, Tokens per sec: 177277.40210122746, Loss: 2.519705295562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7174 , TFLOPS: 68.85666117735869, Tokens per sec: 170634.39158489768, Loss: 2.501007080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7175 , TFLOPS: 68.6473007656061, Tokens per sec: 170115.57342162152, Loss: 2.506547451019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7176 , TFLOPS: 71.45044712463253, Tokens per sec: 177062.07889135194, Loss: 2.495178461074829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7177 , TFLOPS: 70.38541772654477, Tokens per sec: 174422.818720217, Loss: 2.4971256256103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7178 , TFLOPS: 68.16024837972678, Tokens per sec: 168908.60395616305, Loss: 2.514962911605835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7179 , TFLOPS: 70.65348884802577, Tokens per sec: 175087.12848971877, Loss: 2.518702268600464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7180 , TFLOPS: 71.51564981015758, Tokens per sec: 177223.65832875177, Loss: 2.5088021755218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7181 , TFLOPS: 67.88191470762801, Tokens per sec: 168218.86245571717, Loss: 2.5153748989105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7182 , TFLOPS: 70.70128257963779, Tokens per sec: 175205.56662156753, Loss: 2.507357120513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7183 , TFLOPS: 71.55894061784232, Tokens per sec: 177330.9377190707, Loss: 2.5040011405944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7184 , TFLOPS: 69.65046582833571, Tokens per sec: 172601.5269548198, Loss: 2.4879980087280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7185 , TFLOPS: 68.8008081525809, Tokens per sec: 170495.98163677973, Loss: 2.536013603210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7186 , TFLOPS: 70.94057290105913, Tokens per sec: 175798.5543980546, Loss: 2.514010190963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7187 , TFLOPS: 70.29665104517962, Tokens per sec: 174202.84510533698, Loss: 2.5027480125427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7188 , TFLOPS: 68.81989152526359, Tokens per sec: 170543.27233649447, Loss: 2.4954495429992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7189 , TFLOPS: 69.95329858739535, Tokens per sec: 173351.97989155195, Loss: 2.5171737670898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7190 , TFLOPS: 71.55279387902746, Tokens per sec: 177315.70542316829, Loss: 2.505929470062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7191 , TFLOPS: 69.61493601705133, Tokens per sec: 172513.4801685253, Loss: 2.5133910179138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7192 , TFLOPS: 68.80143440409269, Tokens per sec: 170497.53355701323, Loss: 2.503061056137085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7193 , TFLOPS: 71.52079564220737, Tokens per sec: 177236.41026743135, Loss: 2.5234508514404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7194 , TFLOPS: 70.59068762644115, Tokens per sec: 174931.50014450643, Loss: 2.506730318069458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7195 , TFLOPS: 68.7255067159461, Tokens per sec: 170309.37638166064, Loss: 2.471926689147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7196 , TFLOPS: 70.47879646440764, Tokens per sec: 174654.22151915834, Loss: 2.5099873542785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7197 , TFLOPS: 71.47519428380217, Tokens per sec: 177123.4050778989, Loss: 2.493628978729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7198 , TFLOPS: 67.99329109335629, Tokens per sec: 168494.8654086736, Loss: 2.5148844718933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7199 , TFLOPS: 70.65252009521733, Tokens per sec: 175084.7278135425, Loss: 2.5238358974456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7200 , TFLOPS: 70.53935564066846, Tokens per sec: 174804.29382907666, Loss: 2.5139946937561035 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/optimizer.pt +[rank0]:[2024-08-26 17:12:15,161] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006405320003977977, 'preprocessing_with_comm': 0.0012870380014646798, 'state_converting': 0.9983762970005046, : 1.0074359479986015}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200, took 4.55s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005200 + eval ppl=10.08486270904541, eval loss=2.311035633087158 +------------------------------------------------------------------ +iteration: 7201 , TFLOPS: 63.819845064092824, Tokens per sec: 158152.60640512579, Loss: 2.516300916671753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7202 , TFLOPS: 69.69565564797657, Tokens per sec: 172713.51230595945, Loss: 2.489018440246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7203 , TFLOPS: 70.03035317220265, Tokens per sec: 173542.92964096766, Loss: 2.510117769241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7204 , TFLOPS: 69.89350362740335, Tokens per sec: 173203.80139373313, Loss: 2.4987597465515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7205 , TFLOPS: 68.9827524076132, Tokens per sec: 170946.85954356403, Loss: 2.4963302612304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7206 , TFLOPS: 71.49288092407654, Tokens per sec: 177167.23452084308, Loss: 2.5315780639648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7207 , TFLOPS: 69.4511940291105, Tokens per sec: 172107.70948473888, Loss: 2.516960382461548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7208 , TFLOPS: 68.13722138350461, Tokens per sec: 168851.5405228915, Loss: 2.5142910480499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7209 , TFLOPS: 70.82718246487221, Tokens per sec: 175517.56040619567, Loss: 2.495304822921753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7210 , TFLOPS: 71.54572340262892, Tokens per sec: 177298.18400377544, Loss: 2.510458469390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7211 , TFLOPS: 68.72614155352765, Tokens per sec: 170310.94957912088, Loss: 2.492568016052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7212 , TFLOPS: 69.86774044439689, Tokens per sec: 173139.95738819425, Loss: 2.5237114429473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7213 , TFLOPS: 71.53035193644907, Tokens per sec: 177260.09181727498, Loss: 2.490313768386841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7214 , TFLOPS: 70.5403332100424, Tokens per sec: 174806.71635367346, Loss: 2.4981820583343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7215 , TFLOPS: 67.94473921698665, Tokens per sec: 168374.54851059892, Loss: 2.505075216293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7216 , TFLOPS: 71.48581620193099, Tokens per sec: 177149.72736112302, Loss: 2.4801549911499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7217 , TFLOPS: 70.67197429194474, Tokens per sec: 175132.93745608968, Loss: 2.5101592540740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7218 , TFLOPS: 68.1093298771665, Tokens per sec: 168782.42229768258, Loss: 2.516580581665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7219 , TFLOPS: 69.3638411928182, Tokens per sec: 171891.23953369728, Loss: 2.50112247467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7220 , TFLOPS: 71.5276172073893, Tokens per sec: 177253.3148574096, Loss: 2.5127906799316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7221 , TFLOPS: 68.34434068195695, Tokens per sec: 169364.80496053063, Loss: 2.486579179763794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7222 , TFLOPS: 68.88900444007122, Tokens per sec: 170714.5417528038, Loss: 2.537513494491577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7223 , TFLOPS: 71.45709116602437, Tokens per sec: 177078.5435578225, Loss: 2.5072221755981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7224 , TFLOPS: 69.82976573617205, Tokens per sec: 173045.85187795362, Loss: 2.5109457969665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7225 , TFLOPS: 68.13485812843847, Tokens per sec: 168845.68411650351, Loss: 2.5260281562805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7226 , TFLOPS: 70.46259032842671, Tokens per sec: 174614.06092894438, Loss: 2.5286078453063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7227 , TFLOPS: 70.5580971076617, Tokens per sec: 174850.73724882887, Loss: 2.520261287689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7228 , TFLOPS: 68.03176974132683, Tokens per sec: 168590.21973711756, Loss: 2.532780647277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7229 , TFLOPS: 69.64810266733055, Tokens per sec: 172595.67078152535, Loss: 2.511288642883301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7230 , TFLOPS: 71.49026544072213, Tokens per sec: 177160.75306497156, Loss: 2.494924306869507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7231 , TFLOPS: 69.02969294240017, Tokens per sec: 171063.18335969292, Loss: 2.5112104415893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7232 , TFLOPS: 69.49060269809138, Tokens per sec: 172205.36850769678, Loss: 2.505772352218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7233 , TFLOPS: 70.4351106442469, Tokens per sec: 174545.96324440968, Loss: 2.510283946990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7234 , TFLOPS: 70.54713735017522, Tokens per sec: 174823.57776246045, Loss: 2.5091865062713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7235 , TFLOPS: 67.94241438827669, Tokens per sec: 168368.78732895464, Loss: 2.5021750926971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7236 , TFLOPS: 71.50832658340634, Tokens per sec: 177205.51056614215, Loss: 2.4867796897888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7237 , TFLOPS: 70.33962840981069, Tokens per sec: 174309.3477492403, Loss: 2.5246167182922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7238 , TFLOPS: 67.62749443089523, Tokens per sec: 167588.3809832662, Loss: 2.487056255340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7239 , TFLOPS: 70.6026423735839, Tokens per sec: 174961.1253248503, Loss: 2.5305092334747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7240 , TFLOPS: 71.48659457381741, Tokens per sec: 177151.65625240232, Loss: 2.4852137565612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7241 , TFLOPS: 67.83094261054742, Tokens per sec: 168092.54798411147, Loss: 2.4967706203460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7242 , TFLOPS: 69.90334380597764, Tokens per sec: 173228.1864402254, Loss: 2.502671957015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7243 , TFLOPS: 71.55423359330231, Tokens per sec: 177319.273193739, Loss: 2.5145514011383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7244 , TFLOPS: 68.8711801490324, Tokens per sec: 170670.37119610189, Loss: 2.520880699157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7245 , TFLOPS: 68.78329895401835, Tokens per sec: 170452.59191394426, Loss: 2.5266013145446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7246 , TFLOPS: 70.88223941132556, Tokens per sec: 175653.9975845304, Loss: 2.502609968185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7247 , TFLOPS: 70.69358428369412, Tokens per sec: 175186.48939618177, Loss: 2.5171124935150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7248 , TFLOPS: 67.91168031986088, Tokens per sec: 168292.62492178308, Loss: 2.5005927085876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7249 , TFLOPS: 70.16466232758268, Tokens per sec: 173875.7625233742, Loss: 2.5163609981536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7250 , TFLOPS: 71.50737033910855, Tokens per sec: 177203.14088743323, Loss: 2.5124778747558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7251 , TFLOPS: 69.70949386951929, Tokens per sec: 172747.80494334825, Loss: 2.517224073410034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7252 , TFLOPS: 68.62311057056965, Tokens per sec: 170055.6274535512, Loss: 2.5133063793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7253 , TFLOPS: 71.52952939389168, Tokens per sec: 177258.0534662072, Loss: 2.521052122116089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7254 , TFLOPS: 70.66832986183205, Tokens per sec: 175123.90615680336, Loss: 2.5247738361358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7255 , TFLOPS: 67.10905560847017, Tokens per sec: 166303.632470546, Loss: 2.5073585510253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7256 , TFLOPS: 71.54645818845847, Tokens per sec: 177300.0048840044, Loss: 2.5211880207061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7257 , TFLOPS: 71.54222844102269, Tokens per sec: 177289.52310391338, Loss: 2.509744167327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7258 , TFLOPS: 67.17911320353039, Tokens per sec: 166477.24290858643, Loss: 2.502089500427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7259 , TFLOPS: 70.3791654770587, Tokens per sec: 174407.32495724838, Loss: 2.4898552894592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7260 , TFLOPS: 71.52947833894646, Tokens per sec: 177257.92694642878, Loss: 2.5022661685943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7261 , TFLOPS: 68.9210456684569, Tokens per sec: 170793.94344637624, Loss: 2.5106430053710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7262 , TFLOPS: 69.25202267889335, Tokens per sec: 171614.1409383653, Loss: 2.519657850265503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7263 , TFLOPS: 70.52164418120925, Tokens per sec: 174760.40288145843, Loss: 2.510185718536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7264 , TFLOPS: 70.55455885932041, Tokens per sec: 174841.96907966895, Loss: 2.5042061805725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7265 , TFLOPS: 68.44457934453428, Tokens per sec: 169613.2074085975, Loss: 2.5327510833740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7266 , TFLOPS: 69.08283063234488, Tokens per sec: 171194.86440897567, Loss: 2.5034384727478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7267 , TFLOPS: 70.76872894661533, Tokens per sec: 175372.7061487685, Loss: 2.508755683898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7268 , TFLOPS: 69.0088909793059, Tokens per sec: 171011.63380361994, Loss: 2.5309054851531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7269 , TFLOPS: 69.90677492889061, Tokens per sec: 173236.68914077312, Loss: 2.5057408809661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7270 , TFLOPS: 70.19384957431905, Tokens per sec: 173948.0917360276, Loss: 2.5202701091766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7271 , TFLOPS: 69.65190268938235, Tokens per sec: 172605.0876547767, Loss: 2.507997751235962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7272 , TFLOPS: 68.73812243980855, Tokens per sec: 170340.63953512852, Loss: 2.5255329608917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7273 , TFLOPS: 71.51071233658526, Tokens per sec: 177211.4227253319, Loss: 2.5278518199920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7274 , TFLOPS: 69.72045278475892, Tokens per sec: 172774.96234253625, Loss: 2.5338010787963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7275 , TFLOPS: 68.0519750790035, Tokens per sec: 168640.29078968207, Loss: 2.5285327434539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7276 , TFLOPS: 71.53624402952151, Tokens per sec: 177274.69307297614, Loss: 2.502089262008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7277 , TFLOPS: 71.43753786395125, Tokens per sec: 177030.08832131125, Loss: 2.508897542953491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7278 , TFLOPS: 67.94978908225382, Tokens per sec: 168387.06263301952, Loss: 2.5094070434570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7279 , TFLOPS: 70.07543716600013, Tokens per sec: 173654.6527440097, Loss: 2.5180671215057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7280 , TFLOPS: 71.52333228322772, Tokens per sec: 177242.69634331382, Loss: 2.4837541580200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7281 , TFLOPS: 68.87299898216413, Tokens per sec: 170674.87846496335, Loss: 2.5116286277770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7282 , TFLOPS: 68.53836287591322, Tokens per sec: 169845.61333046333, Loss: 2.492480754852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7283 , TFLOPS: 71.51813942382896, Tokens per sec: 177229.827865683, Loss: 2.5089828968048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7284 , TFLOPS: 70.53575439234416, Tokens per sec: 174795.3695390188, Loss: 2.5033483505249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7285 , TFLOPS: 67.90869580030174, Tokens per sec: 168285.2289535436, Loss: 2.515465259552002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7286 , TFLOPS: 70.62619488454865, Tokens per sec: 175019.49104154418, Loss: 2.498230457305908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7287 , TFLOPS: 71.55139282490224, Tokens per sec: 177312.23345670712, Loss: 2.494086742401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7288 , TFLOPS: 68.46047896296585, Tokens per sec: 169652.6084145585, Loss: 2.5184168815612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7289 , TFLOPS: 68.97783024700185, Tokens per sec: 170934.6618873483, Loss: 2.516789674758911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7290 , TFLOPS: 71.49197876832682, Tokens per sec: 177164.99887951463, Loss: 2.509481906890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7291 , TFLOPS: 69.65914194034059, Tokens per sec: 172623.02731612054, Loss: 2.5189435482025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7292 , TFLOPS: 68.78398493916335, Tokens per sec: 170454.29186070108, Loss: 2.501725673675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7293 , TFLOPS: 70.43532982213209, Tokens per sec: 174546.50639135359, Loss: 2.500511407852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7294 , TFLOPS: 70.72911359444537, Tokens per sec: 175274.53494210006, Loss: 2.507783889770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7295 , TFLOPS: 68.85654216200352, Tokens per sec: 170634.09665173694, Loss: 2.5110771656036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7296 , TFLOPS: 69.85090984115278, Tokens per sec: 173098.24929931134, Loss: 2.5059101581573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7297 , TFLOPS: 71.48838960133219, Tokens per sec: 177156.10452832252, Loss: 2.5309677124023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7298 , TFLOPS: 68.80537177725998, Tokens per sec: 170507.29080145922, Loss: 2.515530586242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7299 , TFLOPS: 69.25774528237322, Tokens per sec: 171628.32218018526, Loss: 2.5168755054473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7300 , TFLOPS: 70.47172181047699, Tokens per sec: 174636.68974738076, Loss: 2.4942712783813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7301 , TFLOPS: 70.23542790329604, Tokens per sec: 174051.12741546365, Loss: 2.5143778324127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7302 , TFLOPS: 67.624669834867, Tokens per sec: 167581.38132315333, Loss: 2.497981309890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7303 , TFLOPS: 70.5783161207243, Tokens per sec: 174900.84219050617, Loss: 2.5145249366760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7304 , TFLOPS: 71.50877556123429, Tokens per sec: 177206.6231826588, Loss: 2.5435004234313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7305 , TFLOPS: 67.44947593512423, Tokens per sec: 167147.23154038948, Loss: 2.527688980102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7306 , TFLOPS: 70.69359254320968, Tokens per sec: 175186.5098641712, Loss: 2.5049965381622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7307 , TFLOPS: 70.43285975963535, Tokens per sec: 174540.38530438684, Loss: 2.518618583679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7308 , TFLOPS: 69.07647704648008, Tokens per sec: 171179.11952329744, Loss: 2.5217461585998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7309 , TFLOPS: 69.05986119329451, Tokens per sec: 171137.9436087163, Loss: 2.519036054611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7310 , TFLOPS: 71.49519214517794, Tokens per sec: 177172.96198133435, Loss: 2.5165817737579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7311 , TFLOPS: 69.66063375891925, Tokens per sec: 172626.72420689583, Loss: 2.505974769592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7312 , TFLOPS: 68.98699434072932, Tokens per sec: 170957.37152111327, Loss: 2.511995792388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7313 , TFLOPS: 70.83466468308038, Tokens per sec: 175536.1021671499, Loss: 2.490966320037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7314 , TFLOPS: 70.69858632941879, Tokens per sec: 175198.88501650834, Loss: 2.4854373931884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7315 , TFLOPS: 68.22057560708811, Tokens per sec: 169058.1014711571, Loss: 2.4966695308685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7316 , TFLOPS: 70.16315965706445, Tokens per sec: 173872.03874029726, Loss: 2.506802558898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7317 , TFLOPS: 71.38640693760134, Tokens per sec: 176903.38025327964, Loss: 2.500556230545044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7318 , TFLOPS: 69.51516760202061, Tokens per sec: 172266.24304568095, Loss: 2.5021073818206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7319 , TFLOPS: 68.18676825643102, Tokens per sec: 168974.32313204175, Loss: 2.5054268836975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7320 , TFLOPS: 71.52000397509151, Tokens per sec: 177234.4484291089, Loss: 2.5404469966888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7321 , TFLOPS: 70.71351376103348, Tokens per sec: 175235.8768364419, Loss: 2.522883892059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7322 , TFLOPS: 65.81985168569972, Tokens per sec: 163108.84313238488, Loss: 2.520838499069214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7323 , TFLOPS: 71.48271571031052, Tokens per sec: 177142.04400134014, Loss: 2.521522045135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7324 , TFLOPS: 71.5494562821126, Tokens per sec: 177307.43449034664, Loss: 2.5019021034240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7325 , TFLOPS: 67.33400204001275, Tokens per sec: 166861.0745078036, Loss: 2.4848365783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7326 , TFLOPS: 69.6548023769665, Tokens per sec: 172612.27340003717, Loss: 2.5174930095672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7327 , TFLOPS: 71.53385895975158, Tokens per sec: 177268.7826073491, Loss: 2.5269088745117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7328 , TFLOPS: 69.02568648694157, Tokens per sec: 171053.254921719, Loss: 2.5110580921173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7329 , TFLOPS: 67.22363337860818, Tokens per sec: 166587.56880673132, Loss: 2.5312836170196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7330 , TFLOPS: 71.52811997361911, Tokens per sec: 177254.5607675113, Loss: 2.5154521465301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7331 , TFLOPS: 70.57063309294985, Tokens per sec: 174881.8028013257, Loss: 2.495351791381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7332 , TFLOPS: 68.78447262747353, Tokens per sec: 170455.50040605702, Loss: 2.5065183639526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7333 , TFLOPS: 69.60541650140686, Tokens per sec: 172489.88976009708, Loss: 2.511539936065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7334 , TFLOPS: 71.52491537655344, Tokens per sec: 177246.6194229671, Loss: 2.5102927684783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7335 , TFLOPS: 68.42849953153284, Tokens per sec: 169573.3598606423, Loss: 2.507216453552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7336 , TFLOPS: 69.78147383106766, Tokens per sec: 172926.17921731246, Loss: 2.5082478523254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7337 , TFLOPS: 70.49069962819557, Tokens per sec: 174683.71887026657, Loss: 2.4909486770629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7338 , TFLOPS: 69.7504586450946, Tokens per sec: 172849.32016986643, Loss: 2.471768617630005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7339 , TFLOPS: 68.14462989621066, Tokens per sec: 168869.89963349214, Loss: 2.496910810470581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7340 , TFLOPS: 70.5304297388818, Tokens per sec: 174782.17446118002, Loss: 2.537790536880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7341 , TFLOPS: 70.66181781799074, Tokens per sec: 175107.76859480373, Loss: 2.5367088317871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7342 , TFLOPS: 67.06208081884095, Tokens per sec: 166187.22376714452, Loss: 2.493734359741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7343 , TFLOPS: 71.51322597771416, Tokens per sec: 177217.65180494977, Loss: 2.502229928970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7344 , TFLOPS: 70.56495284841017, Tokens per sec: 174867.72652962583, Loss: 2.5038809776306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7345 , TFLOPS: 68.13545837280078, Tokens per sec: 168847.17158815573, Loss: 2.52543568611145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7346 , TFLOPS: 70.00649243477163, Tokens per sec: 173483.80010513638, Loss: 2.496127128601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7347 , TFLOPS: 71.51189353687252, Tokens per sec: 177214.3498697637, Loss: 2.522001028060913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7348 , TFLOPS: 68.66691191518848, Tokens per sec: 170164.1720106331, Loss: 2.4943439960479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7349 , TFLOPS: 68.31009709864375, Tokens per sec: 169279.94558883843, Loss: 2.495180130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7350 , TFLOPS: 71.51932037679259, Tokens per sec: 177232.75439721963, Loss: 2.5101211071014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7351 , TFLOPS: 70.59909680384439, Tokens per sec: 174952.339012459, Loss: 2.5278239250183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7352 , TFLOPS: 68.6070643756155, Tokens per sec: 170015.8632148205, Loss: 2.512678384780884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7353 , TFLOPS: 70.64434758746067, Tokens per sec: 175064.47543904546, Loss: 2.5089364051818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7354 , TFLOPS: 71.52340671235166, Tokens per sec: 177242.880786882, Loss: 2.491712808609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7355 , TFLOPS: 67.8448409725991, Tokens per sec: 168126.98965630648, Loss: 2.506272554397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7356 , TFLOPS: 69.64693592981277, Tokens per sec: 172592.77947742955, Loss: 2.516008138656616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7357 , TFLOPS: 71.54678565536841, Tokens per sec: 177300.8163830807, Loss: 2.5039639472961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7358 , TFLOPS: 69.84707445466908, Tokens per sec: 173088.74478910203, Loss: 2.5187923908233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7359 , TFLOPS: 67.22456548039544, Tokens per sec: 166589.87865763967, Loss: 2.4988813400268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7360 , TFLOPS: 71.54811750166104, Tokens per sec: 177304.1168449088, Loss: 2.502220392227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7361 , TFLOPS: 70.57954446594375, Tokens per sec: 174903.8861652734, Loss: 2.501065254211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7362 , TFLOPS: 68.14161767548006, Tokens per sec: 168862.4350186982, Loss: 2.520681858062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7363 , TFLOPS: 70.3935718196232, Tokens per sec: 174443.02546111285, Loss: 2.4862875938415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7364 , TFLOPS: 71.45730860078356, Tokens per sec: 177079.0823851082, Loss: 2.507352828979492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7365 , TFLOPS: 68.9423293970812, Tokens per sec: 170846.6868705035, Loss: 2.52898907661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7366 , TFLOPS: 69.34413555028506, Tokens per sec: 171842.4067807444, Loss: 2.49802303314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7367 , TFLOPS: 70.3384459207035, Tokens per sec: 174306.4174109708, Loss: 2.513230562210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7368 , TFLOPS: 70.57337374095162, Tokens per sec: 174888.59442898142, Loss: 2.4943253993988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7369 , TFLOPS: 67.41751378138612, Tokens per sec: 167068.02580250456, Loss: 2.5110557079315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7370 , TFLOPS: 70.47085535378866, Tokens per sec: 174634.54257225897, Loss: 2.5034995079040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7371 , TFLOPS: 71.48483669990571, Tokens per sec: 177147.30004720332, Loss: 2.50679874420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7372 , TFLOPS: 68.31054752408637, Tokens per sec: 169281.06179270367, Loss: 2.512937068939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7373 , TFLOPS: 70.66015558250756, Tokens per sec: 175103.64939216606, Loss: 2.499336004257202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7374 , TFLOPS: 70.42307273266604, Tokens per sec: 174516.13197342685, Loss: 2.483691930770874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7375 , TFLOPS: 68.2883175796926, Tokens per sec: 169225.9735416654, Loss: 2.499785900115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7376 , TFLOPS: 69.71188938753419, Tokens per sec: 172753.74130087724, Loss: 2.5135903358459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7377 , TFLOPS: 70.58540482540866, Tokens per sec: 174918.4087815993, Loss: 2.5285139083862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7378 , TFLOPS: 69.90294843072147, Tokens per sec: 173227.20665678638, Loss: 2.5187487602233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7379 , TFLOPS: 68.90751816899134, Tokens per sec: 170760.42081252352, Loss: 2.505308151245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7380 , TFLOPS: 70.95900272418031, Tokens per sec: 175844.22552996146, Loss: 2.5338385105133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7381 , TFLOPS: 70.42250767058876, Tokens per sec: 174514.73168735264, Loss: 2.505077362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7382 , TFLOPS: 68.99322252437825, Tokens per sec: 170972.8056462864, Loss: 2.519479990005493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7383 , TFLOPS: 69.86462516197398, Tokens per sec: 173132.2373751736, Loss: 2.50529146194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7384 , TFLOPS: 71.47481522125773, Tokens per sec: 177122.4657191566, Loss: 2.513214588165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7385 , TFLOPS: 68.95787035765801, Tokens per sec: 170885.199082791, Loss: 2.510061025619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7386 , TFLOPS: 67.73738244023534, Tokens per sec: 167860.69557557453, Loss: 2.5003325939178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7387 , TFLOPS: 71.51129373154251, Tokens per sec: 177212.86348608194, Loss: 2.526623249053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7388 , TFLOPS: 70.5964017711095, Tokens per sec: 174945.6604244596, Loss: 2.4832682609558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7389 , TFLOPS: 67.16950048908701, Tokens per sec: 166453.42154326785, Loss: 2.4970617294311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7390 , TFLOPS: 71.47940964253692, Tokens per sec: 177133.85120120374, Loss: 2.5039913654327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7391 , TFLOPS: 71.46930621294916, Tokens per sec: 177108.8137897006, Loss: 2.509030818939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7392 , TFLOPS: 66.94175857795013, Tokens per sec: 165889.05199962805, Loss: 2.504882335662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7393 , TFLOPS: 70.54889894147091, Tokens per sec: 174827.94317974648, Loss: 2.494060516357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7394 , TFLOPS: 71.51307714280827, Tokens per sec: 177217.28297565746, Loss: 2.4874253273010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7395 , TFLOPS: 69.55204858525224, Tokens per sec: 172357.63818490464, Loss: 2.5060014724731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7396 , TFLOPS: 68.3290283073455, Tokens per sec: 169326.85920944612, Loss: 2.508349895477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7397 , TFLOPS: 71.47518487569866, Tokens per sec: 177123.38176358203, Loss: 2.5046019554138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7398 , TFLOPS: 70.66950317818555, Tokens per sec: 175126.81376397877, Loss: 2.5278818607330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7399 , TFLOPS: 68.46701587645833, Tokens per sec: 169668.80760629333, Loss: 2.5051403045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7400 , TFLOPS: 69.58464916591889, Tokens per sec: 172438.426015047, Loss: 2.5113251209259033 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/model.pt +[rank0]:[2024-08-26 17:37:39,130] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006399589001375716, 'preprocessing_with_comm': 0.0015273959943442605, 'state_converting': 0.9947414169946569, : 1.0040648889989825}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400, took 4.29s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005400 + eval ppl=9.798367500305176, eval loss=2.2822158336639404 +------------------------------------------------------------------ +iteration: 7401 , TFLOPS: 65.93839810465634, Tokens per sec: 163402.6142175255, Loss: 2.5163636207580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7402 , TFLOPS: 68.94078261484722, Tokens per sec: 170842.85377373558, Loss: 2.5072128772735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7403 , TFLOPS: 69.41211799330654, Tokens per sec: 172010.874763436, Loss: 2.492307186126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7404 , TFLOPS: 71.48827325516908, Tokens per sec: 177155.81620971375, Loss: 2.5130152702331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7405 , TFLOPS: 69.87419731763345, Tokens per sec: 173155.9582313577, Loss: 2.514085054397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7406 , TFLOPS: 69.0098288374461, Tokens per sec: 171013.95791940493, Loss: 2.4830470085144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7407 , TFLOPS: 70.3851160942028, Tokens per sec: 174422.07124204608, Loss: 2.501404285430908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7408 , TFLOPS: 70.51985203301871, Tokens per sec: 174755.9617408208, Loss: 2.517585515975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7409 , TFLOPS: 68.07551384172906, Tokens per sec: 168698.62243672478, Loss: 2.5098867416381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7410 , TFLOPS: 70.46016493956924, Tokens per sec: 174608.0505481769, Loss: 2.5049307346343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7411 , TFLOPS: 71.48383104941797, Tokens per sec: 177144.80793451317, Loss: 2.5178098678588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7412 , TFLOPS: 68.82095851435494, Tokens per sec: 170545.91645300138, Loss: 2.504843235015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7413 , TFLOPS: 70.49660212709837, Tokens per sec: 174698.34591276167, Loss: 2.5100514888763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7414 , TFLOPS: 70.44190512788234, Tokens per sec: 174562.80072333245, Loss: 2.501410961151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7415 , TFLOPS: 68.53354820360539, Tokens per sec: 169833.6820421046, Loss: 2.520395517349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7416 , TFLOPS: 69.85217652353401, Tokens per sec: 173101.38827778862, Loss: 2.521134853363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7417 , TFLOPS: 71.46916094707048, Tokens per sec: 177108.4538048509, Loss: 2.533266544342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7418 , TFLOPS: 69.48556420788805, Tokens per sec: 172192.88257393742, Loss: 2.5208396911621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7419 , TFLOPS: 68.86825965041693, Tokens per sec: 170663.13387881327, Loss: 2.5069375038146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7420 , TFLOPS: 71.54238448357896, Tokens per sec: 177289.90979455732, Loss: 2.525301933288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7421 , TFLOPS: 70.71396469521181, Tokens per sec: 175236.9943010103, Loss: 2.5018768310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7422 , TFLOPS: 67.75863171963998, Tokens per sec: 167913.3536307406, Loss: 2.49320387840271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7423 , TFLOPS: 70.94092690766068, Tokens per sec: 175799.43166541026, Loss: 2.51021671295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7424 , TFLOPS: 71.51303184907005, Tokens per sec: 177217.17073278464, Loss: 2.4967551231384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7425 , TFLOPS: 69.49525580380048, Tokens per sec: 172216.89941622582, Loss: 2.5032448768615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7426 , TFLOPS: 69.08474360597295, Tokens per sec: 171199.60496835748, Loss: 2.4862499237060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7427 , TFLOPS: 71.51314282793925, Tokens per sec: 177217.44575064784, Loss: 2.5161848068237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7428 , TFLOPS: 70.69259476924837, Tokens per sec: 175184.0372703812, Loss: 2.502480983734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7429 , TFLOPS: 66.27331696251653, Tokens per sec: 164232.58004166093, Loss: 2.490318536758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7430 , TFLOPS: 71.50881982575571, Tokens per sec: 177206.73287501902, Loss: 2.4990365505218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7431 , TFLOPS: 71.53009579654503, Tokens per sec: 177259.4570743766, Loss: 2.5085644721984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7432 , TFLOPS: 68.76161647722158, Tokens per sec: 170398.8603479199, Loss: 2.50996994972229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7433 , TFLOPS: 69.66112374698251, Tokens per sec: 172627.93845129327, Loss: 2.4981162548065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7434 , TFLOPS: 71.51114960858096, Tokens per sec: 177212.50633350678, Loss: 2.4934682846069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7435 , TFLOPS: 68.84168652139854, Tokens per sec: 170597.28273783388, Loss: 2.495906114578247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7436 , TFLOPS: 69.19953307005792, Tokens per sec: 171484.06590546342, Loss: 2.5310420989990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7437 , TFLOPS: 70.4180406431927, Tokens per sec: 174503.66190138148, Loss: 2.5164523124694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7438 , TFLOPS: 70.53178623679284, Tokens per sec: 174785.53601243417, Loss: 2.4961957931518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7439 , TFLOPS: 69.3592445125006, Tokens per sec: 171879.84845350412, Loss: 2.5099174976348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7440 , TFLOPS: 69.93527637708637, Tokens per sec: 173307.3188690959, Loss: 2.4745302200317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7441 , TFLOPS: 71.49955830314116, Tokens per sec: 177183.7818017392, Loss: 2.5365142822265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7442 , TFLOPS: 69.07254278381048, Tokens per sec: 171169.36998700738, Loss: 2.4999074935913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7443 , TFLOPS: 69.99895150307094, Tokens per sec: 173465.1128456799, Loss: 2.498573064804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7444 , TFLOPS: 70.44297221526385, Tokens per sec: 174565.44508341321, Loss: 2.5000510215759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7445 , TFLOPS: 70.6176427007792, Tokens per sec: 174998.29778806266, Loss: 2.491577625274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7446 , TFLOPS: 68.8674176891564, Tokens per sec: 170661.04740606, Loss: 2.4961440563201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7447 , TFLOPS: 70.468528621035, Tokens per sec: 174628.77667218467, Loss: 2.4941189289093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7448 , TFLOPS: 70.74385520231522, Tokens per sec: 175311.06627032373, Loss: 2.5013365745544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7449 , TFLOPS: 67.85212937524129, Tokens per sec: 168145.0511209377, Loss: 2.5124030113220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7450 , TFLOPS: 71.56610646015899, Tokens per sec: 177348.6954657141, Loss: 2.5002617835998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7451 , TFLOPS: 70.55607699720872, Tokens per sec: 174845.7311925929, Loss: 2.518450975418091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7452 , TFLOPS: 68.75134634361649, Tokens per sec: 170373.40982549786, Loss: 2.516538619995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7453 , TFLOPS: 70.6307088268355, Tokens per sec: 175030.6770877828, Loss: 2.47995662689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7454 , TFLOPS: 71.46669231899648, Tokens per sec: 177102.3362725417, Loss: 2.5032386779785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7455 , TFLOPS: 68.77291799950244, Tokens per sec: 170426.86676509728, Loss: 2.5165843963623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7456 , TFLOPS: 69.03800811098068, Tokens per sec: 171083.78926342688, Loss: 2.5021097660064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7457 , TFLOPS: 71.55857783411149, Tokens per sec: 177330.03870102172, Loss: 2.5079526901245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7458 , TFLOPS: 70.52442126146586, Tokens per sec: 174767.2847922549, Loss: 2.5330491065979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7459 , TFLOPS: 69.1055620606629, Tokens per sec: 171251.1953924213, Loss: 2.5122852325439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7460 , TFLOPS: 70.57895635992156, Tokens per sec: 174902.4287737652, Loss: 2.5168051719665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7461 , TFLOPS: 71.51728044179389, Tokens per sec: 177227.69921357543, Loss: 2.5036098957061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7462 , TFLOPS: 68.66151787887884, Tokens per sec: 170150.80499445568, Loss: 2.488922119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7463 , TFLOPS: 69.82445604969483, Tokens per sec: 173032.69388995285, Loss: 2.5143442153930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7464 , TFLOPS: 71.52317286467883, Tokens per sec: 177242.30128658828, Loss: 2.5191574096679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7465 , TFLOPS: 70.5788975545957, Tokens per sec: 174902.28304768977, Loss: 2.5184803009033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7466 , TFLOPS: 68.66696405086965, Tokens per sec: 170164.3012085943, Loss: 2.5139458179473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7467 , TFLOPS: 71.51841638460975, Tokens per sec: 177230.51420500758, Loss: 2.518744468688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7468 , TFLOPS: 70.70713300303325, Tokens per sec: 175220.0646152188, Loss: 2.4989418983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7469 , TFLOPS: 68.80295954269259, Tokens per sec: 170501.3130184717, Loss: 2.5039501190185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7470 , TFLOPS: 69.95984439038818, Tokens per sec: 173368.2011124479, Loss: 2.5180861949920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7471 , TFLOPS: 71.49597036404231, Tokens per sec: 177174.89049340817, Loss: 2.521207571029663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7472 , TFLOPS: 69.69208508664585, Tokens per sec: 172704.66406165212, Loss: 2.536599636077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7473 , TFLOPS: 68.89130753871754, Tokens per sec: 170720.24908495668, Loss: 2.5020389556884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7474 , TFLOPS: 71.54440066576952, Tokens per sec: 177294.90611612634, Loss: 2.492541790008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7475 , TFLOPS: 70.68985582135281, Tokens per sec: 175177.249855776, Loss: 2.5300512313842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7476 , TFLOPS: 68.17071918978012, Tokens per sec: 168934.55177693014, Loss: 2.49180269241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7477 , TFLOPS: 70.47982220482781, Tokens per sec: 174656.76341691415, Loss: 2.510599136352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7478 , TFLOPS: 71.49990559917958, Tokens per sec: 177184.64243958026, Loss: 2.522902011871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7479 , TFLOPS: 68.70925731472515, Tokens per sec: 170269.10857541583, Loss: 2.51521635055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7480 , TFLOPS: 70.55540579656969, Tokens per sec: 174844.06788347053, Loss: 2.4914298057556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7481 , TFLOPS: 70.39060640478279, Tokens per sec: 174435.6768364709, Loss: 2.510073661804199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7482 , TFLOPS: 68.87543826218086, Tokens per sec: 170680.9232695537, Loss: 2.5221128463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7483 , TFLOPS: 69.86566921639108, Tokens per sec: 173134.82465703128, Loss: 2.5088722705841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7484 , TFLOPS: 71.52858670110089, Tokens per sec: 177255.71736962546, Loss: 2.481625556945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7485 , TFLOPS: 69.65830769513602, Tokens per sec: 172620.9599646039, Loss: 2.497790575027466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7486 , TFLOPS: 68.72984666609217, Tokens per sec: 170320.1312562076, Loss: 2.514862060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7487 , TFLOPS: 71.37541334919645, Tokens per sec: 176876.1369301686, Loss: 2.5006937980651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7488 , TFLOPS: 70.4869096151143, Tokens per sec: 174674.32680034678, Loss: 2.48854923248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7489 , TFLOPS: 68.47750664404784, Tokens per sec: 169694.80488403357, Loss: 2.5021729469299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7490 , TFLOPS: 70.53218081752274, Tokens per sec: 174786.51382694955, Loss: 2.50454044342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7491 , TFLOPS: 71.44648371150305, Tokens per sec: 177052.2571170111, Loss: 2.5275375843048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7492 , TFLOPS: 68.76562991083048, Tokens per sec: 170408.80607851894, Loss: 2.506685495376587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7493 , TFLOPS: 69.6638524958395, Tokens per sec: 172634.7005915574, Loss: 2.51922607421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7494 , TFLOPS: 71.53742173339853, Tokens per sec: 177277.61155291816, Loss: 2.50765061378479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7495 , TFLOPS: 69.70950785752365, Tokens per sec: 172747.83960716394, Loss: 2.5151214599609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7496 , TFLOPS: 68.0588912877014, Tokens per sec: 168657.42991671874, Loss: 2.495684862136841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7497 , TFLOPS: 71.54896259509702, Tokens per sec: 177306.21107953793, Loss: 2.5194427967071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7498 , TFLOPS: 71.54939816908396, Tokens per sec: 177307.29047985995, Loss: 2.5107579231262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7499 , TFLOPS: 68.60710113763604, Tokens per sec: 170015.95431515708, Loss: 2.5068488121032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7500 , TFLOPS: 70.62210845744627, Tokens per sec: 175009.36442502736, Loss: 2.494474411010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7501 , TFLOPS: 71.54788204208876, Tokens per sec: 177303.53335015068, Loss: 2.498957633972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7502 , TFLOPS: 68.82105696502565, Tokens per sec: 170546.16042460865, Loss: 2.505253314971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7503 , TFLOPS: 68.82571882373003, Tokens per sec: 170557.7130240242, Loss: 2.5034046173095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7504 , TFLOPS: 71.5075858778541, Tokens per sec: 177203.67501618824, Loss: 2.512979745864868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7505 , TFLOPS: 70.57478486628237, Tokens per sec: 174892.0913529993, Loss: 2.5164661407470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7506 , TFLOPS: 69.60814571979158, Tokens per sec: 172496.65306390266, Loss: 2.5135107040405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7507 , TFLOPS: 69.70996820729027, Tokens per sec: 172748.98040459736, Loss: 2.500271797180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7508 , TFLOPS: 71.50274482032853, Tokens per sec: 177191.67834235338, Loss: 2.527031660079956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7509 , TFLOPS: 68.77054174361663, Tokens per sec: 170420.97814124607, Loss: 2.507033586502075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7510 , TFLOPS: 69.1171016622084, Tokens per sec: 171279.79179624308, Loss: 2.511141777038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7511 , TFLOPS: 71.57900038515126, Tokens per sec: 177380.64803222803, Loss: 2.5045416355133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7512 , TFLOPS: 69.6563934260389, Tokens per sec: 172616.2161949071, Loss: 2.508526563644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7513 , TFLOPS: 69.60262707719764, Tokens per sec: 172482.97725962618, Loss: 2.51029634475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7514 , TFLOPS: 70.4935363983395, Tokens per sec: 174690.74869917927, Loss: 2.5164129734039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7515 , TFLOPS: 70.72689671526005, Tokens per sec: 175269.04127126947, Loss: 2.4957401752471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7516 , TFLOPS: 68.19146545759921, Tokens per sec: 168985.96331397595, Loss: 2.4926860332489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7517 , TFLOPS: 71.51123054190076, Tokens per sec: 177212.70689518878, Loss: 2.5102171897888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7518 , TFLOPS: 70.47189886399006, Tokens per sec: 174637.12850549017, Loss: 2.494187355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7519 , TFLOPS: 68.70494834353414, Tokens per sec: 170258.43047013212, Loss: 2.521395683288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7520 , TFLOPS: 70.56034958120823, Tokens per sec: 174856.31912640764, Loss: 2.501756191253662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7521 , TFLOPS: 69.85030380916531, Tokens per sec: 173096.74748528647, Loss: 2.5202550888061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7522 , TFLOPS: 69.82862259704571, Tokens per sec: 173043.0190532714, Loss: 2.5070602893829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7523 , TFLOPS: 69.25057366353023, Tokens per sec: 171610.5501186733, Loss: 2.492964267730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7524 , TFLOPS: 71.51485660502155, Tokens per sec: 177221.6926790459, Loss: 2.519273042678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7525 , TFLOPS: 70.55460552323432, Tokens per sec: 174842.08471798807, Loss: 2.5055959224700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7526 , TFLOPS: 69.0149842291749, Tokens per sec: 171026.73354802217, Loss: 2.5034451484680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7527 , TFLOPS: 70.60870706207633, Tokens per sec: 174976.15429101512, Loss: 2.5006825923919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7528 , TFLOPS: 71.52766744793391, Tokens per sec: 177253.43935901357, Loss: 2.5160603523254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7529 , TFLOPS: 67.82732733960334, Tokens per sec: 168083.58894445706, Loss: 2.5027852058410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7530 , TFLOPS: 69.89090340866458, Tokens per sec: 173197.3577652607, Loss: 2.5146055221557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7531 , TFLOPS: 71.51683035236641, Tokens per sec: 177226.58384239272, Loss: 2.51173996925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7532 , TFLOPS: 69.67687156138196, Tokens per sec: 172666.96327014003, Loss: 2.496154308319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7533 , TFLOPS: 69.60085815686932, Tokens per sec: 172478.59368018995, Loss: 2.517787218093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7534 , TFLOPS: 71.54048007801849, Tokens per sec: 177285.1904677909, Loss: 2.526657819747925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7535 , TFLOPS: 70.61453793568074, Tokens per sec: 174990.6038381306, Loss: 2.5130674839019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7536 , TFLOPS: 67.17384691063604, Tokens per sec: 166464.19245466418, Loss: 2.5242772102355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7537 , TFLOPS: 71.52958926248047, Tokens per sec: 177258.20182716532, Loss: 2.5098249912261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7538 , TFLOPS: 71.53783154167161, Tokens per sec: 177278.62710296296, Loss: 2.502748727798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7539 , TFLOPS: 69.77838893393381, Tokens per sec: 172918.5345023853, Loss: 2.5021510124206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7540 , TFLOPS: 68.97627821396279, Tokens per sec: 170930.815778507, Loss: 2.5061659812927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7541 , TFLOPS: 71.50391222782578, Tokens per sec: 177194.5713067321, Loss: 2.5052897930145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7542 , TFLOPS: 70.58778279075392, Tokens per sec: 174924.3016416794, Loss: 2.540374279022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7543 , TFLOPS: 68.34713618031645, Tokens per sec: 169371.73251341505, Loss: 2.4847097396850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7544 , TFLOPS: 70.40779353658989, Tokens per sec: 174478.2684424078, Loss: 2.5241715908050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7545 , TFLOPS: 71.47796837314625, Tokens per sec: 177130.27957688473, Loss: 2.5256059169769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7546 , TFLOPS: 68.72206446968796, Tokens per sec: 170300.84611623827, Loss: 2.5237653255462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7547 , TFLOPS: 69.73271331823321, Tokens per sec: 172805.34529509477, Loss: 2.5023677349090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7548 , TFLOPS: 71.53379840750475, Tokens per sec: 177268.63255221106, Loss: 2.5066168308258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7549 , TFLOPS: 68.7253347069876, Tokens per sec: 170308.95012451315, Loss: 2.504871368408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7550 , TFLOPS: 69.72866542560106, Tokens per sec: 172795.31417124014, Loss: 2.4965744018554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7551 , TFLOPS: 70.4926033662365, Tokens per sec: 174688.43654284603, Loss: 2.482604742050171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7552 , TFLOPS: 69.75658995106487, Tokens per sec: 172864.51422147985, Loss: 2.5239083766937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7553 , TFLOPS: 69.5583430197362, Tokens per sec: 172373.2364869451, Loss: 2.5155763626098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7554 , TFLOPS: 71.54087320694624, Tokens per sec: 177286.1646845806, Loss: 2.4948208332061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7555 , TFLOPS: 69.63383305501998, Tokens per sec: 172560.3091102922, Loss: 2.510990619659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7556 , TFLOPS: 68.89460537897098, Tokens per sec: 170728.4214963918, Loss: 2.5125789642333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7557 , TFLOPS: 70.85301541790867, Tokens per sec: 175581.5773095547, Loss: 2.5148823261260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7558 , TFLOPS: 71.4751259280045, Tokens per sec: 177123.2356847023, Loss: 2.5165762901306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7559 , TFLOPS: 69.64007345364607, Tokens per sec: 172575.7735055243, Loss: 2.518944263458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7560 , TFLOPS: 69.26795388920942, Tokens per sec: 171653.62023249627, Loss: 2.5120105743408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7561 , TFLOPS: 71.5505992716296, Tokens per sec: 177310.26694428042, Loss: 2.518831968307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7562 , TFLOPS: 69.50079493951665, Tokens per sec: 172230.6260047167, Loss: 2.4971330165863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7563 , TFLOPS: 67.95337648833362, Tokens per sec: 168395.9526204707, Loss: 2.5071096420288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7564 , TFLOPS: 71.54558938935861, Tokens per sec: 177297.85190412882, Loss: 2.5130534172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7565 , TFLOPS: 71.51788092711426, Tokens per sec: 177229.18728234823, Loss: 2.5173676013946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7566 , TFLOPS: 67.84965274742132, Tokens per sec: 168138.9137643767, Loss: 2.471334457397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7567 , TFLOPS: 70.71200782369738, Tokens per sec: 175232.14495783034, Loss: 2.509751558303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7568 , TFLOPS: 71.50330714855068, Tokens per sec: 177193.07185363342, Loss: 2.480551242828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7569 , TFLOPS: 69.42992296873652, Tokens per sec: 172054.99745393143, Loss: 2.4965102672576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7570 , TFLOPS: 68.94040557599624, Tokens per sec: 170841.9194299286, Loss: 2.4908485412597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7571 , TFLOPS: 71.48443771987328, Tokens per sec: 177146.3113307317, Loss: 2.512899875640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7572 , TFLOPS: 70.70583688040922, Tokens per sec: 175216.85268057676, Loss: 2.5346646308898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7573 , TFLOPS: 68.87759711792127, Tokens per sec: 170686.273151896, Loss: 2.477543830871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7574 , TFLOPS: 70.88887810417947, Tokens per sec: 175670.4489967348, Loss: 2.507826805114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7575 , TFLOPS: 71.51266158089277, Tokens per sec: 177216.25316745293, Loss: 2.5184977054595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7576 , TFLOPS: 68.99708233141727, Tokens per sec: 170982.37067332194, Loss: 2.5020217895507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7577 , TFLOPS: 69.02650863592513, Tokens per sec: 171055.29229746747, Loss: 2.5240585803985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7578 , TFLOPS: 71.49392535026874, Tokens per sec: 177169.8227240004, Loss: 2.500138282775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7579 , TFLOPS: 70.644304235881, Tokens per sec: 175064.36800905544, Loss: 2.504000186920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7580 , TFLOPS: 68.79486697545944, Tokens per sec: 170481.25874539805, Loss: 2.5301332473754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7581 , TFLOPS: 70.58652565322913, Tokens per sec: 174921.1863164078, Loss: 2.4978702068328857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7582 , TFLOPS: 70.53691524723337, Tokens per sec: 174798.24626531298, Loss: 2.534672975540161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7583 , TFLOPS: 68.00091224512389, Tokens per sec: 168513.75146229286, Loss: 2.5041749477386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7584 , TFLOPS: 70.4982228161708, Tokens per sec: 174702.36215881674, Loss: 2.5038001537323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7585 , TFLOPS: 71.4962253810667, Tokens per sec: 177175.52245368704, Loss: 2.5157103538513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7586 , TFLOPS: 68.992526263755, Tokens per sec: 170971.08023576255, Loss: 2.5395545959472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7587 , TFLOPS: 70.53257118364412, Tokens per sec: 174787.4811972007, Loss: 2.524399995803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7588 , TFLOPS: 70.52098036125881, Tokens per sec: 174758.7578624954, Loss: 2.5037288665771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7589 , TFLOPS: 69.70295298264135, Tokens per sec: 172731.59590512657, Loss: 2.5116031169891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7590 , TFLOPS: 67.99899913768981, Tokens per sec: 168509.01057132502, Loss: 2.523794412612915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7591 , TFLOPS: 71.50663744721635, Tokens per sec: 177201.32470058967, Loss: 2.4915108680725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7592 , TFLOPS: 70.45933022687493, Tokens per sec: 174605.98203816917, Loss: 2.5094494819641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7593 , TFLOPS: 69.14570906312771, Tokens per sec: 171350.68408708647, Loss: 2.5057168006896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7594 , TFLOPS: 70.57608353643985, Tokens per sec: 174895.30960070988, Loss: 2.5261175632476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7595 , TFLOPS: 70.46589667795438, Tokens per sec: 174622.25442729838, Loss: 2.5254297256469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7596 , TFLOPS: 68.10342473165517, Tokens per sec: 168767.7886966, Loss: 2.5282485485076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7597 , TFLOPS: 69.83079835958699, Tokens per sec: 173048.41083252872, Loss: 2.482238292694092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7598 , TFLOPS: 71.52573765839114, Tokens per sec: 177248.65712794272, Loss: 2.520327568054199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7599 , TFLOPS: 69.71689292046237, Tokens per sec: 172766.14060665795, Loss: 2.5009360313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7600 , TFLOPS: 68.91982258784533, Tokens per sec: 170790.91251788734, Loss: 2.512815475463867 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/optimizer.pt +[rank0]:[2024-08-26 18:02:57,663] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006371612995280884, 'preprocessing_with_comm': 0.0014250349995563738, 'state_converting': 0.9670373819972156, : 0.9761799839980085}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600, took 4.29s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005600 + eval ppl=10.622502326965332, eval loss=2.3629746437072754 +------------------------------------------------------------------ +iteration: 7601 , TFLOPS: 65.87778154932491, Tokens per sec: 163252.39971594934, Loss: 2.4997217655181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7602 , TFLOPS: 70.54195968885013, Tokens per sec: 174810.74694733042, Loss: 2.4886293411254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7603 , TFLOPS: 66.44549820186022, Tokens per sec: 164659.2641200839, Loss: 2.509260892868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7604 , TFLOPS: 71.49167672903621, Tokens per sec: 177164.2503928801, Loss: 2.5155701637268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7605 , TFLOPS: 71.55481797849767, Tokens per sec: 177320.72136462844, Loss: 2.5081183910369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7606 , TFLOPS: 67.82734226827107, Tokens per sec: 168083.6259393402, Loss: 2.5089287757873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7607 , TFLOPS: 70.7207948358268, Tokens per sec: 175253.92014185633, Loss: 2.505476474761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7608 , TFLOPS: 71.5062470510071, Tokens per sec: 177200.35725577743, Loss: 2.5251822471618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7609 , TFLOPS: 69.67914616228558, Tokens per sec: 172672.59998174716, Loss: 2.519094944000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7610 , TFLOPS: 68.06007732629838, Tokens per sec: 168660.36905101503, Loss: 2.515397310256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7611 , TFLOPS: 71.492537866839, Tokens per sec: 177166.3843872176, Loss: 2.4999423027038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7612 , TFLOPS: 71.4616138213674, Tokens per sec: 177089.7511959741, Loss: 2.5056750774383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7613 , TFLOPS: 68.16981150492694, Tokens per sec: 168932.30243387425, Loss: 2.510401725769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7614 , TFLOPS: 70.39622541357575, Tokens per sec: 174449.60135924479, Loss: 2.516066551208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7615 , TFLOPS: 71.48996927537544, Tokens per sec: 177160.01913461718, Loss: 2.4995577335357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7616 , TFLOPS: 68.05567167341796, Tokens per sec: 168649.45135785456, Loss: 2.518472671508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7617 , TFLOPS: 70.89913981768387, Tokens per sec: 175695.87865321976, Loss: 2.5373682975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7618 , TFLOPS: 70.46728337147168, Tokens per sec: 174625.69080659113, Loss: 2.5169973373413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7619 , TFLOPS: 70.61463381468717, Tokens per sec: 174990.84143687045, Loss: 2.499939441680908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7620 , TFLOPS: 69.15916291423792, Tokens per sec: 171384.02421220165, Loss: 2.498310089111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7621 , TFLOPS: 70.38221499703374, Tokens per sec: 174414.8820036789, Loss: 2.488489866256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7622 , TFLOPS: 70.67364514969371, Tokens per sec: 175137.07802565183, Loss: 2.5160980224609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7623 , TFLOPS: 68.06893985114402, Tokens per sec: 168682.3313638702, Loss: 2.530283212661743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7624 , TFLOPS: 71.48668410975094, Tokens per sec: 177151.8781323088, Loss: 2.5071465969085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7625 , TFLOPS: 70.44595931928357, Tokens per sec: 174572.84745623078, Loss: 2.498016595840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7626 , TFLOPS: 69.05290943395099, Tokens per sec: 171120.7163832635, Loss: 2.5109236240386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7627 , TFLOPS: 69.84855077056847, Tokens per sec: 173092.4032625303, Loss: 2.498842239379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7628 , TFLOPS: 71.43464111143228, Tokens per sec: 177022.90984946562, Loss: 2.5059690475463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7629 , TFLOPS: 69.7257030927318, Tokens per sec: 172787.9731840622, Loss: 2.5037946701049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7630 , TFLOPS: 68.21321738434058, Tokens per sec: 169039.86698461362, Loss: 2.5141899585723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7631 , TFLOPS: 71.48043130468157, Tokens per sec: 177136.38299254354, Loss: 2.4906764030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7632 , TFLOPS: 71.50938963828571, Tokens per sec: 177208.1449332384, Loss: 2.5230355262756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7633 , TFLOPS: 67.75935435100675, Tokens per sec: 167915.14439087186, Loss: 2.494450092315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7634 , TFLOPS: 70.96861643474132, Tokens per sec: 175868.0493637693, Loss: 2.5082204341888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7635 , TFLOPS: 71.55269049398484, Tokens per sec: 177315.44922364387, Loss: 2.497136354446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7636 , TFLOPS: 67.65153542948998, Tokens per sec: 167647.95722611863, Loss: 2.5105319023132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7637 , TFLOPS: 69.99043312671006, Tokens per sec: 173444.003370681, Loss: 2.4903781414031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7638 , TFLOPS: 71.47012071392135, Tokens per sec: 177110.8322128344, Loss: 2.513240337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7639 , TFLOPS: 70.59901700891258, Tokens per sec: 174952.14127182783, Loss: 2.510812759399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7640 , TFLOPS: 67.99440877738806, Tokens per sec: 168497.6351528274, Loss: 2.5179598331451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7641 , TFLOPS: 71.53866577826642, Tokens per sec: 177280.69443314365, Loss: 2.5111989974975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7642 , TFLOPS: 71.5486037318243, Tokens per sec: 177305.32177681604, Loss: 2.5025546550750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7643 , TFLOPS: 67.36944074129724, Tokens per sec: 166948.89551348175, Loss: 2.479137659072876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7644 , TFLOPS: 70.51386778454287, Tokens per sec: 174741.1321138783, Loss: 2.4948792457580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7645 , TFLOPS: 71.51588130944437, Tokens per sec: 177224.23200948615, Loss: 2.5025603771209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7646 , TFLOPS: 68.98276624026323, Tokens per sec: 170946.8938223946, Loss: 2.499143123626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7647 , TFLOPS: 69.74007345187204, Tokens per sec: 172823.58451703744, Loss: 2.508938789367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7648 , TFLOPS: 70.45662142896074, Tokens per sec: 174599.26933853913, Loss: 2.5311594009399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7649 , TFLOPS: 70.77679944201078, Tokens per sec: 175392.7057253969, Loss: 2.5167882442474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7650 , TFLOPS: 67.87637097450771, Tokens per sec: 168205.12447435246, Loss: 2.5045127868652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7651 , TFLOPS: 70.5124041936043, Tokens per sec: 174737.50517430465, Loss: 2.5366415977478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7652 , TFLOPS: 71.4659021192563, Tokens per sec: 177100.37807053782, Loss: 2.508244752883911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7653 , TFLOPS: 68.83373935154682, Tokens per sec: 170577.58877548832, Loss: 2.518843650817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7654 , TFLOPS: 70.57409618787425, Tokens per sec: 174890.38473203976, Loss: 2.4894511699676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7655 , TFLOPS: 70.41242372330184, Tokens per sec: 174489.74255513717, Loss: 2.5150671005249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7656 , TFLOPS: 68.86384434883719, Tokens per sec: 170652.19227511305, Loss: 2.5063700675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7657 , TFLOPS: 69.86001594501668, Tokens per sec: 173120.81522780625, Loss: 2.506088972091675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7658 , TFLOPS: 70.49557908022312, Tokens per sec: 174695.81068990688, Loss: 2.5001299381256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7659 , TFLOPS: 70.60201411361038, Tokens per sec: 174959.56842742732, Loss: 2.4840168952941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7660 , TFLOPS: 68.83216147241639, Tokens per sec: 170573.67861718533, Loss: 2.4949636459350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7661 , TFLOPS: 71.53419138564503, Tokens per sec: 177269.6063953328, Loss: 2.519314765930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7662 , TFLOPS: 70.52187412765515, Tokens per sec: 174760.97271408202, Loss: 2.5045502185821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7663 , TFLOPS: 67.14713447267725, Tokens per sec: 166397.99609078432, Loss: 2.5246989727020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7664 , TFLOPS: 71.50906955464117, Tokens per sec: 177207.3517307071, Loss: 2.527672052383423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7665 , TFLOPS: 71.50437349214488, Tokens per sec: 177195.7143705273, Loss: 2.4924771785736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7666 , TFLOPS: 67.97715201540348, Tokens per sec: 168454.87099564032, Loss: 2.4894092082977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7667 , TFLOPS: 69.51373621261554, Tokens per sec: 172262.6959050547, Loss: 2.5220694541931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7668 , TFLOPS: 71.48081819563997, Tokens per sec: 177137.34175095803, Loss: 2.4950644969940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7669 , TFLOPS: 70.67454186548066, Tokens per sec: 175139.3001861533, Loss: 2.4967103004455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7670 , TFLOPS: 66.87101052581795, Tokens per sec: 165713.7305329032, Loss: 2.504791259765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7671 , TFLOPS: 71.57249825687178, Tokens per sec: 177364.53504208292, Loss: 2.5134105682373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7672 , TFLOPS: 71.49768270028493, Tokens per sec: 177179.13385124676, Loss: 2.5020079612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7673 , TFLOPS: 67.3219970664262, Tokens per sec: 166831.32486079913, Loss: 2.5219156742095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7674 , TFLOPS: 70.71213909127435, Tokens per sec: 175232.4702533464, Loss: 2.4951083660125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7675 , TFLOPS: 71.54115503233977, Tokens per sec: 177286.8630789514, Loss: 2.524951457977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7676 , TFLOPS: 69.81439839371268, Tokens per sec: 173007.7699104868, Loss: 2.5119476318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7677 , TFLOPS: 68.19378649175478, Tokens per sec: 168991.71509229648, Loss: 2.520230770111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7678 , TFLOPS: 71.49309459930929, Tokens per sec: 177167.7640316081, Loss: 2.488220453262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7679 , TFLOPS: 71.52163074726083, Tokens per sec: 177238.47974974822, Loss: 2.493529796600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7680 , TFLOPS: 68.21941431210125, Tokens per sec: 169055.2236542525, Loss: 2.504828453063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7681 , TFLOPS: 69.30304075433357, Tokens per sec: 171740.56934941115, Loss: 2.4816715717315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7682 , TFLOPS: 71.48994066293133, Tokens per sec: 177159.94822982873, Loss: 2.519848346710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7683 , TFLOPS: 68.08923325053001, Tokens per sec: 168732.6206430512, Loss: 2.4961585998535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7684 , TFLOPS: 69.82971845119832, Tokens per sec: 173045.73470057984, Loss: 2.5055599212646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7685 , TFLOPS: 71.53644963200135, Tokens per sec: 177275.20257857017, Loss: 2.511605978012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7686 , TFLOPS: 69.71423280076118, Tokens per sec: 172759.54853700203, Loss: 2.4961185455322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7687 , TFLOPS: 68.95614417927563, Tokens per sec: 170880.9214226052, Loss: 2.5019493103027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7688 , TFLOPS: 69.85258562810975, Tokens per sec: 173102.40208399386, Loss: 2.498094081878662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7689 , TFLOPS: 70.60644698978261, Tokens per sec: 174970.55358290375, Loss: 2.517235279083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7690 , TFLOPS: 68.1884863319972, Tokens per sec: 168978.58071255637, Loss: 2.5157456398010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7691 , TFLOPS: 71.4437899056437, Tokens per sec: 177045.58156934436, Loss: 2.4776673316955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7692 , TFLOPS: 70.07121961646712, Tokens per sec: 173644.2011916655, Loss: 2.513028621673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7693 , TFLOPS: 69.70320884071727, Tokens per sec: 172732.22994962375, Loss: 2.498800754547119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7694 , TFLOPS: 69.72057247320274, Tokens per sec: 172775.25894368478, Loss: 2.5060224533081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7695 , TFLOPS: 70.56471389307016, Tokens per sec: 174867.13437197017, Loss: 2.5124733448028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7696 , TFLOPS: 70.68074308129273, Tokens per sec: 175154.66748205593, Loss: 2.487653970718384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7697 , TFLOPS: 68.46824650019627, Tokens per sec: 169671.85722748033, Loss: 2.48388671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7698 , TFLOPS: 71.45981099460293, Tokens per sec: 177085.283592654, Loss: 2.5181522369384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7699 , TFLOPS: 70.48589632755548, Tokens per sec: 174671.81576215322, Loss: 2.5213077068328857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7700 , TFLOPS: 68.79385591267528, Tokens per sec: 170478.75322044114, Loss: 2.5166568756103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7701 , TFLOPS: 70.84934831983097, Tokens per sec: 175572.489836554, Loss: 2.4934422969818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7702 , TFLOPS: 71.54430031169295, Tokens per sec: 177294.65742766968, Loss: 2.5018398761749268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7703 , TFLOPS: 67.93119077366077, Tokens per sec: 168340.97397555315, Loss: 2.5088067054748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7704 , TFLOPS: 70.27182211868569, Tokens per sec: 174141.31629035296, Loss: 2.5049691200256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7705 , TFLOPS: 71.54647565958787, Tokens per sec: 177300.04817938764, Loss: 2.5320096015930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7706 , TFLOPS: 69.72260279579903, Tokens per sec: 172780.2903067369, Loss: 2.523589849472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7707 , TFLOPS: 67.81562042160822, Tokens per sec: 168054.57791204166, Loss: 2.509784460067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7708 , TFLOPS: 71.54293304995998, Tokens per sec: 177291.26920247907, Loss: 2.5090994834899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7709 , TFLOPS: 70.60054458288815, Tokens per sec: 174955.92676841543, Loss: 2.524029016494751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7710 , TFLOPS: 66.77566191945594, Tokens per sec: 165477.44618282205, Loss: 2.5114338397979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7711 , TFLOPS: 71.46464506321811, Tokens per sec: 177097.26294719905, Loss: 2.471865177154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7712 , TFLOPS: 71.54253618610194, Tokens per sec: 177290.2857301207, Loss: 2.530272960662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7713 , TFLOPS: 69.7262615367596, Tokens per sec: 172789.35706988076, Loss: 2.5213799476623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7714 , TFLOPS: 68.67420565589856, Tokens per sec: 170182.24670358456, Loss: 2.513335704803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7715 , TFLOPS: 71.48102721448495, Tokens per sec: 177137.8597226812, Loss: 2.5058255195617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7716 , TFLOPS: 70.55844154294866, Tokens per sec: 174851.59079741267, Loss: 2.5014853477478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7717 , TFLOPS: 68.19712361809394, Tokens per sec: 168999.9848589797, Loss: 2.527193546295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7718 , TFLOPS: 70.44518187608041, Tokens per sec: 174570.9208663309, Loss: 2.4839682579040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7719 , TFLOPS: 71.49618886582313, Tokens per sec: 177175.431964891, Loss: 2.5286331176757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7720 , TFLOPS: 68.7976388302333, Tokens per sec: 170488.12770687268, Loss: 2.5060768127441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7721 , TFLOPS: 70.55611581184775, Tokens per sec: 174845.8273795442, Loss: 2.4880480766296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7722 , TFLOPS: 70.33508322522792, Tokens per sec: 174298.0842811508, Loss: 2.4955554008483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7723 , TFLOPS: 69.79260456597859, Tokens per sec: 172953.76240457338, Loss: 2.5014679431915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7724 , TFLOPS: 69.12861526876944, Tokens per sec: 171308.32378162976, Loss: 2.520883083343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7725 , TFLOPS: 70.51249846187117, Tokens per sec: 174737.73878145448, Loss: 2.520630359649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7726 , TFLOPS: 70.65359939074374, Tokens per sec: 175087.40242675118, Loss: 2.494412899017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7727 , TFLOPS: 67.93962453175219, Tokens per sec: 168361.8737571588, Loss: 2.510895252227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7728 , TFLOPS: 71.53971397860299, Tokens per sec: 177283.2919890465, Loss: 2.5178301334381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7729 , TFLOPS: 70.58398410047879, Tokens per sec: 174914.88806871732, Loss: 2.5104026794433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7730 , TFLOPS: 68.128719624154, Tokens per sec: 168830.47222667458, Loss: 2.508180618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7731 , TFLOPS: 70.96147192672326, Tokens per sec: 175850.344485868, Loss: 2.4992587566375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7732 , TFLOPS: 70.4789397211696, Tokens per sec: 174654.57652519553, Loss: 2.4992103576660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7733 , TFLOPS: 70.57269739468329, Tokens per sec: 174886.9183684228, Loss: 2.5275678634643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7734 , TFLOPS: 69.02043256156708, Tokens per sec: 171040.23511587334, Loss: 2.5079073905944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7735 , TFLOPS: 71.55213180803331, Tokens per sec: 177314.06473831166, Loss: 2.4931740760803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7736 , TFLOPS: 69.67683611975093, Tokens per sec: 172666.87544187412, Loss: 2.498347759246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7737 , TFLOPS: 68.00736691000043, Tokens per sec: 168529.74683289696, Loss: 2.483524799346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7738 , TFLOPS: 71.52409821539167, Tokens per sec: 177244.5944075905, Loss: 2.498849391937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7739 , TFLOPS: 71.51836258245012, Tokens per sec: 177230.3808773292, Loss: 2.5293033123016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7740 , TFLOPS: 67.29533970285686, Tokens per sec: 166765.26497732327, Loss: 2.52866530418396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7741 , TFLOPS: 70.68640397834396, Tokens per sec: 175168.6958085482, Loss: 2.5112664699554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7742 , TFLOPS: 71.50857835641337, Tokens per sec: 177206.13448738866, Loss: 2.5259766578674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7743 , TFLOPS: 69.75573531962921, Tokens per sec: 172862.39635063687, Loss: 2.52348256111145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7744 , TFLOPS: 68.6145113211144, Tokens per sec: 170034.31756611558, Loss: 2.4885425567626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7745 , TFLOPS: 71.51267532632689, Tokens per sec: 177216.28723015284, Loss: 2.488180160522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7746 , TFLOPS: 71.47809034079305, Tokens per sec: 177130.58182614934, Loss: 2.52994704246521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7747 , TFLOPS: 66.97853877805878, Tokens per sec: 165980.1973871708, Loss: 2.5069093704223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7748 , TFLOPS: 71.53485963919947, Tokens per sec: 177271.2624012549, Loss: 2.507774591445923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7749 , TFLOPS: 71.49304924391879, Tokens per sec: 177167.65163595413, Loss: 2.519174337387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7750 , TFLOPS: 67.96875422522807, Tokens per sec: 168434.06034649268, Loss: 2.5045974254608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7751 , TFLOPS: 69.5974583054269, Tokens per sec: 172470.1684737911, Loss: 2.492096424102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7752 , TFLOPS: 71.5192779639101, Tokens per sec: 177232.64929342462, Loss: 2.520890474319458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7753 , TFLOPS: 70.61549670888483, Tokens per sec: 174992.9797837479, Loss: 2.52026629447937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7754 , TFLOPS: 68.92202789496027, Tokens per sec: 170796.37751185277, Loss: 2.4877712726593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7755 , TFLOPS: 70.48294953375371, Tokens per sec: 174664.51328249628, Loss: 2.5126185417175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7756 , TFLOPS: 70.61085025872103, Tokens per sec: 174981.46536839326, Loss: 2.51908540725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7757 , TFLOPS: 67.94537451834361, Tokens per sec: 168376.12285734568, Loss: 2.5109949111938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7758 , TFLOPS: 70.51917623320348, Tokens per sec: 174754.2870344332, Loss: 2.5011587142944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7759 , TFLOPS: 71.47901279788503, Tokens per sec: 177132.86777644034, Loss: 2.512155532836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7760 , TFLOPS: 69.82501085517104, Tokens per sec: 173034.06875903974, Loss: 2.4964752197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7761 , TFLOPS: 69.92916285087584, Tokens per sec: 173292.1688776857, Loss: 2.501178741455078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7762 , TFLOPS: 70.62403930997046, Tokens per sec: 175014.14929028327, Loss: 2.515040874481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7763 , TFLOPS: 70.47909285808392, Tokens per sec: 174654.95601533863, Loss: 2.497365951538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7764 , TFLOPS: 68.26378414043674, Tokens per sec: 169165.1769766079, Loss: 2.504885196685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7765 , TFLOPS: 71.5586323930571, Tokens per sec: 177330.17390409915, Loss: 2.5113258361816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7766 , TFLOPS: 70.41527778781449, Tokens per sec: 174496.81524140065, Loss: 2.5107805728912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7767 , TFLOPS: 68.09914750870028, Tokens per sec: 168757.18926694285, Loss: 2.508843421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7768 , TFLOPS: 70.83158167745383, Tokens per sec: 175528.4621395862, Loss: 2.5321033000946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7769 , TFLOPS: 70.60872226076062, Tokens per sec: 174976.191955029, Loss: 2.4717509746551514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7770 , TFLOPS: 68.74998839255046, Tokens per sec: 170370.0446731656, Loss: 2.5118632316589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7771 , TFLOPS: 70.11671691856264, Tokens per sec: 173756.9485181998, Loss: 2.506328582763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7772 , TFLOPS: 71.53425428731634, Tokens per sec: 177269.76227260355, Loss: 2.4937384128570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7773 , TFLOPS: 69.38875247976203, Tokens per sec: 171952.9724469489, Loss: 2.4762585163116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7774 , TFLOPS: 68.7945955107306, Tokens per sec: 170480.58602589567, Loss: 2.509453058242798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7775 , TFLOPS: 71.55921007969901, Tokens per sec: 177331.60547523524, Loss: 2.493943691253662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7776 , TFLOPS: 70.63062094172594, Tokens per sec: 175030.45929879995, Loss: 2.523110866546631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7777 , TFLOPS: 66.78653616022598, Tokens per sec: 165504.3937493467, Loss: 2.527015447616577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7778 , TFLOPS: 71.51444704060327, Tokens per sec: 177220.67773330017, Loss: 2.5149495601654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7779 , TFLOPS: 71.48529783402458, Tokens per sec: 177148.44278834798, Loss: 2.5166525840759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7780 , TFLOPS: 69.82198578171787, Tokens per sec: 173026.57229378345, Loss: 2.509253978729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7781 , TFLOPS: 68.02529711459347, Tokens per sec: 168574.1798550536, Loss: 2.5096940994262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7782 , TFLOPS: 71.50752623926914, Tokens per sec: 177203.527225205, Loss: 2.512373208999634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7783 , TFLOPS: 70.59814313582206, Tokens per sec: 174949.97571804447, Loss: 2.4901580810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7784 , TFLOPS: 66.57240233741113, Tokens per sec: 164973.74654762365, Loss: 2.5154621601104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7785 , TFLOPS: 71.46270850596856, Tokens per sec: 177092.46394500526, Loss: 2.4767227172851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7786 , TFLOPS: 71.51498846502021, Tokens per sec: 177222.0194426482, Loss: 2.5147571563720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7787 , TFLOPS: 68.04689927457078, Tokens per sec: 168627.71238715178, Loss: 2.483297824859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7788 , TFLOPS: 69.73456437500042, Tokens per sec: 172809.93241767466, Loss: 2.503528594970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7789 , TFLOPS: 71.43423404596052, Tokens per sec: 177021.90109638462, Loss: 2.493469715118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7790 , TFLOPS: 68.53237456128642, Tokens per sec: 169830.7736271508, Loss: 2.50315260887146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7791 , TFLOPS: 69.73701182553114, Tokens per sec: 172815.99746969872, Loss: 2.4954042434692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7792 , TFLOPS: 70.56501358681973, Tokens per sec: 174867.87704609556, Loss: 2.4962353706359863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7793 , TFLOPS: 70.65754599752731, Tokens per sec: 175097.18255311548, Loss: 2.5283541679382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7794 , TFLOPS: 68.7817535448434, Tokens per sec: 170448.76221976787, Loss: 2.5121545791625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7795 , TFLOPS: 71.50261759253347, Tokens per sec: 177191.36305786192, Loss: 2.5057172775268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7796 , TFLOPS: 70.47429613911677, Tokens per sec: 174643.06921733584, Loss: 2.523207426071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7797 , TFLOPS: 67.45403670262709, Tokens per sec: 167158.5336246716, Loss: 2.4977245330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7798 , TFLOPS: 71.49191405852184, Tokens per sec: 177164.83852148952, Loss: 2.5185887813568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7799 , TFLOPS: 70.51307687723333, Tokens per sec: 174739.17215843964, Loss: 2.5196306705474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7800 , TFLOPS: 69.84840457280104, Tokens per sec: 173092.04096835764, Loss: 2.514369249343872 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/optimizer.pt +[rank0]:[2024-08-26 18:28:17,608] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00637167900276836, 'preprocessing_with_comm': 0.0012043449969496578, 'state_converting': 0.9851733409959706, : 0.9941113159948145}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800, took 4.37s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0005800 + eval ppl=10.222455978393555, eval loss=2.324586868286133 +------------------------------------------------------------------ +iteration: 7801 , TFLOPS: 67.01543515103775, Tokens per sec: 166071.6306638837, Loss: 2.4960241317749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7802 , TFLOPS: 71.50566805582791, Tokens per sec: 177198.9224419418, Loss: 2.5116488933563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7803 , TFLOPS: 70.06097009494434, Tokens per sec: 173618.80174825402, Loss: 2.5162363052368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7804 , TFLOPS: 68.62754777145923, Tokens per sec: 170066.62332615364, Loss: 2.478053569793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7805 , TFLOPS: 70.68827563254099, Tokens per sec: 175173.33397382815, Loss: 2.494610548019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7806 , TFLOPS: 70.31428185595088, Tokens per sec: 174246.53619662233, Loss: 2.530646324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7807 , TFLOPS: 69.71756190446678, Tokens per sec: 172767.79842271554, Loss: 2.511080741882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7808 , TFLOPS: 69.41896238382454, Tokens per sec: 172027.83591711143, Loss: 2.5084774494171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7809 , TFLOPS: 71.44796472842135, Tokens per sec: 177055.92724008212, Loss: 2.4826552867889404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7810 , TFLOPS: 69.57521595870506, Tokens per sec: 172415.0494884176, Loss: 2.5096681118011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7811 , TFLOPS: 69.56859362867763, Tokens per sec: 172398.63862509976, Loss: 2.5054771900177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7812 , TFLOPS: 71.5217427285794, Tokens per sec: 177238.75725179136, Loss: 2.512044906616211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7813 , TFLOPS: 71.52038738167671, Tokens per sec: 177235.39855286275, Loss: 2.505704879760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7814 , TFLOPS: 67.76313534627045, Tokens per sec: 167924.51411364635, Loss: 2.5116140842437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7815 , TFLOPS: 71.48154646625898, Tokens per sec: 177139.14648577754, Loss: 2.521845579147339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7816 , TFLOPS: 71.47009995147651, Tokens per sec: 177110.7807612089, Loss: 2.511251449584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7817 , TFLOPS: 69.05134726600528, Tokens per sec: 171116.84515900703, Loss: 2.503679037094116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7818 , TFLOPS: 68.52725074735464, Tokens per sec: 169818.07625179316, Loss: 2.5204789638519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7819 , TFLOPS: 71.47641703011293, Tokens per sec: 177126.43517795362, Loss: 2.490429162979126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7820 , TFLOPS: 69.06490516452764, Tokens per sec: 171150.4431250713, Loss: 2.4965555667877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7821 , TFLOPS: 69.41225619126803, Tokens per sec: 172011.21723320833, Loss: 2.4968719482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7822 , TFLOPS: 71.41671435969022, Tokens per sec: 176978.48538385407, Loss: 2.494438409805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7823 , TFLOPS: 70.6346073129899, Tokens per sec: 175040.3379659271, Loss: 2.485487222671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7824 , TFLOPS: 69.78122299017765, Tokens per sec: 172925.55760595461, Loss: 2.5080485343933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7825 , TFLOPS: 68.75482247117611, Tokens per sec: 170382.02405251766, Loss: 2.4952621459960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7826 , TFLOPS: 71.48971222755434, Tokens per sec: 177159.38214180034, Loss: 2.50748872756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7827 , TFLOPS: 69.03975622533069, Tokens per sec: 171088.12128335686, Loss: 2.491971015930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7828 , TFLOPS: 68.51594308004601, Tokens per sec: 169790.05460656, Loss: 2.4822940826416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7829 , TFLOPS: 71.46881720088992, Tokens per sec: 177107.6019639487, Loss: 2.509932518005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7830 , TFLOPS: 70.52372014139628, Tokens per sec: 174765.54733948776, Loss: 2.4965384006500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7831 , TFLOPS: 69.06265790323214, Tokens per sec: 171144.87416402475, Loss: 2.515608549118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7832 , TFLOPS: 70.36620448590747, Tokens per sec: 174375.20619340462, Loss: 2.515688419342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7833 , TFLOPS: 71.50890054122202, Tokens per sec: 177206.9328968361, Loss: 2.5182244777679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7834 , TFLOPS: 68.03695906780405, Tokens per sec: 168603.07945977963, Loss: 2.492440700531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7835 , TFLOPS: 71.53231711606367, Tokens per sec: 177264.96174884235, Loss: 2.513240098953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7836 , TFLOPS: 70.52634804430835, Tokens per sec: 174772.05957239127, Loss: 2.4928579330444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7837 , TFLOPS: 68.98169461451532, Tokens per sec: 170944.23821574196, Loss: 2.532008171081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7838 , TFLOPS: 69.63596551790118, Tokens per sec: 172565.593588222, Loss: 2.531308889389038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7839 , TFLOPS: 71.50705349925009, Tokens per sec: 177202.3557233612, Loss: 2.5150036811828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7840 , TFLOPS: 68.64407704825639, Tokens per sec: 170107.58469490716, Loss: 2.5134432315826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7841 , TFLOPS: 69.19049590390345, Tokens per sec: 171461.6708122066, Loss: 2.483386516571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7842 , TFLOPS: 71.45440749623585, Tokens per sec: 177071.8931284559, Loss: 2.5123496055603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7843 , TFLOPS: 70.42723447973671, Tokens per sec: 174526.4452411223, Loss: 2.510054111480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7844 , TFLOPS: 69.19341957986846, Tokens per sec: 171468.91600331737, Loss: 2.5050888061523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7845 , TFLOPS: 69.69227991316099, Tokens per sec: 172705.14686321854, Loss: 2.4755637645721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7846 , TFLOPS: 71.52191271098552, Tokens per sec: 177239.17848691886, Loss: 2.4666566848754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7847 , TFLOPS: 68.81168735649362, Tokens per sec: 170522.9415025187, Loss: 2.5185909271240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7848 , TFLOPS: 69.4879304243292, Tokens per sec: 172198.74631893885, Loss: 2.5038154125213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7849 , TFLOPS: 71.44928379576365, Tokens per sec: 177059.1960342636, Loss: 2.491567850112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7850 , TFLOPS: 70.54867344394727, Tokens per sec: 174827.3843720423, Loss: 2.5058984756469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7851 , TFLOPS: 68.71264999675824, Tokens per sec: 170277.5160152286, Loss: 2.502897262573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7852 , TFLOPS: 71.53856938425524, Tokens per sec: 177280.45555816527, Loss: 2.500443935394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7853 , TFLOPS: 71.50219918763734, Tokens per sec: 177190.32620443823, Loss: 2.492971181869507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7854 , TFLOPS: 68.94646082495676, Tokens per sec: 170856.92500389353, Loss: 2.502323627471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7855 , TFLOPS: 70.38056217568935, Tokens per sec: 174410.7861303142, Loss: 2.492344617843628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7856 , TFLOPS: 71.48915901332607, Tokens per sec: 177158.0112160011, Loss: 2.523629903793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7857 , TFLOPS: 68.91444298913481, Tokens per sec: 170777.58127967094, Loss: 2.5112860202789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7858 , TFLOPS: 69.44865353553953, Tokens per sec: 172101.4138618116, Loss: 2.506453514099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7859 , TFLOPS: 71.5490822911404, Tokens per sec: 177306.50769951896, Loss: 2.5209786891937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7860 , TFLOPS: 70.66896483032079, Tokens per sec: 175125.47967866602, Loss: 2.486128807067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7861 , TFLOPS: 68.43942290152472, Tokens per sec: 169600.4291747907, Loss: 2.5003626346588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7862 , TFLOPS: 70.43430951803742, Tokens per sec: 174543.97796541124, Loss: 2.517603874206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7863 , TFLOPS: 71.53679841734275, Tokens per sec: 177276.06690706802, Loss: 2.495450735092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7864 , TFLOPS: 68.93177839317343, Tokens per sec: 170820.54032053106, Loss: 2.501070737838745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7865 , TFLOPS: 68.08453737419957, Tokens per sec: 168720.98374421085, Loss: 2.4887189865112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7866 , TFLOPS: 71.50989286308554, Tokens per sec: 177209.39197972717, Loss: 2.4836816787719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7867 , TFLOPS: 69.0075830901002, Tokens per sec: 171008.3927100923, Loss: 2.5178298950195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7868 , TFLOPS: 69.62707211778584, Tokens per sec: 172543.55476304732, Loss: 2.4942078590393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7869 , TFLOPS: 70.43153809633218, Tokens per sec: 174537.11007712834, Loss: 2.528272867202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7870 , TFLOPS: 70.56148545956854, Tokens per sec: 174859.13395811187, Loss: 2.482970714569092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7871 , TFLOPS: 69.75601790597513, Tokens per sec: 172863.0966307315, Loss: 2.4783823490142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7872 , TFLOPS: 70.71268650281014, Tokens per sec: 175233.82679943446, Loss: 2.504232168197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7873 , TFLOPS: 70.56415370347436, Tokens per sec: 174865.7461604418, Loss: 2.508819341659546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7874 , TFLOPS: 69.81985158784973, Tokens per sec: 173021.28352627732, Loss: 2.5134782791137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7875 , TFLOPS: 69.97856429428997, Tokens per sec: 173414.59109648416, Loss: 2.4795665740966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7876 , TFLOPS: 71.53475075801923, Tokens per sec: 177270.99258169602, Loss: 2.505589723587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7877 , TFLOPS: 68.10397392649662, Tokens per sec: 168769.14966191593, Loss: 2.5050907135009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7878 , TFLOPS: 69.55223906613806, Tokens per sec: 172358.11021752292, Loss: 2.4920384883880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7879 , TFLOPS: 71.55265452058286, Tokens per sec: 177315.36007759094, Loss: 2.5088436603546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7880 , TFLOPS: 70.59315035094394, Tokens per sec: 174937.60304711584, Loss: 2.4984662532806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7881 , TFLOPS: 67.4095876947075, Tokens per sec: 167048.38408658438, Loss: 2.513981342315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7882 , TFLOPS: 71.53750645723657, Tokens per sec: 177277.82150792194, Loss: 2.4876508712768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7883 , TFLOPS: 71.45172618802698, Tokens per sec: 177065.24855134383, Loss: 2.523069143295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7884 , TFLOPS: 68.8451205746618, Tokens per sec: 170605.79270011245, Loss: 2.5151727199554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7885 , TFLOPS: 69.65672483427657, Tokens per sec: 172617.0374610277, Loss: 2.507932424545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7886 , TFLOPS: 71.53275152098027, Tokens per sec: 177266.03825207936, Loss: 2.5030500888824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7887 , TFLOPS: 69.02474849473937, Tokens per sec: 171050.93047371355, Loss: 2.5045509338378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7888 , TFLOPS: 68.63676246834675, Tokens per sec: 170089.45836012354, Loss: 2.49326753616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7889 , TFLOPS: 71.523385136324, Tokens per sec: 177242.82731910865, Loss: 2.522765636444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7890 , TFLOPS: 71.5249498339732, Tokens per sec: 177246.70481224905, Loss: 2.503638744354248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7891 , TFLOPS: 69.99548541505717, Tokens per sec: 173456.52349776065, Loss: 2.504335403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7892 , TFLOPS: 69.78914812824242, Tokens per sec: 172945.1969711051, Loss: 2.501660108566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7893 , TFLOPS: 71.47892541396247, Tokens per sec: 177132.65122945406, Loss: 2.51102352142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7894 , TFLOPS: 69.64322971798798, Tokens per sec: 172583.59507625477, Loss: 2.5008671283721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7895 , TFLOPS: 68.68482848655543, Tokens per sec: 170208.5712481537, Loss: 2.501845598220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7896 , TFLOPS: 71.5255312962239, Tokens per sec: 177248.14573975967, Loss: 2.47843599319458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7897 , TFLOPS: 69.90134760296077, Tokens per sec: 173223.23962924967, Loss: 2.4950966835021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7898 , TFLOPS: 68.93909323472167, Tokens per sec: 170838.66730368388, Loss: 2.493577241897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7899 , TFLOPS: 70.44449287526002, Tokens per sec: 174569.21344639827, Loss: 2.5081615447998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7900 , TFLOPS: 71.45815549511231, Tokens per sec: 177081.18108254793, Loss: 2.5113677978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7901 , TFLOPS: 67.21763566057123, Tokens per sec: 166572.7058007326, Loss: 2.5168070793151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7902 , TFLOPS: 70.51455555101751, Tokens per sec: 174742.83647496617, Loss: 2.505518674850464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7903 , TFLOPS: 71.47842083956677, Tokens per sec: 177131.40083851805, Loss: 2.5165467262268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7904 , TFLOPS: 68.98803252846297, Tokens per sec: 170959.9442646831, Loss: 2.506936550140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7905 , TFLOPS: 69.67618157074456, Tokens per sec: 172665.25339733184, Loss: 2.5073461532592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7906 , TFLOPS: 70.5124582959544, Tokens per sec: 174737.6392458881, Loss: 2.511091709136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7907 , TFLOPS: 69.79258781211996, Tokens per sec: 172953.72088666612, Loss: 2.512709379196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7908 , TFLOPS: 68.97576562391295, Tokens per sec: 170929.54552389702, Loss: 2.489867687225342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7909 , TFLOPS: 71.50790154410218, Tokens per sec: 177204.4572719245, Loss: 2.4893674850463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7910 , TFLOPS: 70.4434985026137, Tokens per sec: 174566.7492814415, Loss: 2.5024755001068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7911 , TFLOPS: 69.72594512324518, Tokens per sec: 172788.5729623362, Loss: 2.4773998260498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7912 , TFLOPS: 69.50015546301358, Tokens per sec: 172229.04131149795, Loss: 2.527352809906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7913 , TFLOPS: 71.51860599211051, Tokens per sec: 177230.98407328143, Loss: 2.501621723175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7914 , TFLOPS: 68.65400523661079, Tokens per sec: 170132.18783932956, Loss: 2.507474184036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7915 , TFLOPS: 69.64468729678313, Tokens per sec: 172587.2071170747, Loss: 2.509230136871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7916 , TFLOPS: 71.52243378403556, Tokens per sec: 177240.46976333778, Loss: 2.5047996044158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7917 , TFLOPS: 69.51356966352564, Tokens per sec: 172262.283178063, Loss: 2.509427547454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7918 , TFLOPS: 68.7245639020436, Tokens per sec: 170307.03998495074, Loss: 2.5140368938446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7919 , TFLOPS: 71.51084819931864, Tokens per sec: 177211.7594081518, Loss: 2.5385777950286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7920 , TFLOPS: 71.48386478207672, Tokens per sec: 177144.8915277575, Loss: 2.4948384761810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7921 , TFLOPS: 67.16090150677113, Tokens per sec: 166432.11231783294, Loss: 2.501486301422119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7922 , TFLOPS: 70.72397617137689, Tokens per sec: 175261.80384180252, Loss: 2.521139144897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7923 , TFLOPS: 71.52537413864043, Tokens per sec: 177247.75628595546, Loss: 2.5102224349975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7924 , TFLOPS: 69.00331949561141, Tokens per sec: 170997.82705327537, Loss: 2.4885413646698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7925 , TFLOPS: 68.5597175656212, Tokens per sec: 169898.53260397297, Loss: 2.5002453327178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7926 , TFLOPS: 71.55360004296797, Tokens per sec: 177317.70318621918, Loss: 2.4849462509155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7927 , TFLOPS: 70.72625227749356, Tokens per sec: 175267.4442834934, Loss: 2.521726369857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7928 , TFLOPS: 67.8359724360982, Tokens per sec: 168105.0124459073, Loss: 2.5297679901123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7929 , TFLOPS: 70.52854112273995, Tokens per sec: 174777.4942623323, Loss: 2.511563777923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7930 , TFLOPS: 71.49916613995511, Tokens per sec: 177182.80997816386, Loss: 2.512690782546997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7931 , TFLOPS: 69.74878637628349, Tokens per sec: 172845.1761035369, Loss: 2.48398494720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7932 , TFLOPS: 68.53864115053287, Tokens per sec: 169846.3029256253, Loss: 2.5093154907226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7933 , TFLOPS: 71.57495536028671, Tokens per sec: 177370.6240150174, Loss: 2.500443696975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7934 , TFLOPS: 69.55191494683257, Tokens per sec: 172357.3070141792, Loss: 2.50253963470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7935 , TFLOPS: 69.69833231572574, Tokens per sec: 172720.14538350116, Loss: 2.5184171199798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7936 , TFLOPS: 70.4727666220496, Tokens per sec: 174639.27890555316, Loss: 2.5023193359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7937 , TFLOPS: 69.88912301588778, Tokens per sec: 173192.94575583335, Loss: 2.4825439453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7938 , TFLOPS: 69.58324785486363, Tokens per sec: 172434.95341188493, Loss: 2.491300582885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7939 , TFLOPS: 69.68679662834506, Tokens per sec: 172691.55867941098, Loss: 2.4914355278015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7940 , TFLOPS: 71.49172946764627, Tokens per sec: 177164.38108496548, Loss: 2.5379788875579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7941 , TFLOPS: 68.98846506873775, Tokens per sec: 170961.01614713224, Loss: 2.487673282623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7942 , TFLOPS: 69.97074737712963, Tokens per sec: 173395.21991465634, Loss: 2.521523952484131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7943 , TFLOPS: 70.51904246870477, Tokens per sec: 174753.95555127, Loss: 2.5477538108825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7944 , TFLOPS: 69.04100413006034, Tokens per sec: 171091.213728745, Loss: 2.5027284622192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7945 , TFLOPS: 69.51149215662957, Tokens per sec: 172257.13488711824, Loss: 2.5173981189727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7946 , TFLOPS: 71.46979705508203, Tokens per sec: 177110.03015057658, Loss: 2.509882926940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7947 , TFLOPS: 70.51269471881277, Tokens per sec: 174738.22512777525, Loss: 2.483394145965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7948 , TFLOPS: 66.81787970284942, Tokens per sec: 165582.06650074385, Loss: 2.480060577392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7949 , TFLOPS: 71.44954704225242, Tokens per sec: 177059.84838806174, Loss: 2.515528678894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7950 , TFLOPS: 71.44443616718169, Tokens per sec: 177047.18307662715, Loss: 2.507455825805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7951 , TFLOPS: 69.44751775691894, Tokens per sec: 172098.5992772734, Loss: 2.4987356662750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7952 , TFLOPS: 69.51999150829512, Tokens per sec: 172278.19721682637, Loss: 2.493008613586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7953 , TFLOPS: 71.54817308902673, Tokens per sec: 177304.25459652444, Loss: 2.5205187797546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7954 , TFLOPS: 68.87799788829648, Tokens per sec: 170687.26630503425, Loss: 2.532155752182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7955 , TFLOPS: 69.66479428922678, Tokens per sec: 172637.03445931815, Loss: 2.512272357940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7956 , TFLOPS: 71.49608568292743, Tokens per sec: 177175.1762663089, Loss: 2.5042858123779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7957 , TFLOPS: 70.67213643845702, Tokens per sec: 175133.33927300904, Loss: 2.5017571449279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7958 , TFLOPS: 68.87665923935, Tokens per sec: 170683.94898548053, Loss: 2.494734764099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7959 , TFLOPS: 69.82638918173036, Tokens per sec: 173037.48440408896, Loss: 2.481043815612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7960 , TFLOPS: 71.54551234233688, Tokens per sec: 177297.66097312156, Loss: 2.5201845169067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7961 , TFLOPS: 69.08539597863678, Tokens per sec: 171201.22161968326, Loss: 2.4976422786712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7962 , TFLOPS: 69.39320167027469, Tokens per sec: 171963.99803115806, Loss: 2.5214462280273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7963 , TFLOPS: 71.55498321498278, Tokens per sec: 177321.13083884085, Loss: 2.4916532039642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7964 , TFLOPS: 69.75617810997763, Tokens per sec: 172863.49363389763, Loss: 2.4975883960723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7965 , TFLOPS: 69.04490640104807, Tokens per sec: 171100.88398612323, Loss: 2.5011096000671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7966 , TFLOPS: 70.43172933041448, Tokens per sec: 174537.58397625046, Loss: 2.488631248474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7967 , TFLOPS: 71.51005019411511, Tokens per sec: 177209.78186334987, Loss: 2.5026612281799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7968 , TFLOPS: 68.2520103870274, Tokens per sec: 169136.00031867356, Loss: 2.514910936355591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7969 , TFLOPS: 69.63566871767988, Tokens per sec: 172564.85808457836, Loss: 2.510507345199585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7970 , TFLOPS: 71.50159820267021, Tokens per sec: 177188.83689748571, Loss: 2.511627674102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7971 , TFLOPS: 69.75622851220875, Tokens per sec: 172863.61853617863, Loss: 2.48976993560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7972 , TFLOPS: 69.60485342111245, Tokens per sec: 172488.4943850991, Loss: 2.505309820175171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7973 , TFLOPS: 70.47072794461374, Tokens per sec: 174634.22683828822, Loss: 2.5196268558502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7974 , TFLOPS: 70.57710539302713, Tokens per sec: 174897.84187389992, Loss: 2.5053586959838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7975 , TFLOPS: 69.18112131477649, Tokens per sec: 171438.43954765375, Loss: 2.5171782970428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7976 , TFLOPS: 70.5684335959924, Tokens per sec: 174876.35220557058, Loss: 2.5050246715545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7977 , TFLOPS: 71.54260637243769, Tokens per sec: 177290.45965959234, Loss: 2.4767560958862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7978 , TFLOPS: 69.2275618816587, Tokens per sec: 171553.5243882386, Loss: 2.5277490615844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7979 , TFLOPS: 69.60143634330838, Tokens per sec: 172480.02648987845, Loss: 2.499340772628784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7980 , TFLOPS: 70.42374640141625, Tokens per sec: 174517.8013988006, Loss: 2.5077402591705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7981 , TFLOPS: 69.02778913610565, Tokens per sec: 171058.46551797356, Loss: 2.501847267150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7982 , TFLOPS: 69.72032580671795, Tokens per sec: 172774.64767696292, Loss: 2.504868745803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7983 , TFLOPS: 71.41347861498467, Tokens per sec: 176970.46685202702, Loss: 2.5136990547180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7984 , TFLOPS: 68.78657579045107, Tokens per sec: 170460.71227559788, Loss: 2.516566276550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7985 , TFLOPS: 68.70356233998766, Tokens per sec: 170254.99580066308, Loss: 2.4918272495269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7986 , TFLOPS: 71.48833826727247, Tokens per sec: 177155.97731686765, Loss: 2.5007028579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7987 , TFLOPS: 71.46959808738443, Tokens per sec: 177109.53708670381, Loss: 2.5027031898498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7988 , TFLOPS: 68.79083746300805, Tokens per sec: 170471.27316965608, Loss: 2.4973196983337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7989 , TFLOPS: 69.93270616527941, Tokens per sec: 173300.94960110608, Loss: 2.4949569702148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7990 , TFLOPS: 71.52833572458812, Tokens per sec: 177255.09542217944, Loss: 2.4913747310638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7991 , TFLOPS: 68.67616318134462, Tokens per sec: 170187.0976672791, Loss: 2.5314455032348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7992 , TFLOPS: 69.55698161519868, Tokens per sec: 172369.86277653003, Loss: 2.5068397521972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7993 , TFLOPS: 71.39319682488802, Tokens per sec: 176920.20634194388, Loss: 2.5223355293273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7994 , TFLOPS: 71.51536960561589, Tokens per sec: 177222.96395103037, Loss: 2.504068374633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7995 , TFLOPS: 68.16429981073344, Tokens per sec: 168918.64384849952, Loss: 2.5035507678985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7996 , TFLOPS: 71.52509456327458, Tokens per sec: 177247.06346740114, Loss: 2.4770889282226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7997 , TFLOPS: 71.52123599407648, Tokens per sec: 177237.50150787173, Loss: 2.5055651664733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7998 , TFLOPS: 68.85783989918659, Tokens per sec: 170637.31258743416, Loss: 2.502368927001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7999 , TFLOPS: 68.55928695798772, Tokens per sec: 169897.46551082187, Loss: 2.5083236694335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8000 , TFLOPS: 71.48162590415384, Tokens per sec: 177139.34334163184, Loss: 2.504572868347168 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/model.pt +[rank0]:[2024-08-26 18:53:35,796] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006433666996599641, 'preprocessing_with_comm': 0.0013895260053686798, 'state_converting': 0.9705399529993883, : 0.9797517900005914}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000, took 4.31s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006000 + eval ppl=9.971844673156738, eval loss=2.2997655868530273 +------------------------------------------------------------------ +iteration: 8001 , TFLOPS: 65.54909478744858, Tokens per sec: 162437.87771218398, Loss: 2.516204595565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8002 , TFLOPS: 68.19767811136582, Tokens per sec: 169001.35895438993, Loss: 2.497868061065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8003 , TFLOPS: 71.49213254513313, Tokens per sec: 177165.37995538057, Loss: 2.5041866302490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8004 , TFLOPS: 70.53442909013962, Tokens per sec: 174792.08529416187, Loss: 2.5108625888824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8005 , TFLOPS: 68.79032016213019, Tokens per sec: 170469.99124109533, Loss: 2.5243287086486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8006 , TFLOPS: 69.80658020531348, Tokens per sec: 172988.39557838903, Loss: 2.5209174156188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8007 , TFLOPS: 71.55933318563966, Tokens per sec: 177331.9105453174, Loss: 2.5180394649505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8008 , TFLOPS: 69.80340264780618, Tokens per sec: 172980.5212408491, Loss: 2.522731065750122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8009 , TFLOPS: 69.94871153345399, Tokens per sec: 173340.61266657952, Loss: 2.515369415283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8010 , TFLOPS: 70.4023392650938, Tokens per sec: 174464.75215680286, Loss: 2.503520965576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8011 , TFLOPS: 69.022049957522, Tokens per sec: 171044.24320122055, Loss: 2.51640248298645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8012 , TFLOPS: 69.73285283242824, Tokens per sec: 172805.6910266388, Loss: 2.516355276107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8013 , TFLOPS: 71.51094618002071, Tokens per sec: 177212.00221512496, Loss: 2.494028091430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8014 , TFLOPS: 69.56135348913348, Tokens per sec: 172380.69676174215, Loss: 2.4970874786376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8015 , TFLOPS: 68.1840335689021, Tokens per sec: 168967.54627509438, Loss: 2.4994895458221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8016 , TFLOPS: 71.47645468314722, Tokens per sec: 177126.52848632043, Loss: 2.5013842582702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8017 , TFLOPS: 70.41720138362844, Tokens per sec: 174501.5821237292, Loss: 2.493142604827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8018 , TFLOPS: 69.03856059006955, Tokens per sec: 171085.15836746967, Loss: 2.5208048820495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8019 , TFLOPS: 69.67574281223095, Tokens per sec: 172664.16610540185, Loss: 2.507751703262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8020 , TFLOPS: 71.50065831244953, Tokens per sec: 177186.5077459814, Loss: 2.510204792022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8021 , TFLOPS: 68.85632488174107, Tokens per sec: 170633.55820731123, Loss: 2.506824016571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8022 , TFLOPS: 69.61772436829138, Tokens per sec: 172520.3900100604, Loss: 2.51039457321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8023 , TFLOPS: 71.56106112289295, Tokens per sec: 177336.19256417907, Loss: 2.5017449855804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8024 , TFLOPS: 70.57281511653001, Tokens per sec: 174887.2100961271, Loss: 2.5075786113739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8025 , TFLOPS: 68.82852587903069, Tokens per sec: 170564.66921628194, Loss: 2.479814291000366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8026 , TFLOPS: 69.82140210198291, Tokens per sec: 173025.1258711026, Loss: 2.4976253509521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8027 , TFLOPS: 71.56999662377773, Tokens per sec: 177358.3357196982, Loss: 2.5006051063537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8028 , TFLOPS: 69.77255204613353, Tokens per sec: 172904.07005142994, Loss: 2.4914205074310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8029 , TFLOPS: 68.76552506284308, Tokens per sec: 170408.5462536562, Loss: 2.4888598918914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8030 , TFLOPS: 71.4686062178544, Tokens per sec: 177107.07912474498, Loss: 2.5005908012390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8031 , TFLOPS: 69.74735509880135, Tokens per sec: 172841.6292402679, Loss: 2.4990363121032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8032 , TFLOPS: 68.21494882542436, Tokens per sec: 169044.1576863532, Loss: 2.488922119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8033 , TFLOPS: 71.47405780298858, Tokens per sec: 177120.58875324915, Loss: 2.5286641120910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8034 , TFLOPS: 71.49572337205804, Tokens per sec: 177174.27842006274, Loss: 2.516705274581909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8035 , TFLOPS: 68.3978523770052, Tokens per sec: 169497.41283565998, Loss: 2.5203657150268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8036 , TFLOPS: 69.77324009388806, Tokens per sec: 172905.7751095604, Loss: 2.5116758346557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8037 , TFLOPS: 71.54685202115644, Tokens per sec: 177300.98084481424, Loss: 2.5022695064544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8038 , TFLOPS: 69.01995956214495, Tokens per sec: 171039.06297120012, Loss: 2.517350435256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8039 , TFLOPS: 68.77704518503428, Tokens per sec: 170437.09438549224, Loss: 2.4817047119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8040 , TFLOPS: 71.50943962145925, Tokens per sec: 177208.2687970485, Loss: 2.5149662494659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8041 , TFLOPS: 69.80711720199689, Tokens per sec: 172989.7263153256, Loss: 2.49916410446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8042 , TFLOPS: 69.5834219171024, Tokens per sec: 172435.38475728716, Loss: 2.491802215576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8043 , TFLOPS: 70.44928340058492, Tokens per sec: 174581.08489587327, Loss: 2.501737356185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8044 , TFLOPS: 71.5145581204372, Tokens per sec: 177220.95300136512, Loss: 2.514860153198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8045 , TFLOPS: 69.1206405754279, Tokens per sec: 171288.56161304447, Loss: 2.4907596111297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8046 , TFLOPS: 69.67036093454124, Tokens per sec: 172650.829219624, Loss: 2.5032870769500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8047 , TFLOPS: 70.58093596535001, Tokens per sec: 174907.33445408844, Loss: 2.497692584991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8048 , TFLOPS: 70.50068680067226, Tokens per sec: 174708.46818384164, Loss: 2.5194194316864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8049 , TFLOPS: 69.19596254210254, Tokens per sec: 171475.21774386382, Loss: 2.474994659423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8050 , TFLOPS: 71.52561013614967, Tokens per sec: 177248.34111378063, Loss: 2.5176198482513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8051 , TFLOPS: 69.61992381054702, Tokens per sec: 172525.84047025844, Loss: 2.518571376800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8052 , TFLOPS: 68.3662801599585, Tokens per sec: 169419.17340385148, Loss: 2.514897584915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8053 , TFLOPS: 71.54540671593185, Tokens per sec: 177297.39921925418, Loss: 2.490161180496216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8054 , TFLOPS: 70.54176674669012, Tokens per sec: 174810.26881540354, Loss: 2.505990743637085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8055 , TFLOPS: 68.0187907298146, Tokens per sec: 168558.05631683412, Loss: 2.4876480102539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8056 , TFLOPS: 70.70183280661053, Tokens per sec: 175206.93014461847, Loss: 2.4929440021514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8057 , TFLOPS: 71.52401750428034, Tokens per sec: 177244.39439656553, Loss: 2.5102245807647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8058 , TFLOPS: 67.99961344502765, Tokens per sec: 168510.53289257977, Loss: 2.5047287940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8059 , TFLOPS: 69.53556462596144, Tokens per sec: 172316.78911792403, Loss: 2.502450704574585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8060 , TFLOPS: 71.52903300529431, Tokens per sec: 177256.8233605812, Loss: 2.5055618286132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8061 , TFLOPS: 69.82812120585909, Tokens per sec: 173041.77655067816, Loss: 2.5022635459899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8062 , TFLOPS: 68.67859379466191, Tokens per sec: 170193.12099483353, Loss: 2.4940357208251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8063 , TFLOPS: 71.4694615516196, Tokens per sec: 177109.19873603794, Loss: 2.4927167892456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8064 , TFLOPS: 71.50424392146141, Tokens per sec: 177195.39328010028, Loss: 2.524904489517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8065 , TFLOPS: 67.96508324265385, Tokens per sec: 168424.96324728164, Loss: 2.516798734664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8066 , TFLOPS: 69.57711646032034, Tokens per sec: 172419.7591407778, Loss: 2.492985725402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8067 , TFLOPS: 71.46735768165556, Tokens per sec: 177103.9851145066, Loss: 2.5052835941314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8068 , TFLOPS: 70.64653686807644, Tokens per sec: 175069.90071758052, Loss: 2.5080602169036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8069 , TFLOPS: 68.79790675193222, Tokens per sec: 170488.7916463566, Loss: 2.505064010620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8070 , TFLOPS: 71.51702284003993, Tokens per sec: 177227.06084805194, Loss: 2.500516891479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8071 , TFLOPS: 70.62557304515654, Tokens per sec: 175017.95005502927, Loss: 2.506613254547119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8072 , TFLOPS: 68.7221208281966, Tokens per sec: 170300.98577883103, Loss: 2.487001419067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8073 , TFLOPS: 69.74878541146269, Tokens per sec: 172845.17371260462, Loss: 2.4872775077819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8074 , TFLOPS: 71.54513659427766, Tokens per sec: 177296.72982803875, Loss: 2.501063346862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8075 , TFLOPS: 69.12414716262079, Tokens per sec: 171297.25132239092, Loss: 2.482357978820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8076 , TFLOPS: 68.95483224458367, Tokens per sec: 170877.670303917, Loss: 2.5035784244537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8077 , TFLOPS: 71.53174092670149, Tokens per sec: 177263.53388813036, Loss: 2.4967474937438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8078 , TFLOPS: 68.94119524377523, Tokens per sec: 170843.87631367403, Loss: 2.4883878231048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8079 , TFLOPS: 69.54737224931266, Tokens per sec: 172346.04970930543, Loss: 2.4977364540100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8080 , TFLOPS: 70.47130055328405, Tokens per sec: 174635.64582565133, Loss: 2.4996283054351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8081 , TFLOPS: 70.7269694771219, Tokens per sec: 175269.22158317847, Loss: 2.505969524383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8082 , TFLOPS: 69.10729749602835, Tokens per sec: 171255.49599243086, Loss: 2.515841245651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8083 , TFLOPS: 71.49339906409178, Tokens per sec: 177168.51852887866, Loss: 2.50563383102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8084 , TFLOPS: 70.58595720278494, Tokens per sec: 174919.77763358713, Loss: 2.501368284225464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8085 , TFLOPS: 68.91586335876968, Tokens per sec: 170781.1011120928, Loss: 2.49662184715271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8086 , TFLOPS: 69.63580502291285, Tokens per sec: 172565.1958639609, Loss: 2.4929893016815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8087 , TFLOPS: 71.54042011382761, Tokens per sec: 177285.04186992027, Loss: 2.5037834644317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8088 , TFLOPS: 69.47699649432737, Tokens per sec: 172171.65083592248, Loss: 2.4916937351226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8089 , TFLOPS: 68.9029454952361, Tokens per sec: 170749.0892232364, Loss: 2.5177407264709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8090 , TFLOPS: 71.55428845515334, Tokens per sec: 177319.4091474494, Loss: 2.505608320236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8091 , TFLOPS: 69.08878652794304, Tokens per sec: 171209.62377436398, Loss: 2.513223648071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8092 , TFLOPS: 69.67798775642127, Tokens per sec: 172669.72932440648, Loss: 2.5052430629730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8093 , TFLOPS: 70.01304123467652, Tokens per sec: 173500.02875271055, Loss: 2.5171728134155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8094 , TFLOPS: 71.57329578899287, Tokens per sec: 177366.51141453438, Loss: 2.4998795986175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8095 , TFLOPS: 68.14145712787519, Tokens per sec: 168862.03716404754, Loss: 2.512430429458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8096 , TFLOPS: 69.49514490605438, Tokens per sec: 172216.62459939462, Loss: 2.4896883964538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8097 , TFLOPS: 71.51871871757626, Tokens per sec: 177231.26341940332, Loss: 2.5238752365112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8098 , TFLOPS: 69.72549207387456, Tokens per sec: 172787.45025608828, Loss: 2.486053943634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8099 , TFLOPS: 68.76292759846238, Tokens per sec: 170402.10945078652, Loss: 2.4998538494110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8100 , TFLOPS: 71.53644231487853, Tokens per sec: 177275.18444593373, Loss: 2.517590045928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8101 , TFLOPS: 71.5389355901943, Tokens per sec: 177281.36305682317, Loss: 2.4945549964904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8102 , TFLOPS: 68.42054390764925, Tokens per sec: 169553.64494827442, Loss: 2.498602867126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8103 , TFLOPS: 70.48796177487492, Tokens per sec: 174676.93416813857, Loss: 2.50052809715271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8104 , TFLOPS: 71.46630689279985, Tokens per sec: 177101.38114396823, Loss: 2.475796699523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8105 , TFLOPS: 68.87141786026743, Tokens per sec: 170670.96027072833, Loss: 2.511728048324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8106 , TFLOPS: 69.60491617830886, Tokens per sec: 172488.6499043451, Loss: 2.5359771251678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8107 , TFLOPS: 71.54379757902014, Tokens per sec: 177293.41160072587, Loss: 2.5139596462249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8108 , TFLOPS: 70.54627945343682, Tokens per sec: 174821.45179983758, Loss: 2.5037364959716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8109 , TFLOPS: 68.80294366290758, Tokens per sec: 170501.2736666151, Loss: 2.5027260780334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8110 , TFLOPS: 70.52465153600406, Tokens per sec: 174767.85543792738, Loss: 2.5173144340515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8111 , TFLOPS: 70.67445584305091, Tokens per sec: 175139.08701309623, Loss: 2.507058620452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8112 , TFLOPS: 69.83969708159367, Tokens per sec: 173070.46284593642, Loss: 2.4995908737182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8113 , TFLOPS: 68.60949378989214, Tokens per sec: 170021.88357102012, Loss: 2.4771194458007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8114 , TFLOPS: 71.53595963303215, Tokens per sec: 177273.98830714662, Loss: 2.4924488067626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8115 , TFLOPS: 69.82367200487566, Tokens per sec: 173030.75094052125, Loss: 2.492053747177124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8116 , TFLOPS: 68.9553714934283, Tokens per sec: 170879.00662195717, Loss: 2.508047580718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8117 , TFLOPS: 70.55776383734992, Tokens per sec: 174849.9113682836, Loss: 2.487657070159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8118 , TFLOPS: 70.70260095966904, Tokens per sec: 175208.8337125165, Loss: 2.5000860691070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8119 , TFLOPS: 68.36039382261572, Tokens per sec: 169404.58641148248, Loss: 2.499640464782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8120 , TFLOPS: 71.54981387763051, Tokens per sec: 177308.3206514324, Loss: 2.5369627475738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8121 , TFLOPS: 70.45500331779895, Tokens per sec: 174595.25948083054, Loss: 2.5118680000305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8122 , TFLOPS: 68.95371060275606, Tokens per sec: 170874.89075190914, Loss: 2.491825580596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8123 , TFLOPS: 70.64788792781748, Tokens per sec: 175073.24879245032, Loss: 2.5079329013824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8124 , TFLOPS: 71.49657480744949, Tokens per sec: 177176.3883707561, Loss: 2.482816457748413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8125 , TFLOPS: 67.85357828827235, Tokens per sec: 168148.64168703958, Loss: 2.5021722316741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8126 , TFLOPS: 69.64341657618074, Tokens per sec: 172584.05813144057, Loss: 2.503060817718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8127 , TFLOPS: 71.5552793292336, Tokens per sec: 177321.86464257404, Loss: 2.480029344558716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8128 , TFLOPS: 69.54688056214074, Tokens per sec: 172344.83125432962, Loss: 2.498293399810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8129 , TFLOPS: 68.79553768144712, Tokens per sec: 170482.92082871986, Loss: 2.5116539001464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8130 , TFLOPS: 71.47142172733871, Tokens per sec: 177114.05626740106, Loss: 2.4967103004455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8131 , TFLOPS: 71.51942756856067, Tokens per sec: 177233.0200302291, Loss: 2.5060245990753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8132 , TFLOPS: 68.34652419801502, Tokens per sec: 169370.21595385665, Loss: 2.4977047443389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8133 , TFLOPS: 69.56309942576364, Tokens per sec: 172385.0233850425, Loss: 2.5251376628875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8134 , TFLOPS: 71.54810705910813, Tokens per sec: 177304.09096711237, Loss: 2.4935553073883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8135 , TFLOPS: 69.76000576308412, Tokens per sec: 172872.97897994632, Loss: 2.499835729598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8136 , TFLOPS: 67.7860682622469, Tokens per sec: 167981.34440570523, Loss: 2.505401134490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8137 , TFLOPS: 71.54224986403605, Tokens per sec: 177289.57619250042, Loss: 2.4962687492370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8138 , TFLOPS: 70.7256583605701, Tokens per sec: 175265.9724919317, Loss: 2.485138416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8139 , TFLOPS: 68.75998710750785, Tokens per sec: 170394.8225902794, Loss: 2.4984629154205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8140 , TFLOPS: 70.85222233640191, Tokens per sec: 175579.61196621583, Loss: 2.513916492462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8141 , TFLOPS: 71.5260321971756, Tokens per sec: 177249.38702749676, Loss: 2.5095980167388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8142 , TFLOPS: 69.04286113440317, Tokens per sec: 171095.81559007245, Loss: 2.5221807956695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8143 , TFLOPS: 69.4316814444676, Tokens per sec: 172059.35515050511, Loss: 2.5181515216827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8144 , TFLOPS: 71.55564330783409, Tokens per sec: 177322.7666216416, Loss: 2.4819178581237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8145 , TFLOPS: 69.80761854929699, Tokens per sec: 172990.9687091632, Loss: 2.4991607666015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8146 , TFLOPS: 68.48872450230863, Tokens per sec: 169722.60397255293, Loss: 2.4889681339263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8147 , TFLOPS: 70.56672395438376, Tokens per sec: 174872.1155253318, Loss: 2.4984211921691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8148 , TFLOPS: 71.54708425433543, Tokens per sec: 177301.55634421436, Loss: 2.486065626144409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8149 , TFLOPS: 68.23809430688588, Tokens per sec: 169101.51473910012, Loss: 2.494508981704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8150 , TFLOPS: 70.40890358086342, Tokens per sec: 174481.01925439906, Loss: 2.5010433197021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8151 , TFLOPS: 71.51529309705009, Tokens per sec: 177222.7743543762, Loss: 2.5223586559295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8152 , TFLOPS: 68.95413389376598, Tokens per sec: 170875.93971366106, Loss: 2.5012993812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8153 , TFLOPS: 69.70178933456026, Tokens per sec: 172728.71225699518, Loss: 2.503824234008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8154 , TFLOPS: 70.41926932592027, Tokens per sec: 174506.70671253098, Loss: 2.501546859741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8155 , TFLOPS: 69.85453731376711, Tokens per sec: 173107.23857604875, Loss: 2.505795478820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8156 , TFLOPS: 68.76506370517491, Tokens per sec: 170407.40295853175, Loss: 2.4926085472106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8157 , TFLOPS: 71.54556911479165, Tokens per sec: 177297.80166151843, Loss: 2.498403549194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8158 , TFLOPS: 69.73134792665346, Tokens per sec: 172801.96170434987, Loss: 2.509267807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8159 , TFLOPS: 69.81785208510023, Tokens per sec: 173016.32853820064, Loss: 2.497476577758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8160 , TFLOPS: 69.88945460337547, Tokens per sec: 173193.76746615526, Loss: 2.4964182376861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8161 , TFLOPS: 71.4731262205999, Tokens per sec: 177118.2801894677, Loss: 2.523238182067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8162 , TFLOPS: 67.97075751915897, Tokens per sec: 168439.02472953452, Loss: 2.511232376098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8163 , TFLOPS: 70.39075601349896, Tokens per sec: 174436.04758335033, Loss: 2.500868082046509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8164 , TFLOPS: 71.46280553491007, Tokens per sec: 177092.70439341097, Loss: 2.514704704284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8165 , TFLOPS: 69.58066625226103, Tokens per sec: 172428.55591624868, Loss: 2.5003652572631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8166 , TFLOPS: 68.92821356375885, Tokens per sec: 170811.7062805433, Loss: 2.492722749710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8167 , TFLOPS: 71.54840841515727, Tokens per sec: 177304.8377605992, Loss: 2.4913394451141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8168 , TFLOPS: 71.49494915555323, Tokens per sec: 177172.3598262768, Loss: 2.4993810653686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8169 , TFLOPS: 67.0199512111779, Tokens per sec: 166082.8219583961, Loss: 2.5041375160217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8170 , TFLOPS: 71.49964162422894, Tokens per sec: 177183.98828057334, Loss: 2.4949660301208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8171 , TFLOPS: 71.47343168290308, Tokens per sec: 177119.03715870445, Loss: 2.521915912628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8172 , TFLOPS: 68.98833080047652, Tokens per sec: 170960.68341559, Loss: 2.4932518005371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8173 , TFLOPS: 68.63763945573434, Tokens per sec: 170091.63163147814, Loss: 2.475950241088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8174 , TFLOPS: 71.53463521774506, Tokens per sec: 177270.70626016887, Loss: 2.5057878494262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8175 , TFLOPS: 69.79273438348554, Tokens per sec: 172954.0841066562, Loss: 2.491212844848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8176 , TFLOPS: 68.71832054976903, Tokens per sec: 170291.56827025232, Loss: 2.51145339012146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8177 , TFLOPS: 70.4109816547542, Tokens per sec: 174486.16895041912, Loss: 2.495515823364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8178 , TFLOPS: 70.67844066294373, Tokens per sec: 175148.9618357548, Loss: 2.503269672393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8179 , TFLOPS: 69.80930929775627, Tokens per sec: 172995.15857009662, Loss: 2.5223236083984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8180 , TFLOPS: 68.62166995644957, Tokens per sec: 170052.0574530649, Loss: 2.5216126441955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8181 , TFLOPS: 71.49889762252373, Tokens per sec: 177182.14456238924, Loss: 2.5214123725891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8182 , TFLOPS: 69.8570720660202, Tokens per sec: 173113.5199713579, Loss: 2.509831666946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8183 , TFLOPS: 69.89654617767057, Tokens per sec: 173211.3411684599, Loss: 2.5064308643341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8184 , TFLOPS: 70.34487430676353, Tokens per sec: 174322.34765977273, Loss: 2.497453212738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8185 , TFLOPS: 70.577536393195, Tokens per sec: 174898.9099397945, Loss: 2.500035524368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8186 , TFLOPS: 68.16796340799259, Tokens per sec: 168927.72264608624, Loss: 2.5109729766845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8187 , TFLOPS: 70.03012717013388, Tokens per sec: 173542.36958294504, Loss: 2.4963908195495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8188 , TFLOPS: 71.54015961927404, Tokens per sec: 177284.39633572072, Loss: 2.5255699157714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8189 , TFLOPS: 68.7795991917479, Tokens per sec: 170443.42349547564, Loss: 2.513190269470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8190 , TFLOPS: 70.9058340958004, Tokens per sec: 175712.46781182813, Loss: 2.506953239440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8191 , TFLOPS: 70.4763956558575, Tokens per sec: 174648.2720511002, Loss: 2.4947171211242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8192 , TFLOPS: 68.97425061566848, Tokens per sec: 170925.79116657533, Loss: 2.516237497329712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8193 , TFLOPS: 69.76859627596325, Tokens per sec: 172894.2672171839, Loss: 2.4951541423797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8194 , TFLOPS: 71.51732027284024, Tokens per sec: 177227.79791929605, Loss: 2.4979677200317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8195 , TFLOPS: 68.85975723405348, Tokens per sec: 170642.06395444617, Loss: 2.5037753582000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8196 , TFLOPS: 68.80347143632807, Tokens per sec: 170502.58154729012, Loss: 2.4879114627838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8197 , TFLOPS: 71.51441907514759, Tokens per sec: 177220.60843182032, Loss: 2.506735324859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8198 , TFLOPS: 71.51135707583995, Tokens per sec: 177213.02046022896, Loss: 2.5138723850250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8199 , TFLOPS: 67.16644219924527, Tokens per sec: 166445.84276414156, Loss: 2.50771427154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8200 , TFLOPS: 69.6415101332849, Tokens per sec: 172579.33375593266, Loss: 2.5125679969787598 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/optimizer.pt +[rank0]:[2024-08-26 19:18:54,775] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006374447999405675, 'preprocessing_with_comm': 0.0012735529962810688, 'state_converting': 0.9922436420019949, : 1.001260719000129}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006200 + eval ppl=10.154234886169434, eval loss=2.3178908824920654 +------------------------------------------------------------------ +iteration: 8201 , TFLOPS: 67.06343766515724, Tokens per sec: 166190.58618178527, Loss: 2.4978644847869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8202 , TFLOPS: 67.77327819341656, Tokens per sec: 167949.64920620123, Loss: 2.5151960849761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8203 , TFLOPS: 69.51582780966201, Tokens per sec: 172267.87911294424, Loss: 2.5098037719726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8204 , TFLOPS: 71.454176086169, Tokens per sec: 177071.31966881832, Loss: 2.503551959991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8205 , TFLOPS: 70.61487587725684, Tokens per sec: 174991.44129458346, Loss: 2.5211102962493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8206 , TFLOPS: 68.75746789963448, Tokens per sec: 170388.57971565548, Loss: 2.4869322776794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8207 , TFLOPS: 70.78186941524778, Tokens per sec: 175405.26967758188, Loss: 2.4865572452545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8208 , TFLOPS: 71.50275790520875, Tokens per sec: 177191.710768128, Loss: 2.507979393005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8209 , TFLOPS: 68.92842864166377, Tokens per sec: 170812.23926728452, Loss: 2.501124858856201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8210 , TFLOPS: 69.51744691939521, Tokens per sec: 172271.89144522278, Loss: 2.506805181503296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8211 , TFLOPS: 71.50034293223166, Tokens per sec: 177185.72619905954, Loss: 2.481828451156616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8212 , TFLOPS: 69.76811214842822, Tokens per sec: 172893.06749582104, Loss: 2.5034806728363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8213 , TFLOPS: 67.8900924235685, Tokens per sec: 168239.12773666752, Loss: 2.4944159984588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8214 , TFLOPS: 71.43324321554313, Tokens per sec: 177019.44570946132, Loss: 2.499650716781616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8215 , TFLOPS: 70.51846484779905, Tokens per sec: 174752.52414303544, Loss: 2.499342203140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8216 , TFLOPS: 67.98964202075081, Tokens per sec: 168485.8226047787, Loss: 2.5199997425079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8217 , TFLOPS: 70.44842085148576, Tokens per sec: 174578.94740418793, Loss: 2.4920718669891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8218 , TFLOPS: 71.53942156980949, Tokens per sec: 177282.5673678447, Loss: 2.4905741214752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8219 , TFLOPS: 69.89780597046597, Tokens per sec: 173214.46307378451, Loss: 2.512650728225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8220 , TFLOPS: 68.40775471866385, Tokens per sec: 169521.95192912314, Loss: 2.5028655529022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8221 , TFLOPS: 71.51514934433345, Tokens per sec: 177222.41811930868, Loss: 2.497502565383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8222 , TFLOPS: 69.77374162602938, Tokens per sec: 172907.01796145487, Loss: 2.5173304080963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8223 , TFLOPS: 68.82947838396481, Tokens per sec: 170567.02962843573, Loss: 2.4853973388671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8224 , TFLOPS: 70.46968400009725, Tokens per sec: 174631.6398287764, Loss: 2.4943206310272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8225 , TFLOPS: 70.65564746531962, Tokens per sec: 175092.47778116423, Loss: 2.5027353763580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8226 , TFLOPS: 70.55210095142483, Tokens per sec: 174835.87811314283, Loss: 2.4966201782226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8227 , TFLOPS: 70.00800855515341, Tokens per sec: 173487.55721845684, Loss: 2.4933018684387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8228 , TFLOPS: 70.43172814985931, Tokens per sec: 174537.58105070473, Loss: 2.4856882095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8229 , TFLOPS: 69.8083285881066, Tokens per sec: 172992.72826355064, Loss: 2.4947264194488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8230 , TFLOPS: 68.9987666237562, Tokens per sec: 170986.5445352779, Loss: 2.498612403869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8231 , TFLOPS: 71.53313461004284, Tokens per sec: 177266.98758897712, Loss: 2.4979100227355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8232 , TFLOPS: 68.74168959009151, Tokens per sec: 170349.47932648374, Loss: 2.5142269134521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8233 , TFLOPS: 68.84521563891697, Tokens per sec: 170606.02827980896, Loss: 2.5087995529174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8234 , TFLOPS: 71.52161720126792, Tokens per sec: 177238.44618128554, Loss: 2.4869678020477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8235 , TFLOPS: 69.65764776620058, Tokens per sec: 172619.32458800473, Loss: 2.5016262531280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8236 , TFLOPS: 68.00716454761341, Tokens per sec: 168529.24535660993, Loss: 2.5240986347198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8237 , TFLOPS: 71.52981946033663, Tokens per sec: 177258.77228281126, Loss: 2.511204719543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8238 , TFLOPS: 71.52299802585871, Tokens per sec: 177241.86801673195, Loss: 2.4901347160339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8239 , TFLOPS: 68.7993528620696, Tokens per sec: 170492.3752665792, Loss: 2.5058343410491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8240 , TFLOPS: 69.68047689879278, Tokens per sec: 172675.89769340472, Loss: 2.528836727142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8241 , TFLOPS: 71.465560401377, Tokens per sec: 177099.53125598963, Loss: 2.4980521202087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8242 , TFLOPS: 69.76304837616314, Tokens per sec: 172880.51891032764, Loss: 2.4952194690704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8243 , TFLOPS: 67.65487089240071, Tokens per sec: 167656.222870643, Loss: 2.5089023113250732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8244 , TFLOPS: 71.4990524622666, Tokens per sec: 177182.5282723291, Loss: 2.495358943939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8245 , TFLOPS: 71.5420334588076, Tokens per sec: 177289.0399165053, Loss: 2.492178440093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8246 , TFLOPS: 70.09215086728238, Tokens per sec: 173696.07113695334, Loss: 2.5128426551818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8247 , TFLOPS: 68.78680694803893, Tokens per sec: 170461.2851095647, Loss: 2.5180249214172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8248 , TFLOPS: 71.51983118064352, Tokens per sec: 177234.0202254319, Loss: 2.5085458755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8249 , TFLOPS: 69.90188354834319, Tokens per sec: 173224.56776094658, Loss: 2.5056612491607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8250 , TFLOPS: 68.2250618581375, Tokens per sec: 169069.21889545885, Loss: 2.493950128555298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8251 , TFLOPS: 71.51256896612894, Tokens per sec: 177216.0236578658, Loss: 2.482128143310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8252 , TFLOPS: 69.65186307642237, Tokens per sec: 172604.9894894981, Loss: 2.5276906490325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8253 , TFLOPS: 68.73633909280575, Tokens per sec: 170336.22020480369, Loss: 2.4972050189971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8254 , TFLOPS: 70.55625659986673, Tokens per sec: 174846.1762677643, Loss: 2.495971202850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8255 , TFLOPS: 70.66859863842224, Tokens per sec: 175124.57221480203, Loss: 2.519378900527954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8256 , TFLOPS: 69.0917985241046, Tokens per sec: 171217.0878326511, Loss: 2.5070343017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8257 , TFLOPS: 69.36070906535272, Tokens per sec: 171883.4777768055, Loss: 2.480046033859253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8258 , TFLOPS: 71.50940981504986, Tokens per sec: 177208.19493348262, Loss: 2.488330364227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8259 , TFLOPS: 69.70710405039628, Tokens per sec: 172741.88270830113, Loss: 2.5112485885620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8260 , TFLOPS: 69.59576302828042, Tokens per sec: 172465.9673902719, Loss: 2.5026493072509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8261 , TFLOPS: 70.49949458127605, Tokens per sec: 174705.51373284406, Loss: 2.4856340885162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8262 , TFLOPS: 70.67270816446255, Tokens per sec: 175134.75607303157, Loss: 2.514738082885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8263 , TFLOPS: 68.41762923943777, Tokens per sec: 169546.42207936916, Loss: 2.495558738708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8264 , TFLOPS: 71.48735451662162, Tokens per sec: 177153.53947438573, Loss: 2.4960079193115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8265 , TFLOPS: 70.41140677018736, Tokens per sec: 174487.22243329283, Loss: 2.5092051029205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8266 , TFLOPS: 69.70383211770972, Tokens per sec: 172733.7744986707, Loss: 2.4855551719665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8267 , TFLOPS: 69.66584070326083, Tokens per sec: 172639.62758856654, Loss: 2.504678726196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8268 , TFLOPS: 71.51007636929837, Tokens per sec: 177209.84672833746, Loss: 2.503253221511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8269 , TFLOPS: 69.03241361675848, Tokens per sec: 171069.92549045983, Loss: 2.5071451663970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8270 , TFLOPS: 68.88486315279663, Tokens per sec: 170704.27918673694, Loss: 2.51200532913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8271 , TFLOPS: 71.55000370969128, Tokens per sec: 177308.7910761907, Loss: 2.5019354820251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8272 , TFLOPS: 68.59677908144603, Tokens per sec: 169990.37512284992, Loss: 2.4867167472839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8273 , TFLOPS: 69.788507234977, Tokens per sec: 172943.6087669932, Loss: 2.4921889305114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8274 , TFLOPS: 70.78013105905868, Tokens per sec: 175400.9618394495, Loss: 2.5009236335754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8275 , TFLOPS: 70.74081045626274, Tokens per sec: 175303.5210541992, Loss: 2.5021674633026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8276 , TFLOPS: 68.8831185717351, Tokens per sec: 170699.955922685, Loss: 2.515882730484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8277 , TFLOPS: 69.89799859907113, Tokens per sec: 173214.94042868793, Loss: 2.4871344566345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8278 , TFLOPS: 71.48233915461834, Tokens per sec: 177141.1108548539, Loss: 2.491628885269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8279 , TFLOPS: 70.57736782172793, Tokens per sec: 174898.49220112944, Loss: 2.509315252304077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8280 , TFLOPS: 68.63338695601843, Tokens per sec: 170081.0934687306, Loss: 2.517545223236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8281 , TFLOPS: 71.52386144516714, Tokens per sec: 177244.00766489175, Loss: 2.500392436981201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8282 , TFLOPS: 69.78727411876748, Tokens per sec: 172940.55296918706, Loss: 2.505967617034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8283 , TFLOPS: 66.87985701995217, Tokens per sec: 165735.65311988958, Loss: 2.511430501937866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8284 , TFLOPS: 71.48937010810502, Tokens per sec: 177158.53433211733, Loss: 2.513354539871216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8285 , TFLOPS: 71.49184016792962, Tokens per sec: 177164.65541246225, Loss: 2.5298776626586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8286 , TFLOPS: 69.63985639405718, Tokens per sec: 172575.23560795194, Loss: 2.505073070526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8287 , TFLOPS: 68.97161509403053, Tokens per sec: 170919.26005362978, Loss: 2.502396583557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8288 , TFLOPS: 71.53608318089138, Tokens per sec: 177274.29447235173, Loss: 2.499720811843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8289 , TFLOPS: 69.91506651809976, Tokens per sec: 173257.23661222734, Loss: 2.5061652660369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8290 , TFLOPS: 68.75042021168721, Tokens per sec: 170371.11476855527, Loss: 2.509293556213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8291 , TFLOPS: 70.52946638719624, Tokens per sec: 174779.78716958128, Loss: 2.4858570098876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8292 , TFLOPS: 70.59397744073037, Tokens per sec: 174939.6526667181, Loss: 2.5013155937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8293 , TFLOPS: 69.95201559123501, Tokens per sec: 173348.80048573288, Loss: 2.5132663249969482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8294 , TFLOPS: 68.39013222631858, Tokens per sec: 169478.2814518138, Loss: 2.4831156730651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8295 , TFLOPS: 71.51676787741722, Tokens per sec: 177226.42902258644, Loss: 2.505714178085327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8296 , TFLOPS: 70.09594092121137, Tokens per sec: 173705.46330809826, Loss: 2.5115370750427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8297 , TFLOPS: 69.07660394329046, Tokens per sec: 171179.4339875724, Loss: 2.5324196815490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8298 , TFLOPS: 69.95672079674388, Tokens per sec: 173360.46050330508, Loss: 2.499098062515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8299 , TFLOPS: 70.5607382279872, Tokens per sec: 174857.28223593894, Loss: 2.5155699253082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8300 , TFLOPS: 68.86705350978676, Tokens per sec: 170660.14492946435, Loss: 2.501594066619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8301 , TFLOPS: 71.46809858624295, Tokens per sec: 177105.82115769156, Loss: 2.532109022140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8302 , TFLOPS: 69.77038131840733, Tokens per sec: 172898.690748999, Loss: 2.5083680152893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8303 , TFLOPS: 67.98847218668473, Tokens per sec: 168482.92362709506, Loss: 2.5217676162719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8304 , TFLOPS: 71.45267045284898, Tokens per sec: 177067.58854359208, Loss: 2.513631582260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8305 , TFLOPS: 71.55805090515051, Tokens per sec: 177328.73291301043, Loss: 2.4797606468200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8306 , TFLOPS: 68.6236702426495, Tokens per sec: 170057.01438261804, Loss: 2.501267433166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8307 , TFLOPS: 69.37067771972349, Tokens per sec: 171908.18120047418, Loss: 2.5082297325134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8308 , TFLOPS: 71.50877660431355, Tokens per sec: 177206.62576752412, Loss: 2.509603977203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8309 , TFLOPS: 68.97218737495034, Tokens per sec: 170920.67822879105, Loss: 2.533452033996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8310 , TFLOPS: 68.80524386314545, Tokens per sec: 170506.97381619253, Loss: 2.49879789352417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8311 , TFLOPS: 71.44166490169387, Tokens per sec: 177040.3155754682, Loss: 2.5019428730010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8312 , TFLOPS: 71.53089807624266, Tokens per sec: 177261.44521184577, Loss: 2.500710964202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8313 , TFLOPS: 68.2133330700496, Tokens per sec: 169040.15366654441, Loss: 2.506844997406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8314 , TFLOPS: 69.60442337520811, Tokens per sec: 172487.42868397472, Loss: 2.505028009414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8315 , TFLOPS: 71.4874933601185, Tokens per sec: 177153.8835438659, Loss: 2.522181510925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8316 , TFLOPS: 69.83529425977245, Tokens per sec: 173059.55216845256, Loss: 2.4840290546417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8317 , TFLOPS: 67.31583792489741, Tokens per sec: 166816.0618296053, Loss: 2.4882562160491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8318 , TFLOPS: 71.51599824783916, Tokens per sec: 177224.5217957103, Loss: 2.486776113510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8319 , TFLOPS: 70.56563876190202, Tokens per sec: 174869.42629881823, Loss: 2.501885175704956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8320 , TFLOPS: 68.92939995957126, Tokens per sec: 170814.64630005875, Loss: 2.4848384857177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8321 , TFLOPS: 70.42884737144583, Tokens per sec: 174530.44216445216, Loss: 2.484066963195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8322 , TFLOPS: 70.67081856222796, Tokens per sec: 175130.0734305374, Loss: 2.5125555992126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8323 , TFLOPS: 68.94698132084176, Tokens per sec: 170858.2148500339, Loss: 2.506481170654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8324 , TFLOPS: 69.78474788766542, Tokens per sec: 172934.29269022882, Loss: 2.492358684539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8325 , TFLOPS: 71.51347932112594, Tokens per sec: 177218.27961783225, Loss: 2.5074493885040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8326 , TFLOPS: 69.8889354918956, Tokens per sec: 173192.48105072323, Loss: 2.5144472122192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8327 , TFLOPS: 69.57361060471031, Tokens per sec: 172411.07124437427, Loss: 2.5017642974853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8328 , TFLOPS: 70.55364710526989, Tokens per sec: 174839.70965269377, Loss: 2.5209038257598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8329 , TFLOPS: 70.66316916980618, Tokens per sec: 175111.1173934661, Loss: 2.5153493881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8330 , TFLOPS: 68.2655021119971, Tokens per sec: 169169.43429938465, Loss: 2.5019288063049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8331 , TFLOPS: 70.58433753073375, Tokens per sec: 174915.76390782266, Loss: 2.4953927993774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8332 , TFLOPS: 71.54487998099644, Tokens per sec: 177296.09391205956, Loss: 2.4833481311798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8333 , TFLOPS: 69.03535320392453, Tokens per sec: 171077.21011127956, Loss: 2.4871609210968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8334 , TFLOPS: 69.53342737645977, Tokens per sec: 172311.4927782214, Loss: 2.4882850646972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8335 , TFLOPS: 70.47579538315149, Tokens per sec: 174646.7845092092, Loss: 2.497349262237549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8336 , TFLOPS: 69.79871479469317, Tokens per sec: 172968.90422442538, Loss: 2.5103583335876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8337 , TFLOPS: 68.84147312131044, Tokens per sec: 170596.75390890765, Loss: 2.4987401962280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8338 , TFLOPS: 71.54040850375779, Tokens per sec: 177285.0130988883, Loss: 2.506153106689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8339 , TFLOPS: 69.65578740131018, Tokens per sec: 172614.71439887022, Loss: 2.4723799228668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8340 , TFLOPS: 69.9008393618645, Tokens per sec: 173221.98015182573, Loss: 2.4915738105773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8341 , TFLOPS: 70.28210524489745, Tokens per sec: 174166.79900988555, Loss: 2.504884719848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8342 , TFLOPS: 70.69183939693883, Tokens per sec: 175182.16537458703, Loss: 2.512160539627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8343 , TFLOPS: 68.85000962135875, Tokens per sec: 170617.90829640042, Loss: 2.494516372680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8344 , TFLOPS: 69.8685516559392, Tokens per sec: 173141.9676597584, Loss: 2.5176432132720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8345 , TFLOPS: 71.50422869217523, Tokens per sec: 177195.35554025148, Loss: 2.5248520374298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8346 , TFLOPS: 69.5760132963423, Tokens per sec: 172417.02537891665, Loss: 2.496321201324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8347 , TFLOPS: 69.18778803529462, Tokens per sec: 171454.96041549754, Loss: 2.5013530254364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8348 , TFLOPS: 71.51942071872656, Tokens per sec: 177233.00305558558, Loss: 2.5034470558166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8349 , TFLOPS: 70.66269886076441, Tokens per sec: 175109.95191585177, Loss: 2.4919378757476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8350 , TFLOPS: 67.83220707777703, Tokens per sec: 168095.68147319896, Loss: 2.5012974739074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8351 , TFLOPS: 71.47781115425478, Tokens per sec: 177129.8899711526, Loss: 2.4831619262695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8352 , TFLOPS: 71.54336041033184, Tokens per sec: 177292.3282485582, Loss: 2.51546311378479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8353 , TFLOPS: 68.97817794870899, Tokens per sec: 170935.52353048115, Loss: 2.5214362144470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8354 , TFLOPS: 68.56934794509941, Tokens per sec: 169922.39774519386, Loss: 2.4966812133789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8355 , TFLOPS: 71.5271291848267, Tokens per sec: 177252.10548373935, Loss: 2.520885944366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8356 , TFLOPS: 69.74465350505595, Tokens per sec: 172834.93439336168, Loss: 2.5068047046661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8357 , TFLOPS: 68.04004298660692, Tokens per sec: 168610.72175029488, Loss: 2.5214691162109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8358 , TFLOPS: 71.52150970346166, Tokens per sec: 177238.1797898798, Loss: 2.5016250610351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8359 , TFLOPS: 70.66494171038417, Tokens per sec: 175115.5099442798, Loss: 2.490670919418335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8360 , TFLOPS: 70.66951511101013, Tokens per sec: 175126.8433348326, Loss: 2.5290908813476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8361 , TFLOPS: 68.80560927587511, Tokens per sec: 170507.87934919016, Loss: 2.500211477279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8362 , TFLOPS: 71.52806930726165, Tokens per sec: 177254.4352106962, Loss: 2.493726968765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8363 , TFLOPS: 69.94244393830036, Tokens per sec: 173325.08087535622, Loss: 2.4911155700683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8364 , TFLOPS: 69.19464147524909, Tokens per sec: 171471.94399467346, Loss: 2.4988644123077393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8365 , TFLOPS: 70.47171618225705, Tokens per sec: 174636.67580003172, Loss: 2.520475149154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8366 , TFLOPS: 70.70647848156338, Tokens per sec: 175218.44263891497, Loss: 2.516433000564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8367 , TFLOPS: 68.82498423770727, Tokens per sec: 170555.8926389385, Loss: 2.501333713531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8368 , TFLOPS: 70.39804046096798, Tokens per sec: 174454.09924662672, Loss: 2.493350028991699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8369 , TFLOPS: 70.5730209567659, Tokens per sec: 174887.7201909068, Loss: 2.4906787872314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8370 , TFLOPS: 67.97326860969447, Tokens per sec: 168445.24748850096, Loss: 2.497567892074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8371 , TFLOPS: 70.06287884111094, Tokens per sec: 173623.53183152064, Loss: 2.4937331676483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8372 , TFLOPS: 70.42263451213537, Tokens per sec: 174515.0460146779, Loss: 2.501556158065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8373 , TFLOPS: 69.064631168148, Tokens per sec: 171149.76413185932, Loss: 2.494081497192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8374 , TFLOPS: 69.67469190116547, Tokens per sec: 172661.56183201418, Loss: 2.508247137069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8375 , TFLOPS: 71.52788635605184, Tokens per sec: 177253.98183744453, Loss: 2.4995594024658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8376 , TFLOPS: 68.73391979225974, Tokens per sec: 170330.22491154273, Loss: 2.5037004947662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8377 , TFLOPS: 68.83826313067088, Tokens per sec: 170588.79919849313, Loss: 2.5062978267669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8378 , TFLOPS: 71.48237615752473, Tokens per sec: 177141.2025521321, Loss: 2.5006041526794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8379 , TFLOPS: 70.4458473349321, Tokens per sec: 174572.5699466717, Loss: 2.4967339038848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8380 , TFLOPS: 70.12772828129631, Tokens per sec: 173784.23588805555, Loss: 2.4932801723480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8381 , TFLOPS: 69.65536146377376, Tokens per sec: 172613.65887873404, Loss: 2.5181946754455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8382 , TFLOPS: 71.53271586719265, Tokens per sec: 177265.94989806606, Loss: 2.525607109069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8383 , TFLOPS: 68.61984372203297, Tokens per sec: 170047.53184300425, Loss: 2.5132033824920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8384 , TFLOPS: 68.84292428615888, Tokens per sec: 170600.35005526105, Loss: 2.5051774978637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8385 , TFLOPS: 71.51429343831381, Tokens per sec: 177220.2970899062, Loss: 2.4944818019866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8386 , TFLOPS: 70.68200352058338, Tokens per sec: 175157.790989467, Loss: 2.5076324939727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8387 , TFLOPS: 68.7218756722134, Tokens per sec: 170300.3782552985, Loss: 2.495936870574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8388 , TFLOPS: 71.50494717408749, Tokens per sec: 177197.13601757737, Loss: 2.510645627975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8389 , TFLOPS: 70.58543077038598, Tokens per sec: 174918.4730761112, Loss: 2.4806694984436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8390 , TFLOPS: 69.85556017180197, Tokens per sec: 173109.77333093624, Loss: 2.5092666149139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8391 , TFLOPS: 69.40752715995848, Tokens per sec: 171999.4981726774, Loss: 2.4917140007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8392 , TFLOPS: 71.51778507215322, Tokens per sec: 177228.9497431955, Loss: 2.4954142570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8393 , TFLOPS: 69.826955575774, Tokens per sec: 173038.88799092252, Loss: 2.5123159885406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8394 , TFLOPS: 68.90086055935221, Tokens per sec: 170743.92252244148, Loss: 2.5019118785858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8395 , TFLOPS: 71.50346600532792, Tokens per sec: 177193.465518227, Loss: 2.4880800247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8396 , TFLOPS: 70.62008352139492, Tokens per sec: 175004.34641042698, Loss: 2.526247024536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8397 , TFLOPS: 68.34096161804261, Tokens per sec: 169356.43126791625, Loss: 2.502701997756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8398 , TFLOPS: 69.8273841855847, Tokens per sec: 173039.95013324873, Loss: 2.483684539794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8399 , TFLOPS: 71.52332145175247, Tokens per sec: 177242.66950172494, Loss: 2.5114846229553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8400 , TFLOPS: 69.81594685192866, Tokens per sec: 173011.60716052272, Loss: 2.51444149017334 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/optimizer.pt +[rank0]:[2024-08-26 19:44:14,255] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006332559998554643, 'preprocessing_with_comm': 0.0013331209993339144, 'state_converting': 0.9899513450000086, : 0.9990123610041337}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006400 + eval ppl=10.176913261413574, eval loss=2.3201217651367188 +------------------------------------------------------------------ +iteration: 8401 , TFLOPS: 65.16325412847581, Tokens per sec: 161481.72205539208, Loss: 2.5144412517547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8402 , TFLOPS: 69.11226567743871, Tokens per sec: 171267.8076932561, Loss: 2.519683361053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8403 , TFLOPS: 70.60800290917777, Tokens per sec: 174974.40932256362, Loss: 2.511923313140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8404 , TFLOPS: 68.42495536585531, Tokens per sec: 169564.5770276713, Loss: 2.4935357570648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8405 , TFLOPS: 70.39830493884709, Tokens per sec: 174454.75465194578, Loss: 2.5124428272247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8406 , TFLOPS: 69.90983767894282, Tokens per sec: 173244.27897279226, Loss: 2.5112311840057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8407 , TFLOPS: 68.02305005087986, Tokens per sec: 168568.61138363677, Loss: 2.511889934539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8408 , TFLOPS: 71.52825300345359, Tokens per sec: 177254.89043009563, Loss: 2.471902370452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8409 , TFLOPS: 69.50710495734229, Tokens per sec: 172246.26292399448, Loss: 2.4966416358947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8410 , TFLOPS: 68.00193011038247, Tokens per sec: 168516.27384453078, Loss: 2.4787375926971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8411 , TFLOPS: 70.6961730915073, Tokens per sec: 175192.9047471236, Loss: 2.5075457096099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8412 , TFLOPS: 71.52949612066212, Tokens per sec: 177257.971011479, Loss: 2.50559663772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8413 , TFLOPS: 67.90580167046481, Tokens per sec: 168278.05698099293, Loss: 2.517397165298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8414 , TFLOPS: 68.74483868314749, Tokens per sec: 170357.2831259777, Loss: 2.490241765975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8415 , TFLOPS: 71.51286140663814, Tokens per sec: 177216.74835766255, Loss: 2.499727964401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8416 , TFLOPS: 69.7732822266952, Tokens per sec: 172905.8795192978, Loss: 2.5131731033325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8417 , TFLOPS: 67.92273580800469, Tokens per sec: 168320.021639266, Loss: 2.4990532398223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8418 , TFLOPS: 70.68844164800859, Tokens per sec: 175173.74537844496, Loss: 2.484528064727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8419 , TFLOPS: 70.53813815767515, Tokens per sec: 174801.2767721024, Loss: 2.491532325744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8420 , TFLOPS: 69.64815680745699, Tokens per sec: 172595.80494672267, Loss: 2.4916768074035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8421 , TFLOPS: 69.02582862202996, Tokens per sec: 171053.6071481255, Loss: 2.472313404083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8422 , TFLOPS: 71.47190746559161, Tokens per sec: 177115.25998030024, Loss: 2.4979679584503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8423 , TFLOPS: 70.05820191541002, Tokens per sec: 173611.94189442781, Loss: 2.5128118991851807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8424 , TFLOPS: 67.31630594952222, Tokens per sec: 166817.22164618273, Loss: 2.4883460998535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8425 , TFLOPS: 71.50296473476624, Tokens per sec: 177192.22331455557, Loss: 2.516536235809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8426 , TFLOPS: 70.59471043798533, Tokens per sec: 174941.46911466218, Loss: 2.499269485473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8427 , TFLOPS: 67.8353735195031, Tokens per sec: 168103.52826460853, Loss: 2.49902081489563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8428 , TFLOPS: 70.89185844536563, Tokens per sec: 175677.8346105058, Loss: 2.519925117492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8429 , TFLOPS: 70.59208069477475, Tokens per sec: 174934.95232129624, Loss: 2.4868037700653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8430 , TFLOPS: 69.04054124722941, Tokens per sec: 171090.0666540992, Loss: 2.508342981338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8431 , TFLOPS: 68.90283333229065, Tokens per sec: 170748.81127110185, Loss: 2.5023632049560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8432 , TFLOPS: 71.51925827427281, Tokens per sec: 177232.60050033443, Loss: 2.50221586227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8433 , TFLOPS: 68.43173822649098, Tokens per sec: 169581.38570352027, Loss: 2.469125747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8434 , TFLOPS: 68.79327140029096, Tokens per sec: 170477.30473436342, Loss: 2.5108649730682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8435 , TFLOPS: 70.17224333203762, Tokens per sec: 173894.54908753018, Loss: 2.5128591060638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8436 , TFLOPS: 69.78530123447207, Tokens per sec: 172935.66394457186, Loss: 2.500478982925415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8437 , TFLOPS: 68.73761826489716, Tokens per sec: 170339.39013415863, Loss: 2.5020110607147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8438 , TFLOPS: 70.64445957218919, Tokens per sec: 175064.75294953884, Loss: 2.4619407653808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8439 , TFLOPS: 70.4406072767511, Tokens per sec: 174559.58450525912, Loss: 2.5037732124328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8440 , TFLOPS: 70.06209788848876, Tokens per sec: 173621.59654489384, Loss: 2.506666421890259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8441 , TFLOPS: 68.75238778965719, Tokens per sec: 170375.99064351144, Loss: 2.5077133178710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8442 , TFLOPS: 70.45267771606248, Tokens per sec: 174589.4963835414, Loss: 2.51212477684021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8443 , TFLOPS: 69.8199797573888, Tokens per sec: 173021.60114451416, Loss: 2.483341932296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8444 , TFLOPS: 68.01049570829448, Tokens per sec: 168537.5003397352, Loss: 2.4995980262756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8445 , TFLOPS: 71.53679856236164, Tokens per sec: 177276.0672664408, Loss: 2.4681618213653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8446 , TFLOPS: 70.40439987450036, Tokens per sec: 174469.85857390717, Loss: 2.503894329071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8447 , TFLOPS: 68.94308059431512, Tokens per sec: 170848.54842000036, Loss: 2.502573251724243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8448 , TFLOPS: 70.6666103303424, Tokens per sec: 175119.64496834998, Loss: 2.5106873512268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8449 , TFLOPS: 70.45078267739184, Tokens per sec: 174584.8002689598, Loss: 2.4883384704589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8450 , TFLOPS: 68.74087785334851, Tokens per sec: 170347.46775341438, Loss: 2.483701705932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8451 , TFLOPS: 69.02494144376999, Tokens per sec: 171051.40862266652, Loss: 2.503851890563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8452 , TFLOPS: 71.55113721119642, Tokens per sec: 177311.60001778594, Loss: 2.4769325256347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8453 , TFLOPS: 69.7429833796678, Tokens per sec: 172830.79563867036, Loss: 2.5048389434814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8454 , TFLOPS: 67.92727791360686, Tokens per sec: 168331.27747730244, Loss: 2.511604070663452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8455 , TFLOPS: 71.50027223975745, Tokens per sec: 177185.55101532102, Loss: 2.507962703704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8456 , TFLOPS: 70.71703430604471, Tokens per sec: 175244.60113479977, Loss: 2.508925676345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8457 , TFLOPS: 68.12157564615417, Tokens per sec: 168812.76866221684, Loss: 2.4909353256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8458 , TFLOPS: 70.60755365487674, Tokens per sec: 174973.29602091623, Loss: 2.5210511684417725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8459 , TFLOPS: 71.50815083200612, Tokens per sec: 177205.0750348118, Loss: 2.5028340816497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8460 , TFLOPS: 69.80288246706348, Tokens per sec: 172979.23217566602, Loss: 2.4905152320861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8461 , TFLOPS: 66.72580681777177, Tokens per sec: 165353.89974885585, Loss: 2.499711036682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8462 , TFLOPS: 71.54037611951485, Tokens per sec: 177284.9328471669, Loss: 2.4936678409576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8463 , TFLOPS: 69.9415209190539, Tokens per sec: 173322.79353198456, Loss: 2.5001630783081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8464 , TFLOPS: 67.84627808041157, Tokens per sec: 168130.5509677763, Loss: 2.4981374740600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8465 , TFLOPS: 70.90942494042675, Tokens per sec: 175721.3663203764, Loss: 2.5038974285125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8466 , TFLOPS: 70.54380094528696, Tokens per sec: 174815.30978361206, Loss: 2.4976956844329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8467 , TFLOPS: 69.91522804451176, Tokens per sec: 173257.6368924699, Loss: 2.49444842338562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8468 , TFLOPS: 68.96397390557992, Tokens per sec: 170900.32434690278, Loss: 2.513172149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8469 , TFLOPS: 71.47742601345648, Tokens per sec: 177128.9355498276, Loss: 2.521444082260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8470 , TFLOPS: 69.37532802351328, Tokens per sec: 171919.70516553856, Loss: 2.501673698425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8471 , TFLOPS: 68.73799650694356, Tokens per sec: 170340.3274596164, Loss: 2.5029778480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8472 , TFLOPS: 70.53905367431166, Tokens per sec: 174803.54552318016, Loss: 2.4961061477661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8473 , TFLOPS: 69.65681382250924, Tokens per sec: 172617.2579836711, Loss: 2.5094473361968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8474 , TFLOPS: 68.01115402048175, Tokens per sec: 168539.13170985435, Loss: 2.5129005908966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8475 , TFLOPS: 71.52269940568021, Tokens per sec: 177241.1280030339, Loss: 2.5004889965057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8476 , TFLOPS: 69.6716334179709, Tokens per sec: 172653.9825737383, Loss: 2.5062403678894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8477 , TFLOPS: 68.0601697011504, Tokens per sec: 168660.59796607425, Loss: 2.5072617530822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8478 , TFLOPS: 70.6096001532283, Tokens per sec: 174978.36746927176, Loss: 2.4950308799743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8479 , TFLOPS: 69.52312259105575, Tokens per sec: 172285.95638482444, Loss: 2.5151708126068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8480 , TFLOPS: 69.77782681300796, Tokens per sec: 172917.14150480833, Loss: 2.5116498470306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8481 , TFLOPS: 68.7307513738784, Tokens per sec: 170322.37322176364, Loss: 2.50433349609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8482 , TFLOPS: 71.4948018063541, Tokens per sec: 177171.99467872953, Loss: 2.4994277954101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8483 , TFLOPS: 68.89462104095806, Tokens per sec: 170728.46030852105, Loss: 2.5023069381713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8484 , TFLOPS: 68.91831828970246, Tokens per sec: 170787.18470137715, Loss: 2.486323833465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8485 , TFLOPS: 70.65164725412627, Tokens per sec: 175082.5648171675, Loss: 2.486635208129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8486 , TFLOPS: 70.62784430101699, Tokens per sec: 175023.57847725094, Loss: 2.4840831756591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8487 , TFLOPS: 69.74630465431763, Tokens per sec: 172839.02612312124, Loss: 2.5039563179016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8488 , TFLOPS: 69.07638766027428, Tokens per sec: 171178.89801443278, Loss: 2.513404369354248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8489 , TFLOPS: 71.54658627826363, Tokens per sec: 177300.32230465184, Loss: 2.4852452278137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8490 , TFLOPS: 69.8922887667037, Tokens per sec: 173200.79083509147, Loss: 2.520596981048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8491 , TFLOPS: 67.20971237211035, Tokens per sec: 166553.07101910422, Loss: 2.5172464847564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8492 , TFLOPS: 71.52079894648436, Tokens per sec: 177236.41845579373, Loss: 2.4922022819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8493 , TFLOPS: 69.6534165940971, Tokens per sec: 172608.8392774302, Loss: 2.4824514389038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8494 , TFLOPS: 67.17026341136557, Tokens per sec: 166455.3121487169, Loss: 2.490414619445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8495 , TFLOPS: 71.5353861590154, Tokens per sec: 177272.56717535973, Loss: 2.4904468059539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8496 , TFLOPS: 70.72787787387736, Tokens per sec: 175271.4726904067, Loss: 2.495333671569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8497 , TFLOPS: 68.1067968786908, Tokens per sec: 168776.145248427, Loss: 2.517714262008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8498 , TFLOPS: 69.7799995491164, Tokens per sec: 172922.52578423166, Loss: 2.4956886768341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8499 , TFLOPS: 71.54297073441904, Tokens per sec: 177291.36258871993, Loss: 2.4973647594451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8500 , TFLOPS: 69.7435734022052, Tokens per sec: 172832.2577795144, Loss: 2.483705997467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8501 , TFLOPS: 67.78834436968822, Tokens per sec: 167986.98485067874, Loss: 2.4994661808013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8502 , TFLOPS: 71.52661970695458, Tokens per sec: 177250.8429414486, Loss: 2.5075924396514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8503 , TFLOPS: 69.85029420654296, Tokens per sec: 173096.72368893042, Loss: 2.5056700706481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8504 , TFLOPS: 69.00562151107708, Tokens per sec: 171003.53170118725, Loss: 2.494048833847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8505 , TFLOPS: 69.68767530992687, Tokens per sec: 172693.7361491655, Loss: 2.492241621017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8506 , TFLOPS: 71.51763455657714, Tokens per sec: 177228.57674901726, Loss: 2.502178430557251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8507 , TFLOPS: 70.01750411121081, Tokens per sec: 173511.08825238896, Loss: 2.4922549724578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8508 , TFLOPS: 68.88312717524727, Tokens per sec: 170699.97724313595, Loss: 2.4996187686920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8509 , TFLOPS: 70.42221164032951, Tokens per sec: 174513.9980917598, Loss: 2.5166711807250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8510 , TFLOPS: 69.90446550600045, Tokens per sec: 173230.96613644736, Loss: 2.520681858062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8511 , TFLOPS: 68.03261399194635, Tokens per sec: 168592.31188315465, Loss: 2.511542558670044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8512 , TFLOPS: 71.53040210348713, Tokens per sec: 177260.21613672166, Loss: 2.503589630126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8513 , TFLOPS: 68.81163153958582, Tokens per sec: 170522.8031820724, Loss: 2.4660966396331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8514 , TFLOPS: 68.69995073961101, Tokens per sec: 170246.04585707912, Loss: 2.4733636379241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8515 , TFLOPS: 70.96548004707793, Tokens per sec: 175860.27704963778, Loss: 2.479863405227661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8516 , TFLOPS: 69.64854980964898, Tokens per sec: 172596.7788494472, Loss: 2.497485876083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8517 , TFLOPS: 68.91096270406251, Tokens per sec: 170768.95674987676, Loss: 2.4966464042663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8518 , TFLOPS: 69.91815813841052, Tokens per sec: 173264.8979879275, Loss: 2.523284435272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8519 , TFLOPS: 71.52895424473046, Tokens per sec: 177256.62818322782, Loss: 2.491748809814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8520 , TFLOPS: 68.79068492477505, Tokens per sec: 170470.8951631112, Loss: 2.511583089828491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8521 , TFLOPS: 68.81690315323746, Tokens per sec: 170535.86682141735, Loss: 2.4907312393188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8522 , TFLOPS: 71.52217087922777, Tokens per sec: 177239.81825626275, Loss: 2.4810376167297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8523 , TFLOPS: 70.72392395463476, Tokens per sec: 175261.67444296335, Loss: 2.4955124855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8524 , TFLOPS: 67.14364350512196, Tokens per sec: 166389.3450886205, Loss: 2.5082647800445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8525 , TFLOPS: 70.73349021861645, Tokens per sec: 175285.38069892096, Loss: 2.4960029125213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8526 , TFLOPS: 71.56680200085462, Tokens per sec: 177350.419092178, Loss: 2.495535373687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8527 , TFLOPS: 69.78974859162085, Tokens per sec: 172946.6849855034, Loss: 2.525237798690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8528 , TFLOPS: 67.88005707900558, Tokens per sec: 168214.2590473561, Loss: 2.5085408687591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8529 , TFLOPS: 71.5039925845436, Tokens per sec: 177194.77043953087, Loss: 2.4950451850891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8530 , TFLOPS: 69.85714676373556, Tokens per sec: 173113.70508052516, Loss: 2.5371971130371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8531 , TFLOPS: 68.67312174507276, Tokens per sec: 170179.5606531556, Loss: 2.491344928741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8532 , TFLOPS: 70.93686926629, Tokens per sec: 175789.37638310762, Loss: 2.511836528778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8533 , TFLOPS: 69.7603137626298, Tokens per sec: 172873.74223674947, Loss: 2.516798734664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8534 , TFLOPS: 69.82314331742195, Tokens per sec: 173029.44079477125, Loss: 2.502304792404175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8535 , TFLOPS: 68.92444459177118, Tokens per sec: 170802.36635277132, Loss: 2.511678695678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8536 , TFLOPS: 70.64352576439858, Tokens per sec: 175062.4388709663, Loss: 2.487539052963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8537 , TFLOPS: 69.75148825338088, Tokens per sec: 172851.87165262052, Loss: 2.483902931213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8538 , TFLOPS: 68.11600645960017, Tokens per sec: 168798.96760443947, Loss: 2.509408473968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8539 , TFLOPS: 71.51262565742778, Tokens per sec: 177216.16414514938, Loss: 2.5164949893951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8540 , TFLOPS: 70.59194383935447, Tokens per sec: 174934.61317848891, Loss: 2.480923652648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8541 , TFLOPS: 68.25054071297028, Tokens per sec: 169132.35830446205, Loss: 2.502883195877075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8542 , TFLOPS: 70.53314553069461, Tokens per sec: 174788.90449246136, Loss: 2.4923765659332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8543 , TFLOPS: 70.70515963580209, Tokens per sec: 175215.17439383565, Loss: 2.485750675201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8544 , TFLOPS: 68.00552309413614, Tokens per sec: 168525.17765407218, Loss: 2.516913890838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8545 , TFLOPS: 70.69543354891783, Tokens per sec: 175191.07207911953, Loss: 2.496000051498413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8546 , TFLOPS: 70.54933453261344, Tokens per sec: 174829.022622582, Loss: 2.5145859718322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8547 , TFLOPS: 69.85205359028595, Tokens per sec: 173101.0836356577, Loss: 2.5082263946533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8548 , TFLOPS: 68.82437927151085, Tokens per sec: 170554.39346606124, Loss: 2.491798162460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8549 , TFLOPS: 71.54860197202804, Tokens per sec: 177305.31741584704, Loss: 2.520507335662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8550 , TFLOPS: 68.72725051338841, Tokens per sec: 170313.69770381783, Loss: 2.5231785774230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8551 , TFLOPS: 68.6409764432682, Tokens per sec: 170099.90105418657, Loss: 2.4997997283935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8552 , TFLOPS: 70.62781692108418, Tokens per sec: 175023.5106267613, Loss: 2.502143621444702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8553 , TFLOPS: 69.58029113433643, Tokens per sec: 172427.6263327089, Loss: 2.513148784637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8554 , TFLOPS: 70.2012025381125, Tokens per sec: 173966.31319030363, Loss: 2.510274887084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8555 , TFLOPS: 69.0764847611651, Tokens per sec: 171179.1386411368, Loss: 2.5142059326171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8556 , TFLOPS: 71.55626294828124, Tokens per sec: 177324.30215892868, Loss: 2.5001912117004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8557 , TFLOPS: 69.00111704091988, Tokens per sec: 170992.36912792936, Loss: 2.4978535175323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8558 , TFLOPS: 68.40024168798274, Tokens per sec: 169503.3338114667, Loss: 2.49853253364563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8559 , TFLOPS: 71.53553469050482, Tokens per sec: 177272.9352527526, Loss: 2.503868579864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8560 , TFLOPS: 70.632294923641, Tokens per sec: 175034.6076103898, Loss: 2.5019993782043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8561 , TFLOPS: 67.73464136418463, Tokens per sec: 167853.90288716642, Loss: 2.4987974166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8562 , TFLOPS: 71.53693782036856, Tokens per sec: 177276.41236312248, Loss: 2.5056629180908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8563 , TFLOPS: 70.67155069382605, Tokens per sec: 175131.88773328843, Loss: 2.49996280670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8564 , TFLOPS: 68.0159980053347, Tokens per sec: 168551.13563792894, Loss: 2.5014820098876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8565 , TFLOPS: 69.70713274949779, Tokens per sec: 172741.9538278361, Loss: 2.501699686050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8566 , TFLOPS: 71.54489106070825, Tokens per sec: 177296.121368806, Loss: 2.4798531532287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8567 , TFLOPS: 69.85718191890759, Tokens per sec: 173113.79219891413, Loss: 2.4924378395080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8568 , TFLOPS: 68.74839008688288, Tokens per sec: 170366.0838956534, Loss: 2.4879486560821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8569 , TFLOPS: 71.46309365219943, Tokens per sec: 177093.4183797927, Loss: 2.4983654022216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8570 , TFLOPS: 69.78943920056061, Tokens per sec: 172945.91828037385, Loss: 2.5079145431518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8571 , TFLOPS: 68.76078178319436, Tokens per sec: 170396.7918841713, Loss: 2.4914283752441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8572 , TFLOPS: 69.64963152512544, Tokens per sec: 172599.45945955868, Loss: 2.5010643005371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8573 , TFLOPS: 71.5089962372307, Tokens per sec: 177207.17004208738, Loss: 2.481157064437866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8574 , TFLOPS: 69.26539259289744, Tokens per sec: 171647.2730580851, Loss: 2.4898324012756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8575 , TFLOPS: 67.75573997820933, Tokens per sec: 167906.18757692384, Loss: 2.5163888931274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8576 , TFLOPS: 71.56275724209671, Tokens per sec: 177340.39573440893, Loss: 2.506002426147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8577 , TFLOPS: 69.77777929462995, Tokens per sec: 172917.02374903302, Loss: 2.4997611045837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8578 , TFLOPS: 68.19834552517928, Tokens per sec: 169003.01287934138, Loss: 2.513684034347534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8579 , TFLOPS: 70.47839053742504, Tokens per sec: 174653.21558737892, Loss: 2.508843421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8580 , TFLOPS: 70.62200152691602, Tokens per sec: 175009.09943939405, Loss: 2.5133750438690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8581 , TFLOPS: 68.85148029863423, Tokens per sec: 170621.55279669754, Loss: 2.4973978996276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8582 , TFLOPS: 70.85906927371326, Tokens per sec: 175596.57943112578, Loss: 2.502112627029419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8583 , TFLOPS: 69.60493663982558, Tokens per sec: 172488.70061023757, Loss: 2.5197982788085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8584 , TFLOPS: 68.35029660329198, Tokens per sec: 169379.5643896977, Loss: 2.494271993637085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8585 , TFLOPS: 69.99220229764087, Tokens per sec: 173448.38757113778, Loss: 2.470876932144165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8586 , TFLOPS: 71.48043780922819, Tokens per sec: 177136.3991115266, Loss: 2.489152669906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8587 , TFLOPS: 69.04683674322057, Tokens per sec: 171105.6675866714, Loss: 2.51068115234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8588 , TFLOPS: 68.75056449192137, Tokens per sec: 170371.47231086929, Loss: 2.5252938270568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8589 , TFLOPS: 71.52542465634757, Tokens per sec: 177247.88147439878, Loss: 2.505204439163208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8590 , TFLOPS: 69.59010468799718, Tokens per sec: 172451.9453997324, Loss: 2.5171430110931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8591 , TFLOPS: 68.10053158739292, Tokens per sec: 168760.61916641204, Loss: 2.5122714042663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8592 , TFLOPS: 70.68804644950609, Tokens per sec: 175172.76603302106, Loss: 2.498076915740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8593 , TFLOPS: 71.52059076750773, Tokens per sec: 177235.9025653568, Loss: 2.5164589881896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8594 , TFLOPS: 68.8614762881378, Tokens per sec: 170646.32395983656, Loss: 2.5085175037384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8595 , TFLOPS: 68.72796898066815, Tokens per sec: 170315.47814488277, Loss: 2.504680633544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8596 , TFLOPS: 71.48143692706623, Tokens per sec: 177138.87503559116, Loss: 2.471952199935913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8597 , TFLOPS: 69.65855640671859, Tokens per sec: 172621.57629930336, Loss: 2.4944007396698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8598 , TFLOPS: 67.88535050030403, Tokens per sec: 168227.37672845216, Loss: 2.5033068656921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8599 , TFLOPS: 71.01010213660103, Tokens per sec: 175970.855503005, Loss: 2.497192859649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8600 , TFLOPS: 70.69930735797126, Tokens per sec: 175200.67180468922, Loss: 2.4963865280151367 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/optimizer.pt +[rank0]:[2024-08-26 20:09:38,613] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063732590060681105, 'preprocessing_with_comm': 0.0013018419995205477, 'state_converting': 0.9919552510109497, : 1.0010036099993158}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600, took 4.27s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006600 + eval ppl=9.963233947753906, eval loss=2.2989017963409424 +------------------------------------------------------------------ +iteration: 8601 , TFLOPS: 62.3383035953554, Tokens per sec: 154481.18344032872, Loss: 2.4875855445861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8602 , TFLOPS: 70.93857326093757, Tokens per sec: 175793.59906955482, Loss: 2.4692323207855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8603 , TFLOPS: 70.50568855108797, Tokens per sec: 174720.8630723601, Loss: 2.5106635093688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8604 , TFLOPS: 68.2445782274316, Tokens per sec: 169117.5826084752, Loss: 2.515028953552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8605 , TFLOPS: 68.96071044343687, Tokens per sec: 170892.23712821226, Loss: 2.5102198123931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8606 , TFLOPS: 71.52818569608993, Tokens per sec: 177254.72363503397, Loss: 2.509467124938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8607 , TFLOPS: 69.77770879477995, Tokens per sec: 172916.84904263853, Loss: 2.516047716140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8608 , TFLOPS: 68.41513515232683, Tokens per sec: 169540.2414567619, Loss: 2.474836587905884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8609 , TFLOPS: 70.41418571984077, Tokens per sec: 174494.10897666056, Loss: 2.493225574493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8610 , TFLOPS: 69.71390190084348, Tokens per sec: 172758.72853055433, Loss: 2.503256320953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8611 , TFLOPS: 69.61747618618271, Tokens per sec: 172519.77498745557, Loss: 2.485193967819214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8612 , TFLOPS: 70.53127733729957, Tokens per sec: 174784.27490342996, Loss: 2.4844768047332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8613 , TFLOPS: 69.6459234049652, Tokens per sec: 172590.27032931845, Loss: 2.4897725582122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8614 , TFLOPS: 70.04098673251262, Tokens per sec: 173569.28077479915, Loss: 2.5094528198242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8615 , TFLOPS: 69.16013445791828, Tokens per sec: 171386.43180446594, Loss: 2.4850494861602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8616 , TFLOPS: 70.4470398308911, Tokens per sec: 174575.5250830224, Loss: 2.524312734603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8617 , TFLOPS: 70.01334766736413, Tokens per sec: 173500.7881266668, Loss: 2.5112874507904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8618 , TFLOPS: 68.77101959367147, Tokens per sec: 170422.1623063215, Loss: 2.500563144683838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8619 , TFLOPS: 71.52397541428704, Tokens per sec: 177244.29009292554, Loss: 2.5059444904327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8620 , TFLOPS: 69.6216952108099, Tokens per sec: 172530.23019524568, Loss: 2.5066962242126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8621 , TFLOPS: 68.10078744942142, Tokens per sec: 168761.25322070418, Loss: 2.4826440811157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8622 , TFLOPS: 71.52829183211344, Tokens per sec: 177254.98665179207, Loss: 2.487731456756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8623 , TFLOPS: 70.54188744816886, Tokens per sec: 174810.56792696426, Loss: 2.4911375045776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8624 , TFLOPS: 67.6972872047893, Tokens per sec: 167761.33516528446, Loss: 2.5094332695007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8625 , TFLOPS: 70.61640578605268, Tokens per sec: 174995.23257711294, Loss: 2.5040934085845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8626 , TFLOPS: 71.5166540181437, Tokens per sec: 177226.146866764, Loss: 2.497521162033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8627 , TFLOPS: 70.71549118385128, Tokens per sec: 175240.77710801552, Loss: 2.507364273071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8628 , TFLOPS: 67.05246664572547, Tokens per sec: 166163.39878707577, Loss: 2.4972574710845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8629 , TFLOPS: 71.48265359423979, Tokens per sec: 177141.89007087421, Loss: 2.511558771133423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8630 , TFLOPS: 69.60877173663195, Tokens per sec: 172498.20440259454, Loss: 2.492619514465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8631 , TFLOPS: 68.5534161071735, Tokens per sec: 169882.91689577664, Loss: 2.4947023391723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8632 , TFLOPS: 70.60875727258602, Tokens per sec: 174976.27871818922, Loss: 2.5029118061065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8633 , TFLOPS: 71.52318212112277, Tokens per sec: 177242.32422507607, Loss: 2.491227149963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8634 , TFLOPS: 69.35194340724766, Tokens per sec: 171861.7555104042, Loss: 2.4992270469665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8635 , TFLOPS: 68.36819854869458, Tokens per sec: 169423.92738246746, Loss: 2.5147805213928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8636 , TFLOPS: 71.51813725980233, Tokens per sec: 177229.82250298664, Loss: 2.4943056106567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8637 , TFLOPS: 69.7236327388692, Tokens per sec: 172782.84261912236, Loss: 2.498875856399536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8638 , TFLOPS: 67.80768191594497, Tokens per sec: 168034.90542050957, Loss: 2.4963245391845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8639 , TFLOPS: 71.52634827751852, Tokens per sec: 177250.17030940566, Loss: 2.520878314971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8640 , TFLOPS: 70.66029433416672, Tokens per sec: 175103.99323406225, Loss: 2.4973502159118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8641 , TFLOPS: 68.80670082472682, Tokens per sec: 170510.5843274887, Loss: 2.5048470497131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8642 , TFLOPS: 70.05411610098604, Tokens per sec: 173601.8167961956, Loss: 2.5202229022979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8643 , TFLOPS: 70.61990863303535, Tokens per sec: 175003.91301780645, Loss: 2.5023412704467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8644 , TFLOPS: 69.79565622487722, Tokens per sec: 172961.3247514922, Loss: 2.5099895000457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8645 , TFLOPS: 69.88938562118125, Tokens per sec: 173193.59652067896, Loss: 2.491239547729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8646 , TFLOPS: 70.45814022169266, Tokens per sec: 174603.03307423764, Loss: 2.5070676803588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8647 , TFLOPS: 70.67213874014153, Tokens per sec: 175133.34497683684, Loss: 2.5057108402252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8648 , TFLOPS: 68.17183563954998, Tokens per sec: 168937.318462447, Loss: 2.498615264892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8649 , TFLOPS: 71.52953932855164, Tokens per sec: 177258.078085389, Loss: 2.498380422592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8650 , TFLOPS: 69.59644269924875, Tokens per sec: 172467.65168980337, Loss: 2.499621629714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8651 , TFLOPS: 68.98445897497842, Tokens per sec: 170951.08860548897, Loss: 2.496718406677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8652 , TFLOPS: 70.54605574927885, Tokens per sec: 174820.8974362911, Loss: 2.492169141769409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8653 , TFLOPS: 70.55240904664757, Tokens per sec: 174836.64160704426, Loss: 2.5131683349609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8654 , TFLOPS: 68.58805272862408, Tokens per sec: 169968.75025927, Loss: 2.507105827331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8655 , TFLOPS: 68.97547536831394, Tokens per sec: 170928.8262385484, Loss: 2.496056079864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8656 , TFLOPS: 71.4466952484876, Tokens per sec: 177052.7813289615, Loss: 2.4957351684570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8657 , TFLOPS: 68.79347708236068, Tokens per sec: 170477.81443718995, Loss: 2.5198681354522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8658 , TFLOPS: 69.64144873910016, Tokens per sec: 172579.18161437972, Loss: 2.4955124855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8659 , TFLOPS: 70.83669412455714, Tokens per sec: 175541.13134669076, Loss: 2.512110948562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8660 , TFLOPS: 70.5658240564354, Tokens per sec: 174869.88547908384, Loss: 2.4915807247161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8661 , TFLOPS: 68.94865627190498, Tokens per sec: 170862.36556327963, Loss: 2.465609550476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8662 , TFLOPS: 70.00340008779223, Tokens per sec: 173476.13692867127, Loss: 2.5035314559936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8663 , TFLOPS: 70.58214128205456, Tokens per sec: 174910.3213616589, Loss: 2.4836864471435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8664 , TFLOPS: 69.71859174463798, Tokens per sec: 172770.35048010602, Loss: 2.4779462814331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8665 , TFLOPS: 68.90333592533318, Tokens per sec: 170750.0567520264, Loss: 2.5208568572998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8666 , TFLOPS: 71.46016807076492, Tokens per sec: 177086.1684667186, Loss: 2.541313409805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8667 , TFLOPS: 70.53690851859817, Tokens per sec: 174798.2295910137, Loss: 2.5049214363098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8668 , TFLOPS: 67.19178624375277, Tokens per sec: 166508.6480983081, Loss: 2.51550555229187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8669 , TFLOPS: 71.44649034491503, Tokens per sec: 177052.2735553367, Loss: 2.5005295276641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8670 , TFLOPS: 70.58573509140753, Tokens per sec: 174919.22721712632, Loss: 2.520786762237549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8671 , TFLOPS: 68.69198769411713, Tokens per sec: 170226.31255314342, Loss: 2.4851760864257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8672 , TFLOPS: 70.39780157687964, Tokens per sec: 174453.50726554057, Loss: 2.529444456100464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8673 , TFLOPS: 71.47966732069192, Tokens per sec: 177134.48975605742, Loss: 2.5046043395996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8674 , TFLOPS: 69.02494721223206, Tokens per sec: 171051.42291755104, Loss: 2.508481502532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8675 , TFLOPS: 67.93170096932434, Tokens per sec: 168342.2382966102, Loss: 2.4931225776672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8676 , TFLOPS: 71.51454809336597, Tokens per sec: 177220.92815317804, Loss: 2.494536876678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8677 , TFLOPS: 69.84037332580874, Tokens per sec: 173072.13865359582, Loss: 2.5103611946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8678 , TFLOPS: 69.58998058069247, Tokens per sec: 172451.6378481598, Loss: 2.5109708309173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8679 , TFLOPS: 69.7221135217208, Tokens per sec: 172779.07783167303, Loss: 2.503321647644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8680 , TFLOPS: 70.53163153656384, Tokens per sec: 174785.15264822508, Loss: 2.5053956508636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8681 , TFLOPS: 70.74221175310461, Tokens per sec: 175306.99362213895, Loss: 2.4949288368225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8682 , TFLOPS: 69.1335061626634, Tokens per sec: 171320.44395547546, Loss: 2.498307228088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8683 , TFLOPS: 70.42328759212504, Tokens per sec: 174516.664418835, Loss: 2.5143463611602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8684 , TFLOPS: 70.15508752295463, Tokens per sec: 173852.0351027541, Loss: 2.5114400386810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8685 , TFLOPS: 68.67557215092124, Tokens per sec: 170185.63302878264, Loss: 2.5080528259277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8686 , TFLOPS: 71.55305767618002, Tokens per sec: 177316.35914157197, Loss: 2.5073494911193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8687 , TFLOPS: 69.6451061043338, Tokens per sec: 172588.24496832074, Loss: 2.4825756549835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8688 , TFLOPS: 68.73802654662113, Tokens per sec: 170340.40190124657, Loss: 2.5021133422851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8689 , TFLOPS: 71.07032056762327, Tokens per sec: 176120.0834086859, Loss: 2.5068254470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8690 , TFLOPS: 69.67577161387949, Tokens per sec: 172664.23747905967, Loss: 2.4894673824310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8691 , TFLOPS: 69.86024114522588, Tokens per sec: 173121.3732987325, Loss: 2.497025728225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8692 , TFLOPS: 69.80304545404155, Tokens per sec: 172979.6360753522, Loss: 2.508061647415161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8693 , TFLOPS: 71.50993726413179, Tokens per sec: 177209.50201041094, Loss: 2.5288875102996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8694 , TFLOPS: 68.6195417084264, Tokens per sec: 170046.7834200176, Loss: 2.4907422065734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8695 , TFLOPS: 68.25144857815084, Tokens per sec: 169134.6080943891, Loss: 2.479369640350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8696 , TFLOPS: 71.48749720182711, Tokens per sec: 177153.89306404305, Loss: 2.478332996368408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8697 , TFLOPS: 69.54873685916303, Tokens per sec: 172349.43136283886, Loss: 2.497152328491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8698 , TFLOPS: 68.58172803519888, Tokens per sec: 169953.07697223985, Loss: 2.4946908950805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8699 , TFLOPS: 70.63140552700042, Tokens per sec: 175032.4035875393, Loss: 2.515472412109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8700 , TFLOPS: 71.50788138751406, Tokens per sec: 177204.4073216787, Loss: 2.509457588195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8701 , TFLOPS: 69.95456490052362, Tokens per sec: 173355.11795497968, Loss: 2.494121551513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8702 , TFLOPS: 67.38347870394972, Tokens per sec: 166983.68313134386, Loss: 2.5125021934509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8703 , TFLOPS: 71.55870825948341, Tokens per sec: 177330.36190946092, Loss: 2.4875173568725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8704 , TFLOPS: 69.87953943808871, Tokens per sec: 173169.19659432903, Loss: 2.506516695022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8705 , TFLOPS: 67.77209240423318, Tokens per sec: 167946.71068997937, Loss: 2.506014347076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8706 , TFLOPS: 71.55289389900719, Tokens per sec: 177315.95328369614, Loss: 2.4862611293792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8707 , TFLOPS: 70.54687233569858, Tokens per sec: 174822.9210273936, Loss: 2.5058541297912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8708 , TFLOPS: 68.73337514068228, Tokens per sec: 170328.87520493547, Loss: 2.5349698066711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8709 , TFLOPS: 69.71935060700343, Tokens per sec: 172772.23102464323, Loss: 2.504753828048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8710 , TFLOPS: 70.72142944392297, Tokens per sec: 175255.4927706265, Loss: 2.5009870529174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8711 , TFLOPS: 70.72226747796165, Tokens per sec: 175257.56951129145, Loss: 2.479137897491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8712 , TFLOPS: 66.8454466593147, Tokens per sec: 165650.38045561578, Loss: 2.5060112476348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8713 , TFLOPS: 71.53079723821645, Tokens per sec: 177261.1953241086, Loss: 2.490170955657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8714 , TFLOPS: 69.61375300835161, Tokens per sec: 172510.5485426482, Loss: 2.4913604259490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8715 , TFLOPS: 68.06324083193243, Tokens per sec: 168668.2085664651, Loss: 2.513890027999878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8716 , TFLOPS: 69.79283147566703, Tokens per sec: 172954.32471177753, Loss: 2.4937355518341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8717 , TFLOPS: 70.75047638405907, Tokens per sec: 175327.47428806836, Loss: 2.489528179168701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8718 , TFLOPS: 69.10679133670416, Tokens per sec: 171254.24167386748, Loss: 2.4881951808929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8719 , TFLOPS: 70.71183654109925, Tokens per sec: 175231.72050068385, Loss: 2.5038137435913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8720 , TFLOPS: 70.52340810691837, Tokens per sec: 174764.77408367806, Loss: 2.52710223197937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8721 , TFLOPS: 69.6913212675259, Tokens per sec: 172702.77123373124, Loss: 2.5107345581054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8722 , TFLOPS: 68.71230836465772, Tokens per sec: 170276.66941324965, Loss: 2.481452703475952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8723 , TFLOPS: 71.50091486919402, Tokens per sec: 177187.1435218563, Loss: 2.489393472671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8724 , TFLOPS: 68.7144790162804, Tokens per sec: 170282.04852708764, Loss: 2.5112199783325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8725 , TFLOPS: 69.66080511049937, Tokens per sec: 172627.14883498728, Loss: 2.5197091102600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8726 , TFLOPS: 70.98780095062288, Tokens per sec: 175915.5907074727, Loss: 2.4987947940826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8727 , TFLOPS: 69.61586053321118, Tokens per sec: 172515.77122141328, Loss: 2.503690719604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8728 , TFLOPS: 69.73503160488359, Tokens per sec: 172811.09026479477, Loss: 2.489936351776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8729 , TFLOPS: 69.88549259623944, Tokens per sec: 173183.94917602104, Loss: 2.5211119651794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8730 , TFLOPS: 71.49350000912222, Tokens per sec: 177168.7686817841, Loss: 2.49050235748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8731 , TFLOPS: 69.1163919902679, Tokens per sec: 171278.03315099768, Loss: 2.5195910930633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8732 , TFLOPS: 68.75165577695827, Tokens per sec: 170374.1766354057, Loss: 2.4995555877685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8733 , TFLOPS: 71.49557825946168, Tokens per sec: 177173.9188150635, Loss: 2.5081522464752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8734 , TFLOPS: 69.58258061388, Tokens per sec: 172433.29991522484, Loss: 2.5025405883789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8735 , TFLOPS: 67.04156552554335, Tokens per sec: 166136.38461040694, Loss: 2.493100643157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8736 , TFLOPS: 71.55567083162285, Tokens per sec: 177322.83482862217, Loss: 2.476377487182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8737 , TFLOPS: 70.69011527606935, Tokens per sec: 175177.89281314486, Loss: 2.494227647781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8738 , TFLOPS: 67.74294008810314, Tokens per sec: 167874.4680392164, Loss: 2.494619369506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8739 , TFLOPS: 70.69468221282399, Tokens per sec: 175189.2101855125, Loss: 2.471946954727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8740 , TFLOPS: 71.4938873698182, Tokens per sec: 177169.72860426005, Loss: 2.5040597915649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8741 , TFLOPS: 69.76033795580497, Tokens per sec: 172873.80219010264, Loss: 2.49784779548645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8742 , TFLOPS: 68.78860830536664, Tokens per sec: 170465.74907145867, Loss: 2.519033193588257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8743 , TFLOPS: 71.5352328532773, Tokens per sec: 177272.1872668527, Loss: 2.4970602989196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8744 , TFLOPS: 69.92221741506592, Tokens per sec: 173274.95732264582, Loss: 2.4855308532714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8745 , TFLOPS: 69.63582900132062, Tokens per sec: 172565.25528509682, Loss: 2.499788999557495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8746 , TFLOPS: 69.78607311254808, Tokens per sec: 172937.57674347362, Loss: 2.494320869445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8747 , TFLOPS: 70.54155977305399, Tokens per sec: 174809.7559119332, Loss: 2.5065300464630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8748 , TFLOPS: 70.68048203472766, Tokens per sec: 175154.02057991104, Loss: 2.5161819458007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8749 , TFLOPS: 67.90655441866475, Tokens per sec: 168279.9223739545, Loss: 2.51251220703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8750 , TFLOPS: 71.52980383362987, Tokens per sec: 177258.7335581105, Loss: 2.486809492111206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8751 , TFLOPS: 69.55780734887179, Tokens per sec: 172371.90903553407, Loss: 2.487866163253784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8752 , TFLOPS: 68.93942339117781, Tokens per sec: 170839.48546775195, Loss: 2.5186946392059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8753 , TFLOPS: 70.5937772201317, Tokens per sec: 174939.15649801845, Loss: 2.49570631980896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8754 , TFLOPS: 69.78990953026405, Tokens per sec: 172947.0838091901, Loss: 2.4937663078308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8755 , TFLOPS: 68.03290412967993, Tokens per sec: 168593.03087641977, Loss: 2.4936940670013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8756 , TFLOPS: 71.51457900536339, Tokens per sec: 177221.00475651288, Loss: 2.4973878860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8757 , TFLOPS: 69.62320662147238, Tokens per sec: 172533.97563736292, Loss: 2.5022623538970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8758 , TFLOPS: 68.75481280118208, Tokens per sec: 170382.0000892072, Loss: 2.4928948879241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8759 , TFLOPS: 70.76365771538941, Tokens per sec: 175360.13907914847, Loss: 2.5027921199798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8760 , TFLOPS: 70.55035406364816, Tokens per sec: 174831.54913279664, Loss: 2.4944887161254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8761 , TFLOPS: 69.86467262336222, Tokens per sec: 173132.35498972196, Loss: 2.5002593994140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8762 , TFLOPS: 67.86360862291119, Tokens per sec: 168173.4979612072, Loss: 2.495192527770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8763 , TFLOPS: 71.45375124669103, Tokens per sec: 177070.26686979196, Loss: 2.510974168777466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8764 , TFLOPS: 68.81843026583499, Tokens per sec: 170539.6511746594, Loss: 2.4842381477355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8765 , TFLOPS: 69.67028125401984, Tokens per sec: 172650.63176251441, Loss: 2.486760139465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8766 , TFLOPS: 70.74185969862621, Tokens per sec: 175306.12119235948, Loss: 2.5009660720825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8767 , TFLOPS: 71.44461898096944, Tokens per sec: 177047.63610933177, Loss: 2.481393814086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8768 , TFLOPS: 69.26557399446783, Tokens per sec: 171647.72259115952, Loss: 2.4949913024902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8769 , TFLOPS: 68.96280356643585, Tokens per sec: 170897.42411758026, Loss: 2.493335247039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8770 , TFLOPS: 71.49821954524738, Tokens per sec: 177180.4642122021, Loss: 2.493056297302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8771 , TFLOPS: 69.77175707519189, Tokens per sec: 172902.10002586327, Loss: 2.4870424270629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8772 , TFLOPS: 67.81011647577458, Tokens per sec: 168040.93852795623, Loss: 2.5129776000976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8773 , TFLOPS: 71.48709733302363, Tokens per sec: 177152.9021450988, Loss: 2.4893603324890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8774 , TFLOPS: 69.69785888861013, Tokens per sec: 172718.9721789564, Loss: 2.4979233741760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8775 , TFLOPS: 68.09273831706412, Tokens per sec: 168741.30658403767, Loss: 2.5016531944274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8776 , TFLOPS: 70.88722439822561, Tokens per sec: 175666.35093121039, Loss: 2.5025391578674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8777 , TFLOPS: 70.68886792137465, Tokens per sec: 175174.80173080397, Loss: 2.4990153312683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8778 , TFLOPS: 69.60619565230971, Tokens per sec: 172491.82058186491, Loss: 2.518651008605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8779 , TFLOPS: 69.74393471895358, Tokens per sec: 172833.15316221918, Loss: 2.507089138031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8780 , TFLOPS: 71.52396745235646, Tokens per sec: 177244.2703623845, Loss: 2.503575563430786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8781 , TFLOPS: 70.64867375458853, Tokens per sec: 175075.19615775498, Loss: 2.4871747493743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8782 , TFLOPS: 68.12281374566714, Tokens per sec: 168815.83680919794, Loss: 2.4873766899108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8783 , TFLOPS: 70.35885731142193, Tokens per sec: 174356.99908567136, Loss: 2.498818874359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8784 , TFLOPS: 70.59695349218866, Tokens per sec: 174947.0276500708, Loss: 2.5172362327575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8785 , TFLOPS: 68.98490106144735, Tokens per sec: 170952.18414445882, Loss: 2.4987263679504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8786 , TFLOPS: 69.66834035481563, Tokens per sec: 172645.82200047866, Loss: 2.5339744091033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8787 , TFLOPS: 71.44516160444165, Tokens per sec: 177048.98079007078, Loss: 2.5088095664978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8788 , TFLOPS: 69.78420261628906, Tokens per sec: 172932.94144769158, Loss: 2.520524263381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8789 , TFLOPS: 68.87917015386185, Tokens per sec: 170690.17130824094, Loss: 2.5004630088806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8790 , TFLOPS: 70.51130088471615, Tokens per sec: 174734.77105334034, Loss: 2.5011842250823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8791 , TFLOPS: 69.7772371579782, Tokens per sec: 172915.68027468873, Loss: 2.4955978393554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8792 , TFLOPS: 69.54821257339505, Tokens per sec: 172348.13212495096, Loss: 2.4933793544769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8793 , TFLOPS: 70.98734160589825, Tokens per sec: 175914.45240064408, Loss: 2.4884986877441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8794 , TFLOPS: 68.90719172593843, Tokens per sec: 170759.61185067773, Loss: 2.509119987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8795 , TFLOPS: 69.80536520753175, Tokens per sec: 172985.38468004286, Loss: 2.5141799449920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8796 , TFLOPS: 69.92777785928801, Tokens per sec: 173288.73671596168, Loss: 2.504746913909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8797 , TFLOPS: 69.595239689238, Tokens per sec: 172464.67049847424, Loss: 2.5120792388916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8798 , TFLOPS: 70.5370707506892, Tokens per sec: 174798.6316200066, Loss: 2.5102176666259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8799 , TFLOPS: 69.15979266197479, Tokens per sec: 171385.58479646608, Loss: 2.4885752201080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8800 , TFLOPS: 71.45728744067337, Tokens per sec: 177079.0299480241, Loss: 2.5076422691345215 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/model.pt +[rank0]:[2024-08-26 20:35:00,913] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006481121003162116, 'preprocessing_with_comm': 0.0012834830122301355, 'state_converting': 0.9712000569998054, : 0.9803612770047039}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800, took 4.37s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0006800 + eval ppl=9.772669792175293, eval loss=2.2795896530151367 +------------------------------------------------------------------ +iteration: 8801 , TFLOPS: 64.33336694315209, Tokens per sec: 159425.17019053953, Loss: 2.477200746536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8802 , TFLOPS: 68.9234806634439, Tokens per sec: 170799.97763219086, Loss: 2.498539686203003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8803 , TFLOPS: 70.91548373812894, Tokens per sec: 175736.3806885141, Loss: 2.4984090328216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8804 , TFLOPS: 69.82313796719299, Tokens per sec: 173029.4275363145, Loss: 2.506608724594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8805 , TFLOPS: 68.68522741771491, Tokens per sec: 170209.55984351278, Loss: 2.518346071243286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8806 , TFLOPS: 69.68211627035704, Tokens per sec: 172679.96023673267, Loss: 2.535949230194092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8807 , TFLOPS: 71.49458043583476, Tokens per sec: 177171.44609819664, Loss: 2.503706693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8808 , TFLOPS: 69.79630534662358, Tokens per sec: 172962.93334668633, Loss: 2.49336838722229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8809 , TFLOPS: 67.1568671762655, Tokens per sec: 166422.1148024218, Loss: 2.522705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8810 , TFLOPS: 71.52353056464473, Tokens per sec: 177243.18770650768, Loss: 2.486346483230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8811 , TFLOPS: 69.55349960855098, Tokens per sec: 172361.23398048233, Loss: 2.5097923278808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8812 , TFLOPS: 68.03548178416192, Tokens per sec: 168599.4185881782, Loss: 2.4834916591644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8813 , TFLOPS: 71.48097497414187, Tokens per sec: 177137.73026535634, Loss: 2.492457866668701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8814 , TFLOPS: 70.72046950174438, Tokens per sec: 175253.11392816156, Loss: 2.508819341659546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8815 , TFLOPS: 69.03792221820935, Tokens per sec: 171083.5764116776, Loss: 2.499786615371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8816 , TFLOPS: 69.66677066973352, Tokens per sec: 172641.93214793014, Loss: 2.5007693767547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8817 , TFLOPS: 71.52663265552638, Tokens per sec: 177250.87502943596, Loss: 2.5029945373535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8818 , TFLOPS: 68.90597903779043, Tokens per sec: 170756.60667585835, Loss: 2.4949848651885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8819 , TFLOPS: 67.67922308541603, Tokens per sec: 167716.57028755045, Loss: 2.5146677494049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8820 , TFLOPS: 71.50591848702815, Tokens per sec: 177199.5430380433, Loss: 2.477943181991577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8821 , TFLOPS: 69.6533643150852, Tokens per sec: 172608.7097242797, Loss: 2.4907357692718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8822 , TFLOPS: 68.97582703613621, Tokens per sec: 170929.6977101514, Loss: 2.5005955696105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8823 , TFLOPS: 69.98417035676886, Tokens per sec: 173428.4835368656, Loss: 2.507960319519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8824 , TFLOPS: 70.57933132405839, Tokens per sec: 174903.35797620195, Loss: 2.5088326930999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8825 , TFLOPS: 69.26680388600487, Tokens per sec: 171650.77039787205, Loss: 2.518373966217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8826 , TFLOPS: 69.60726527208003, Tokens per sec: 172494.4712174841, Loss: 2.5105338096618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8827 , TFLOPS: 70.48722959439262, Tokens per sec: 174675.1197442471, Loss: 2.5051748752593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8828 , TFLOPS: 69.82777084139069, Tokens per sec: 173040.90830892962, Loss: 2.514389753341675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8829 , TFLOPS: 68.3641035909531, Tokens per sec: 169413.77962608726, Loss: 2.52050518989563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8830 , TFLOPS: 71.54227671062856, Tokens per sec: 177289.64272131407, Loss: 2.5132007598876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8831 , TFLOPS: 68.63959004721771, Tokens per sec: 170096.46541204932, Loss: 2.49470853805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8832 , TFLOPS: 68.89214885616654, Tokens per sec: 170722.33396227408, Loss: 2.471205711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8833 , TFLOPS: 71.45614295563145, Tokens per sec: 177076.19378801482, Loss: 2.5027308464050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8834 , TFLOPS: 69.70277565157186, Tokens per sec: 172731.1564592017, Loss: 2.498866319656372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8835 , TFLOPS: 69.79354225332764, Tokens per sec: 172956.08609711914, Loss: 2.510310649871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8836 , TFLOPS: 69.89363022193947, Tokens per sec: 173204.11510893918, Loss: 2.4914932250976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8837 , TFLOPS: 71.52892058552128, Tokens per sec: 177256.5447719996, Loss: 2.500335693359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8838 , TFLOPS: 68.03225421710313, Tokens per sec: 168591.4203214607, Loss: 2.4940032958984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8839 , TFLOPS: 68.85465247941033, Tokens per sec: 170629.41381010532, Loss: 2.5037379264831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8840 , TFLOPS: 71.51538112061498, Tokens per sec: 177222.9924864666, Loss: 2.5135772228240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8841 , TFLOPS: 69.82402882946802, Tokens per sec: 173031.6351911686, Loss: 2.4929044246673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8842 , TFLOPS: 67.72304295248668, Tokens per sec: 167825.16074530839, Loss: 2.480191230773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8843 , TFLOPS: 70.72359668341952, Tokens per sec: 175260.8634288401, Loss: 2.5271642208099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8844 , TFLOPS: 70.55915042519138, Tokens per sec: 174853.34748569995, Loss: 2.5088815689086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8845 , TFLOPS: 69.8288030925967, Tokens per sec: 173043.46634112997, Loss: 2.503861665725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8846 , TFLOPS: 68.10789684037411, Tokens per sec: 168778.871074649, Loss: 2.4856104850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8847 , TFLOPS: 71.55036009232786, Tokens per sec: 177309.67423162295, Loss: 2.4912474155426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8848 , TFLOPS: 69.8567673407932, Tokens per sec: 173112.76482867714, Loss: 2.4926352500915527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8849 , TFLOPS: 67.98208460411642, Tokens per sec: 168467.09449384053, Loss: 2.5238683223724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8850 , TFLOPS: 71.49805795013197, Tokens per sec: 177180.06376170492, Loss: 2.4909467697143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8851 , TFLOPS: 69.76723234733387, Tokens per sec: 172890.88725179117, Loss: 2.4879226684570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8852 , TFLOPS: 69.81289882220342, Tokens per sec: 173004.05380709542, Loss: 2.5022964477539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8853 , TFLOPS: 69.63956198826502, Tokens per sec: 172574.50603796748, Loss: 2.489806890487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8854 , TFLOPS: 70.72012758257928, Tokens per sec: 175252.26661480495, Loss: 2.522047996520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8855 , TFLOPS: 69.72597927403376, Tokens per sec: 172788.65759175233, Loss: 2.4721357822418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8856 , TFLOPS: 68.90628567648791, Tokens per sec: 170757.36656032977, Loss: 2.5048444271087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8857 , TFLOPS: 71.55879867241644, Tokens per sec: 177330.58596266876, Loss: 2.5193424224853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8858 , TFLOPS: 69.80258030358893, Tokens per sec: 172978.48338129002, Loss: 2.4954018592834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8859 , TFLOPS: 68.31198091298434, Tokens per sec: 169284.61388829368, Loss: 2.5006260871887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8860 , TFLOPS: 70.46500467671, Tokens per sec: 174620.04394995276, Loss: 2.5018887519836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8861 , TFLOPS: 69.75284817554866, Tokens per sec: 172855.2416895601, Loss: 2.483060598373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8862 , TFLOPS: 68.77806056060756, Tokens per sec: 170439.61059801563, Loss: 2.4881975650787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8863 , TFLOPS: 70.6733557588634, Tokens per sec: 175136.3608832953, Loss: 2.486576795578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8864 , TFLOPS: 70.53609339385525, Tokens per sec: 174796.20962210736, Loss: 2.492332696914673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8865 , TFLOPS: 69.4600774741667, Tokens per sec: 172129.72364018043, Loss: 2.5042052268981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8866 , TFLOPS: 68.882604425683, Tokens per sec: 170698.68181213006, Loss: 2.5130341053009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8867 , TFLOPS: 71.48862660017252, Tokens per sec: 177156.69183755646, Loss: 2.483628034591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8868 , TFLOPS: 68.68601288997573, Tokens per sec: 170211.506330302, Loss: 2.4958643913269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8869 , TFLOPS: 68.06707843959514, Tokens per sec: 168677.71858100066, Loss: 2.5055503845214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8870 , TFLOPS: 71.50555254188353, Tokens per sec: 177198.63618566273, Loss: 2.4807400703430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8871 , TFLOPS: 69.56914127656582, Tokens per sec: 172399.99575689508, Loss: 2.4863929748535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8872 , TFLOPS: 69.97352846851115, Tokens per sec: 173402.11176546224, Loss: 2.5222339630126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8873 , TFLOPS: 70.37462483542362, Tokens per sec: 174396.07274708367, Loss: 2.4919986724853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8874 , TFLOPS: 71.54262837570683, Tokens per sec: 177290.51418611716, Loss: 2.5033931732177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8875 , TFLOPS: 68.96578926149435, Tokens per sec: 170904.8229988367, Loss: 2.4950640201568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8876 , TFLOPS: 68.9986365417902, Tokens per sec: 170986.22217783646, Loss: 2.5038838386535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8877 , TFLOPS: 71.46474618746261, Tokens per sec: 177097.51354421666, Loss: 2.5140786170959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8878 , TFLOPS: 69.8326018764812, Tokens per sec: 173052.88014606637, Loss: 2.4954559803009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8879 , TFLOPS: 67.93047752900469, Tokens per sec: 168339.206476725, Loss: 2.498872756958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8880 , TFLOPS: 71.51826996268989, Tokens per sec: 177230.15135536043, Loss: 2.497713565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8881 , TFLOPS: 70.5847679133819, Tokens per sec: 174916.83044343535, Loss: 2.518594264984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8882 , TFLOPS: 68.24510776176565, Tokens per sec: 169118.89485288798, Loss: 2.4917922019958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8883 , TFLOPS: 69.78577636526663, Tokens per sec: 172936.84137102077, Loss: 2.510435104370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8884 , TFLOPS: 71.511913038953, Tokens per sec: 177214.39819806747, Loss: 2.506443977355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8885 , TFLOPS: 69.1843902213339, Tokens per sec: 171446.54025820276, Loss: 2.4923980236053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8886 , TFLOPS: 68.70043867217102, Tokens per sec: 170247.255007713, Loss: 2.516167640686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8887 , TFLOPS: 71.52940114288828, Tokens per sec: 177257.7356460928, Loss: 2.5035722255706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8888 , TFLOPS: 69.64980434374436, Tokens per sec: 172599.8877231339, Loss: 2.461533546447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8889 , TFLOPS: 68.04280257544615, Tokens per sec: 168617.56031543226, Loss: 2.499197006225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8890 , TFLOPS: 70.00574177670235, Tokens per sec: 173481.9398917489, Loss: 2.4955711364746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8891 , TFLOPS: 70.71736403868132, Tokens per sec: 175245.41824859634, Loss: 2.491234540939331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8892 , TFLOPS: 69.38284616649159, Tokens per sec: 171938.3359520306, Loss: 2.4904472827911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8893 , TFLOPS: 69.44481704201324, Tokens per sec: 172091.90660822875, Loss: 2.4981560707092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8894 , TFLOPS: 71.50754664680824, Tokens per sec: 177203.57779733493, Loss: 2.5005550384521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8895 , TFLOPS: 69.69031994588197, Tokens per sec: 172700.28984839292, Loss: 2.5170106887817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8896 , TFLOPS: 68.7226930082655, Tokens per sec: 170302.4037040726, Loss: 2.4967947006225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8897 , TFLOPS: 70.48664514668955, Tokens per sec: 174673.67141845665, Loss: 2.513645887374878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8898 , TFLOPS: 69.85887107784905, Tokens per sec: 173117.97812084729, Loss: 2.481283664703369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8899 , TFLOPS: 68.2053597385716, Tokens per sec: 169020.39487280694, Loss: 2.495410442352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8900 , TFLOPS: 71.47443224513142, Tokens per sec: 177121.51666212734, Loss: 2.5333712100982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8901 , TFLOPS: 69.59864835433756, Tokens per sec: 172473.1175460864, Loss: 2.5080270767211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8902 , TFLOPS: 68.8950926917306, Tokens per sec: 170729.62911109207, Loss: 2.4956750869750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8903 , TFLOPS: 69.90708869358629, Tokens per sec: 173237.46668425304, Loss: 2.5194432735443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8904 , TFLOPS: 71.54094079907107, Tokens per sec: 177286.3321853118, Loss: 2.498015880584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8905 , TFLOPS: 68.09934165076804, Tokens per sec: 168757.6703723732, Loss: 2.474229335784912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8906 , TFLOPS: 68.86492314556648, Tokens per sec: 170654.8656522475, Loss: 2.5055792331695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8907 , TFLOPS: 71.4788536154608, Tokens per sec: 177132.47330485767, Loss: 2.5143990516662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8908 , TFLOPS: 68.66396674420561, Tokens per sec: 170156.87355250018, Loss: 2.508709192276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8909 , TFLOPS: 68.96811827792258, Tokens per sec: 170910.59455810822, Loss: 2.5099265575408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8910 , TFLOPS: 70.59749632589657, Tokens per sec: 174948.37285179712, Loss: 2.5137276649475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8911 , TFLOPS: 70.7074640691713, Tokens per sec: 175220.88503357882, Loss: 2.501922130584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8912 , TFLOPS: 68.91609433656566, Tokens per sec: 170781.67350051555, Loss: 2.502558469772339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8913 , TFLOPS: 69.8821183618607, Tokens per sec: 173175.5874515279, Loss: 2.485866069793701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8914 , TFLOPS: 71.52470168521636, Tokens per sec: 177246.08987229376, Loss: 2.5195863246917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8915 , TFLOPS: 69.71280488704214, Tokens per sec: 172756.01000950855, Loss: 2.5058088302612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8916 , TFLOPS: 67.82000344064976, Tokens per sec: 168065.439516056, Loss: 2.4819254875183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8917 , TFLOPS: 71.57450953763149, Tokens per sec: 177369.51921736632, Loss: 2.497873306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8918 , TFLOPS: 69.51912846067162, Tokens per sec: 172276.05848974278, Loss: 2.487311363220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8919 , TFLOPS: 68.94884128027184, Tokens per sec: 170862.82403439307, Loss: 2.490175247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8920 , TFLOPS: 69.75914431974941, Tokens per sec: 172870.8442284671, Loss: 2.4779884815216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8921 , TFLOPS: 70.52424251599027, Tokens per sec: 174766.84184127595, Loss: 2.50517201423645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8922 , TFLOPS: 69.03161504438891, Tokens per sec: 171067.94654015804, Loss: 2.513439893722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8923 , TFLOPS: 69.42834308619292, Tokens per sec: 172051.08233094998, Loss: 2.4905686378479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8924 , TFLOPS: 71.56084678981236, Tokens per sec: 177335.6614231946, Loss: 2.5023951530456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8925 , TFLOPS: 69.9068131154332, Tokens per sec: 173236.7837712322, Loss: 2.4851956367492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8926 , TFLOPS: 68.24211279388679, Tokens per sec: 169111.47299256126, Loss: 2.4872188568115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8927 , TFLOPS: 70.52830331581633, Tokens per sec: 174776.904950579, Loss: 2.4908838272094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8928 , TFLOPS: 69.69437301148815, Tokens per sec: 172710.33379144705, Loss: 2.508962631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8929 , TFLOPS: 68.75725434274588, Tokens per sec: 170388.05049816033, Loss: 2.5074424743652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8930 , TFLOPS: 69.64387641065952, Tokens per sec: 172585.1976519339, Loss: 2.5195634365081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8931 , TFLOPS: 71.48305142516121, Tokens per sec: 177142.87593972217, Loss: 2.5116257667541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8932 , TFLOPS: 69.1750927560907, Tokens per sec: 171423.50011512978, Loss: 2.4863691329956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8933 , TFLOPS: 69.75956134320782, Tokens per sec: 172871.87765853596, Loss: 2.497575283050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8934 , TFLOPS: 70.52182733793518, Tokens per sec: 174760.85676400163, Loss: 2.498933792114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8935 , TFLOPS: 69.75752120214287, Tokens per sec: 172866.8219642369, Loss: 2.484060287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8936 , TFLOPS: 68.66156469026468, Tokens per sec: 170150.92099822647, Loss: 2.4981436729431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8937 , TFLOPS: 71.52493273093098, Tokens per sec: 177246.6624290264, Loss: 2.4713294506073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8938 , TFLOPS: 68.91983470855644, Tokens per sec: 170790.94255434466, Loss: 2.526021957397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8939 , TFLOPS: 69.72501710243293, Tokens per sec: 172786.2732245335, Loss: 2.5029752254486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8940 , TFLOPS: 70.05170932392147, Tokens per sec: 173595.8525375012, Loss: 2.5033252239227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8941 , TFLOPS: 71.52424118264848, Tokens per sec: 177244.94869620202, Loss: 2.4976911544799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8942 , TFLOPS: 68.17426763383054, Tokens per sec: 168943.3452121805, Loss: 2.508171319961548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8943 , TFLOPS: 69.04164081219234, Tokens per sec: 171092.79149720442, Loss: 2.485409736633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8944 , TFLOPS: 71.50899048061817, Tokens per sec: 177207.15577656735, Loss: 2.4820375442504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8945 , TFLOPS: 68.92615180198636, Tokens per sec: 170806.59700774928, Loss: 2.5063915252685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8946 , TFLOPS: 68.12454243229395, Tokens per sec: 168820.1206850897, Loss: 2.5046374797821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8947 , TFLOPS: 71.52491705791856, Tokens per sec: 177246.6235895751, Loss: 2.490626573562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8948 , TFLOPS: 70.61288307556116, Tokens per sec: 174986.50291245634, Loss: 2.487379550933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8949 , TFLOPS: 68.7628469971715, Tokens per sec: 170401.9097119087, Loss: 2.4893577098846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8950 , TFLOPS: 69.84538700882572, Tokens per sec: 173084.56311241467, Loss: 2.510242223739624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8951 , TFLOPS: 71.53497149773642, Tokens per sec: 177271.5395990317, Loss: 2.4981019496917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8952 , TFLOPS: 69.08784434074897, Tokens per sec: 171207.2889307066, Loss: 2.480788230895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8953 , TFLOPS: 68.60139042599464, Tokens per sec: 170001.8025426229, Loss: 2.4891934394836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8954 , TFLOPS: 71.58677200476966, Tokens per sec: 177399.906961757, Loss: 2.492645740509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8955 , TFLOPS: 69.8443178442488, Tokens per sec: 173081.91360481488, Loss: 2.488166570663452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8956 , TFLOPS: 69.65859543153097, Tokens per sec: 172621.6730070874, Loss: 2.5097885131835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8957 , TFLOPS: 70.43266260700518, Tokens per sec: 174539.89673845112, Loss: 2.50490665435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8958 , TFLOPS: 70.55493238881523, Tokens per sec: 174842.8947269048, Loss: 2.484621047973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8959 , TFLOPS: 69.24509904543122, Tokens per sec: 171596.98341194497, Loss: 2.5048108100891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8960 , TFLOPS: 68.77972761813754, Tokens per sec: 170443.7417502165, Loss: 2.4927818775177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8961 , TFLOPS: 71.54109993930952, Tokens per sec: 177286.72655235342, Loss: 2.5035338401794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8962 , TFLOPS: 69.78306558580344, Tokens per sec: 172930.123760894, Loss: 2.5009360313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8963 , TFLOPS: 68.28529456303029, Tokens per sec: 169218.48217336342, Loss: 2.5289199352264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8964 , TFLOPS: 70.47717701542965, Tokens per sec: 174650.20834619383, Loss: 2.4911437034606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8965 , TFLOPS: 69.76596244622998, Tokens per sec: 172887.7402969646, Loss: 2.515990972518921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8966 , TFLOPS: 67.84961214062648, Tokens per sec: 168138.81313626585, Loss: 2.4902477264404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8967 , TFLOPS: 70.3279123205917, Tokens per sec: 174280.31398952188, Loss: 2.4853930473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8968 , TFLOPS: 70.54027280542601, Tokens per sec: 174806.5666643799, Loss: 2.4933977127075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8969 , TFLOPS: 69.10406347974488, Tokens per sec: 171247.48174382412, Loss: 2.4990074634552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8970 , TFLOPS: 69.03289610301726, Tokens per sec: 171071.12114455935, Loss: 2.525268077850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8971 , TFLOPS: 70.59288784252853, Tokens per sec: 174936.95252234474, Loss: 2.498753786087036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8972 , TFLOPS: 69.758646986614, Tokens per sec: 172869.6117821718, Loss: 2.5010712146759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8973 , TFLOPS: 68.94642267479432, Tokens per sec: 170856.8304635884, Loss: 2.5014760494232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8974 , TFLOPS: 71.53652737474809, Tokens per sec: 177275.39523366065, Loss: 2.498767375946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8975 , TFLOPS: 68.90024121215744, Tokens per sec: 170742.38771186612, Loss: 2.4854533672332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8976 , TFLOPS: 69.6420392342091, Tokens per sec: 172580.64492630807, Loss: 2.510054349899292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8977 , TFLOPS: 70.65175448189589, Tokens per sec: 175082.83053939271, Loss: 2.5041584968566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8978 , TFLOPS: 71.5497521378685, Tokens per sec: 177308.16765350092, Loss: 2.492318630218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8979 , TFLOPS: 68.32903536224126, Tokens per sec: 169326.87669225497, Loss: 2.487138271331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8980 , TFLOPS: 69.2149827479495, Tokens per sec: 171522.35190920124, Loss: 2.499708414077759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8981 , TFLOPS: 71.4964999418468, Tokens per sec: 177176.20284554551, Loss: 2.4797439575195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8982 , TFLOPS: 68.86861108568365, Tokens per sec: 170664.00477411802, Loss: 2.471917152404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8983 , TFLOPS: 67.92086339716569, Tokens per sec: 168315.3815989439, Loss: 2.5141263008117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8984 , TFLOPS: 71.55192177361702, Tokens per sec: 177313.54424989087, Loss: 2.49479341506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8985 , TFLOPS: 69.67155710844054, Tokens per sec: 172653.79347031587, Loss: 2.4816176891326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8986 , TFLOPS: 68.23396826694137, Tokens per sec: 169091.2899575968, Loss: 2.5003647804260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8987 , TFLOPS: 70.72731749063276, Tokens per sec: 175270.08399899525, Loss: 2.5090172290802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8988 , TFLOPS: 71.56765597182734, Tokens per sec: 177352.53532632106, Loss: 2.5062789916992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8989 , TFLOPS: 68.4414185843072, Tokens per sec: 169605.3746965683, Loss: 2.4711811542510986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8990 , TFLOPS: 68.67953322710743, Tokens per sec: 170195.44901192057, Loss: 2.509671688079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8991 , TFLOPS: 71.53989416762484, Tokens per sec: 177283.73851729205, Loss: 2.4791407585144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8992 , TFLOPS: 69.68858963247399, Tokens per sec: 172696.00194115817, Loss: 2.513235569000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8993 , TFLOPS: 68.9783650205741, Tokens per sec: 170935.98711517057, Loss: 2.493457794189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8994 , TFLOPS: 70.538353372191, Tokens per sec: 174801.8100973805, Loss: 2.5095033645629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8995 , TFLOPS: 69.92315337985137, Tokens per sec: 173277.27674648922, Loss: 2.482339382171631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8996 , TFLOPS: 69.58522840924827, Tokens per sec: 172439.86144382614, Loss: 2.4767537117004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8997 , TFLOPS: 69.57979406200899, Tokens per sec: 172426.39453272446, Loss: 2.4987306594848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8998 , TFLOPS: 71.42068355602792, Tokens per sec: 176988.32148962194, Loss: 2.491100788116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8999 , TFLOPS: 68.9162679924327, Tokens per sec: 170782.10383888378, Loss: 2.5053768157958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9000 , TFLOPS: 68.83803229853521, Tokens per sec: 170588.2271710327, Loss: 2.5102157592773438 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/model.pt +[rank0]:[2024-08-26 21:00:25,371] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006408401008229703, 'preprocessing_with_comm': 0.001217038996401243, 'state_converting': 0.9764704650006024, : 0.9854716219997499}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000, took 4.28s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007000 + eval ppl=9.97706413269043, eval loss=2.3002889156341553 +------------------------------------------------------------------ +iteration: 9001 , TFLOPS: 65.23263919112978, Tokens per sec: 161653.66588404423, Loss: 2.484429121017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9002 , TFLOPS: 69.98645864510686, Tokens per sec: 173434.15416744296, Loss: 2.482241153717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9003 , TFLOPS: 68.04938022974375, Tokens per sec: 168633.86046737016, Loss: 2.5127487182617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9004 , TFLOPS: 71.48106293248726, Tokens per sec: 177137.9482358256, Loss: 2.4928791522979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9005 , TFLOPS: 69.47824611409003, Tokens per sec: 172174.74753135134, Loss: 2.512673854827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9006 , TFLOPS: 69.26392717827025, Tokens per sec: 171643.64159921015, Loss: 2.5188536643981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9007 , TFLOPS: 69.9278602820732, Tokens per sec: 173288.94096870298, Loss: 2.4898555278778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9008 , TFLOPS: 70.44781034080194, Tokens per sec: 174577.43449146012, Loss: 2.500784158706665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9009 , TFLOPS: 68.97088200863482, Tokens per sec: 170917.4433872632, Loss: 2.4971964359283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9010 , TFLOPS: 69.70645542131894, Tokens per sec: 172740.27533399506, Loss: 2.508918285369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9011 , TFLOPS: 71.48394406100776, Tokens per sec: 177145.0879896819, Loss: 2.493927478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9012 , TFLOPS: 68.61910141987046, Tokens per sec: 170045.69233647423, Loss: 2.4981260299682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9013 , TFLOPS: 68.28284115804881, Tokens per sec: 169212.40236555442, Loss: 2.488870620727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9014 , TFLOPS: 71.44015673020607, Tokens per sec: 177036.57816038263, Loss: 2.5103936195373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9015 , TFLOPS: 70.63672732461866, Tokens per sec: 175045.59158828174, Loss: 2.518660306930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9016 , TFLOPS: 67.94631377592968, Tokens per sec: 168378.45044111172, Loss: 2.499485969543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9017 , TFLOPS: 69.61944341939945, Tokens per sec: 172524.6500080753, Loss: 2.4993550777435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9018 , TFLOPS: 71.51429481006615, Tokens per sec: 177220.3004892596, Loss: 2.507511615753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9019 , TFLOPS: 67.92171575482945, Tokens per sec: 168317.49383512966, Loss: 2.497081995010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9020 , TFLOPS: 68.11236024813451, Tokens per sec: 168789.93189076422, Loss: 2.5070109367370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9021 , TFLOPS: 71.51733238734657, Tokens per sec: 177227.8279403773, Loss: 2.5045135021209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9022 , TFLOPS: 69.85029541375184, Tokens per sec: 173096.72668052703, Loss: 2.5068092346191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9023 , TFLOPS: 67.13237354349133, Tokens per sec: 166361.416882211, Loss: 2.4951372146606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9024 , TFLOPS: 71.47588930330187, Tokens per sec: 177125.12741278188, Loss: 2.511335849761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9025 , TFLOPS: 71.5001186597814, Tokens per sec: 177185.17042722215, Loss: 2.49468994140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9026 , TFLOPS: 68.28064868479478, Tokens per sec: 169206.96917531008, Loss: 2.512918710708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9027 , TFLOPS: 68.65834937375712, Tokens per sec: 170142.95308972438, Loss: 2.4884884357452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9028 , TFLOPS: 71.54220555811806, Tokens per sec: 177289.46639755496, Loss: 2.4847543239593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9029 , TFLOPS: 69.80656534068478, Tokens per sec: 172988.35874220164, Loss: 2.4738223552703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9030 , TFLOPS: 67.94226374097032, Tokens per sec: 168368.41400833425, Loss: 2.485511541366577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9031 , TFLOPS: 71.51953620992833, Tokens per sec: 177233.28925550595, Loss: 2.505120038986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9032 , TFLOPS: 69.76828853427297, Tokens per sec: 172893.50459937484, Loss: 2.4865314960479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9033 , TFLOPS: 69.05876240827347, Tokens per sec: 171135.22069839295, Loss: 2.504835605621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9034 , TFLOPS: 69.74879088796276, Tokens per sec: 172845.1872839751, Loss: 2.4984028339385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9035 , TFLOPS: 71.50364900532888, Tokens per sec: 177193.91901238848, Loss: 2.4901747703552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9036 , TFLOPS: 68.45891284722867, Tokens per sec: 169648.72740723984, Loss: 2.4901371002197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9037 , TFLOPS: 69.60015864547852, Tokens per sec: 172476.860213906, Loss: 2.4966683387756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9038 , TFLOPS: 70.44407542090453, Tokens per sec: 174568.17894851894, Loss: 2.4957449436187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9039 , TFLOPS: 68.9588834807262, Tokens per sec: 170887.7097133587, Loss: 2.496631145477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9040 , TFLOPS: 68.46988501576722, Tokens per sec: 169675.91764956253, Loss: 2.4821927547454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9041 , TFLOPS: 71.53917838579345, Tokens per sec: 177281.96473106398, Loss: 2.500089406967163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9042 , TFLOPS: 68.93380944578853, Tokens per sec: 170825.57349265015, Loss: 2.5027964115142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9043 , TFLOPS: 68.8093695994687, Tokens per sec: 170517.19784528893, Loss: 2.4982404708862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9044 , TFLOPS: 71.00929365822424, Tokens per sec: 175968.85200452607, Loss: 2.4929182529449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9045 , TFLOPS: 70.48913791262031, Tokens per sec: 174679.8487670339, Loss: 2.501718521118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9046 , TFLOPS: 68.2248557277579, Tokens per sec: 169068.7080816711, Loss: 2.48171329498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9047 , TFLOPS: 69.59079091659257, Tokens per sec: 172453.6459497861, Loss: 2.4799716472625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9048 , TFLOPS: 71.49327369420307, Tokens per sec: 177168.2078484838, Loss: 2.4896090030670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9049 , TFLOPS: 68.82420773343867, Tokens per sec: 170553.96837582186, Loss: 2.507627010345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9050 , TFLOPS: 68.73522071067434, Tokens per sec: 170333.4487306821, Loss: 2.4988882541656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9051 , TFLOPS: 71.52177671910486, Tokens per sec: 177238.84148405786, Loss: 2.5003833770751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9052 , TFLOPS: 69.88980044826187, Tokens per sec: 173194.62450788167, Loss: 2.492671012878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9053 , TFLOPS: 68.14469524909426, Tokens per sec: 168870.06158513686, Loss: 2.498619556427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9054 , TFLOPS: 70.50600700156684, Tokens per sec: 174721.65222772697, Loss: 2.4896843433380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9055 , TFLOPS: 71.43803606793757, Tokens per sec: 177031.32292567106, Loss: 2.4976603984832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9056 , TFLOPS: 67.99584572271831, Tokens per sec: 168501.19606164834, Loss: 2.5085196495056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9057 , TFLOPS: 68.58663628775969, Tokens per sec: 169965.2401627742, Loss: 2.4936139583587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9058 , TFLOPS: 71.52478155708697, Tokens per sec: 177246.28780358774, Loss: 2.503159523010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9059 , TFLOPS: 69.12540524101523, Tokens per sec: 171300.36897924115, Loss: 2.491804361343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9060 , TFLOPS: 67.91653216982425, Tokens per sec: 168304.64834046774, Loss: 2.4768526554107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9061 , TFLOPS: 71.53822238924835, Tokens per sec: 177279.59566631351, Loss: 2.520390510559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9062 , TFLOPS: 70.5763915587878, Tokens per sec: 174896.07291401955, Loss: 2.49086332321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9063 , TFLOPS: 67.79879332831062, Tokens per sec: 168012.87852119288, Loss: 2.503955364227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9064 , TFLOPS: 69.65377157979128, Tokens per sec: 172609.7189710853, Loss: 2.4860870838165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9065 , TFLOPS: 71.5086715500179, Tokens per sec: 177206.36543140683, Loss: 2.5005407333374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9066 , TFLOPS: 69.02162870276646, Tokens per sec: 171043.19928553127, Loss: 2.485595941543579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9067 , TFLOPS: 68.149382207769, Tokens per sec: 168881.676385044, Loss: 2.4982995986938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9068 , TFLOPS: 71.49721416499155, Tokens per sec: 177177.97276917638, Loss: 2.491050958633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9069 , TFLOPS: 69.64861149115028, Tokens per sec: 172596.9317030022, Loss: 2.489917516708374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9070 , TFLOPS: 68.80919307467056, Tokens per sec: 170516.76039739334, Loss: 2.483456611633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9071 , TFLOPS: 70.42921456803099, Tokens per sec: 174531.35211804023, Loss: 2.518979787826538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9072 , TFLOPS: 70.69172854566703, Tokens per sec: 175181.89067292423, Loss: 2.477686882019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9073 , TFLOPS: 68.51950981986504, Tokens per sec: 169798.89338074037, Loss: 2.501584768295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9074 , TFLOPS: 70.56714390777087, Tokens per sec: 174873.15621608673, Loss: 2.4998247623443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9075 , TFLOPS: 70.52657225285802, Tokens per sec: 174772.61518587594, Loss: 2.4943056106567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9076 , TFLOPS: 67.95180415738699, Tokens per sec: 168392.0562111784, Loss: 2.4914400577545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9077 , TFLOPS: 69.65080210973646, Tokens per sec: 172602.36029717533, Loss: 2.4769415855407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9078 , TFLOPS: 71.51715916143023, Tokens per sec: 177227.39866747416, Loss: 2.5033726692199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9079 , TFLOPS: 68.90654188048632, Tokens per sec: 170758.00146206096, Loss: 2.476536989212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9080 , TFLOPS: 68.36682134878458, Tokens per sec: 169420.5145293794, Loss: 2.511064052581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9081 , TFLOPS: 71.54903150798424, Tokens per sec: 177306.3818532638, Loss: 2.4966564178466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9082 , TFLOPS: 69.59973176849708, Tokens per sec: 172475.8023657216, Loss: 2.4769632816314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9083 , TFLOPS: 69.0463536388957, Tokens per sec: 171104.47040093606, Loss: 2.501781940460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9084 , TFLOPS: 69.98621446964556, Tokens per sec: 173433.54907375172, Loss: 2.504007577896118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9085 , TFLOPS: 71.49128830140059, Tokens per sec: 177163.2878264102, Loss: 2.5018062591552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9086 , TFLOPS: 67.43355439702817, Tokens per sec: 167107.77621508524, Loss: 2.502910852432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9087 , TFLOPS: 68.64264484496283, Tokens per sec: 170104.03553737552, Loss: 2.51009202003479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9088 , TFLOPS: 71.505254501895, Tokens per sec: 177197.89760973933, Loss: 2.5079383850097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9089 , TFLOPS: 69.68488858947369, Tokens per sec: 172686.83034889988, Loss: 2.5023651123046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9090 , TFLOPS: 67.99411092551908, Tokens per sec: 168496.89704308496, Loss: 2.4968390464782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9091 , TFLOPS: 71.45375371555359, Tokens per sec: 177070.27298790537, Loss: 2.494126081466675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9092 , TFLOPS: 70.67624820632524, Tokens per sec: 175143.52868673517, Loss: 2.4998838901519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9093 , TFLOPS: 68.06174943103183, Tokens per sec: 168664.512710742, Loss: 2.495459794998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9094 , TFLOPS: 69.85400171627394, Tokens per sec: 173105.91130645975, Loss: 2.488818645477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9095 , TFLOPS: 71.55120248283592, Tokens per sec: 177311.76176809892, Loss: 2.5081417560577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9096 , TFLOPS: 68.83071867163552, Tokens per sec: 170570.10319791274, Loss: 2.5036468505859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9097 , TFLOPS: 68.30250486370173, Tokens per sec: 169261.13119429833, Loss: 2.4709231853485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9098 , TFLOPS: 71.5556664797456, Tokens per sec: 177322.8240441909, Loss: 2.5070455074310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9099 , TFLOPS: 69.90962266385898, Tokens per sec: 173243.74614172865, Loss: 2.4845333099365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9100 , TFLOPS: 68.8307185911607, Tokens per sec: 170570.10299848727, Loss: 2.4979026317596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9101 , TFLOPS: 69.79996549599802, Tokens per sec: 172972.00360003495, Loss: 2.4843876361846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9102 , TFLOPS: 70.63700742464145, Tokens per sec: 175046.28570699377, Loss: 2.4898838996887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9103 , TFLOPS: 69.05115661890711, Tokens per sec: 171116.3727144963, Loss: 2.538762092590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9104 , TFLOPS: 68.62270822476474, Tokens per sec: 170054.6303963246, Loss: 2.485211133956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9105 , TFLOPS: 71.51447981697314, Tokens per sec: 177220.75895675534, Loss: 2.500053882598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9106 , TFLOPS: 69.03667545119484, Tokens per sec: 171080.48678567295, Loss: 2.504934787750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9107 , TFLOPS: 68.58460902705083, Tokens per sec: 169960.21638741644, Loss: 2.5162041187286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9108 , TFLOPS: 70.52841964947471, Tokens per sec: 174777.19323819966, Loss: 2.5085530281066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9109 , TFLOPS: 69.66645144065419, Tokens per sec: 172641.14106310555, Loss: 2.495189666748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9110 , TFLOPS: 69.64308043927952, Tokens per sec: 172583.2251471708, Loss: 2.5000393390655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9111 , TFLOPS: 70.97645746580392, Tokens per sec: 175887.4803025033, Loss: 2.4764273166656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9112 , TFLOPS: 70.47562718478399, Tokens per sec: 174646.36769512595, Loss: 2.5062291622161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9113 , TFLOPS: 68.415724573041, Tokens per sec: 169541.70210622167, Loss: 2.4918570518493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9114 , TFLOPS: 69.66571079804743, Tokens per sec: 172639.30566913748, Loss: 2.5091352462768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9115 , TFLOPS: 71.52869483872803, Tokens per sec: 177255.98534657786, Loss: 2.5018820762634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9116 , TFLOPS: 68.71515054589489, Tokens per sec: 170283.71265144745, Loss: 2.4943244457244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9117 , TFLOPS: 68.39560115862409, Tokens per sec: 169491.83406851944, Loss: 2.49804949760437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9118 , TFLOPS: 71.57274928220934, Tokens per sec: 177365.15711052212, Loss: 2.5302023887634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9119 , TFLOPS: 68.79773820661356, Tokens per sec: 170488.37397249017, Loss: 2.5208327770233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9120 , TFLOPS: 68.83707965517218, Tokens per sec: 170585.86641583676, Loss: 2.4932992458343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9121 , TFLOPS: 70.51634105265731, Tokens per sec: 174747.26114471885, Loss: 2.4889273643493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9122 , TFLOPS: 71.49035051618927, Tokens per sec: 177160.96389135098, Loss: 2.503758668899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9123 , TFLOPS: 67.68417914028241, Tokens per sec: 167728.85193746386, Loss: 2.4895761013031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9124 , TFLOPS: 69.6761557121037, Tokens per sec: 172665.18931677117, Loss: 2.485720634460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9125 , TFLOPS: 71.54645171058243, Tokens per sec: 177299.988831114, Loss: 2.4899773597717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9126 , TFLOPS: 69.81397172014955, Tokens per sec: 173006.7125663954, Loss: 2.498715877532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9127 , TFLOPS: 67.7996306171244, Tokens per sec: 168014.95341510835, Loss: 2.4978506565093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9128 , TFLOPS: 71.53494876455557, Tokens per sec: 177271.4832637053, Loss: 2.4874191284179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9129 , TFLOPS: 70.66174754787518, Tokens per sec: 175107.5944577165, Loss: 2.5013558864593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9130 , TFLOPS: 69.05018150633734, Tokens per sec: 171113.95627813085, Loss: 2.4969513416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9131 , TFLOPS: 69.6719427005402, Tokens per sec: 172654.7490100153, Loss: 2.497217893600464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9132 , TFLOPS: 71.51628227308078, Tokens per sec: 177225.22564154697, Loss: 2.518425226211548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9133 , TFLOPS: 69.04032676560422, Tokens per sec: 171089.5351450049, Loss: 2.485436201095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9134 , TFLOPS: 67.9269535805804, Tokens per sec: 168330.47374433465, Loss: 2.4991679191589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9135 , TFLOPS: 71.51805157693704, Tokens per sec: 177229.6101714077, Loss: 2.499774932861328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9136 , TFLOPS: 69.67275520322099, Tokens per sec: 172656.76248116288, Loss: 2.4929986000061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9137 , TFLOPS: 68.11474290450062, Tokens per sec: 168795.8363757107, Loss: 2.489156484603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9138 , TFLOPS: 70.56908223443533, Tokens per sec: 174877.9596030856, Loss: 2.507174491882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9139 , TFLOPS: 70.68424937944611, Tokens per sec: 175163.35647513065, Loss: 2.498685121536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9140 , TFLOPS: 68.48806388489463, Tokens per sec: 169720.966889828, Loss: 2.4822347164154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9141 , TFLOPS: 69.52377864966614, Tokens per sec: 172287.5821703318, Loss: 2.5118045806884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9142 , TFLOPS: 71.52531470218142, Tokens per sec: 177247.6089958626, Loss: 2.5038654804229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9143 , TFLOPS: 68.00122999182524, Tokens per sec: 168514.5388736215, Loss: 2.4915385246276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9144 , TFLOPS: 69.61678787492136, Tokens per sec: 172518.0692763262, Loss: 2.510727882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9145 , TFLOPS: 70.4436414378562, Tokens per sec: 174567.10349071803, Loss: 2.4896981716156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9146 , TFLOPS: 69.82533292943216, Tokens per sec: 173034.86689453822, Loss: 2.512937307357788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9147 , TFLOPS: 68.21051034871516, Tokens per sec: 169033.1586521292, Loss: 2.494354724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9148 , TFLOPS: 71.50812631941294, Tokens per sec: 177205.01428990564, Loss: 2.4820480346679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9149 , TFLOPS: 69.67267578313573, Tokens per sec: 172656.56566944273, Loss: 2.4981391429901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9150 , TFLOPS: 69.6029565008403, Tokens per sec: 172483.79360770172, Loss: 2.501858711242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9151 , TFLOPS: 69.86114860752548, Tokens per sec: 173123.6220902761, Loss: 2.513672351837158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9152 , TFLOPS: 71.50625896866251, Tokens per sec: 177200.38678904038, Loss: 2.5121617317199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9153 , TFLOPS: 68.15342469681954, Tokens per sec: 168891.69411822822, Loss: 2.470860719680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9154 , TFLOPS: 68.74792039803813, Tokens per sec: 170364.91995495564, Loss: 2.5250298976898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9155 , TFLOPS: 71.53013267494836, Tokens per sec: 177259.54846312257, Loss: 2.4888739585876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9156 , TFLOPS: 69.71882243316112, Tokens per sec: 172770.92215167882, Loss: 2.4763777256011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9157 , TFLOPS: 68.06341837149452, Tokens per sec: 168668.64852905754, Loss: 2.4943268299102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9158 , TFLOPS: 71.45753383938363, Tokens per sec: 177079.64055117124, Loss: 2.5100107192993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9159 , TFLOPS: 70.63280673318714, Tokens per sec: 175035.87593082563, Loss: 2.4956135749816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9160 , TFLOPS: 68.14494159326588, Tokens per sec: 168870.67205313122, Loss: 2.501225233078003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9161 , TFLOPS: 69.8922803611276, Tokens per sec: 173200.7700051479, Loss: 2.509230852127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9162 , TFLOPS: 71.51362346317737, Tokens per sec: 177218.63681771423, Loss: 2.4899022579193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9163 , TFLOPS: 69.03570519445802, Tokens per sec: 171078.0823825965, Loss: 2.506479263305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9164 , TFLOPS: 68.70402080440643, Tokens per sec: 170256.1319259968, Loss: 2.5190675258636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9165 , TFLOPS: 71.51940547313356, Tokens per sec: 177232.96527532666, Loss: 2.507882595062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9166 , TFLOPS: 69.65041830019054, Tokens per sec: 172601.40917484043, Loss: 2.499022960662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9167 , TFLOPS: 68.20209841843413, Tokens per sec: 169012.3129622422, Loss: 2.5030245780944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9168 , TFLOPS: 70.77387506871297, Tokens per sec: 175385.45880621864, Loss: 2.5111136436462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9169 , TFLOPS: 70.69513926940765, Tokens per sec: 175190.34282207573, Loss: 2.4939303398132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9170 , TFLOPS: 69.75555705814115, Tokens per sec: 172861.95459903235, Loss: 2.501734972000122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9171 , TFLOPS: 68.69303219120859, Tokens per sec: 170228.90093199688, Loss: 2.488537311553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9172 , TFLOPS: 71.50976782172161, Tokens per sec: 177209.08211345287, Loss: 2.5119383335113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9173 , TFLOPS: 69.01050791533974, Tokens per sec: 171015.64074923197, Loss: 2.510939598083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9174 , TFLOPS: 68.94468808234676, Tokens per sec: 170852.5319524222, Loss: 2.492479085922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9175 , TFLOPS: 70.51283940000393, Tokens per sec: 174738.5836637049, Loss: 2.5007448196411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9176 , TFLOPS: 69.81435451729457, Tokens per sec: 173007.6611798893, Loss: 2.4833757877349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9177 , TFLOPS: 68.80871695650508, Tokens per sec: 170515.58052413052, Loss: 2.4906764030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9178 , TFLOPS: 69.7904548815241, Tokens per sec: 172948.4352496879, Loss: 2.51775860786438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9179 , TFLOPS: 71.44623226935177, Tokens per sec: 177051.63401566155, Loss: 2.504866123199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9180 , TFLOPS: 68.42865669400106, Tokens per sec: 169573.74932655142, Loss: 2.4866158962249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9181 , TFLOPS: 69.6773076656228, Tokens per sec: 172668.0439844897, Loss: 2.5106019973754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9182 , TFLOPS: 70.48158214354213, Tokens per sec: 174661.1247389215, Loss: 2.4922244548797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9183 , TFLOPS: 69.92001335648894, Tokens per sec: 173269.49542269498, Loss: 2.5091171264648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9184 , TFLOPS: 68.25116857470296, Tokens per sec: 169133.91421500032, Loss: 2.485560417175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9185 , TFLOPS: 71.5256524117097, Tokens per sec: 177248.44587727546, Loss: 2.522642135620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9186 , TFLOPS: 68.64603519740145, Tokens per sec: 170112.43720419653, Loss: 2.5160810947418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9187 , TFLOPS: 69.13030053690913, Tokens per sec: 171312.50006172757, Loss: 2.4991071224212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9188 , TFLOPS: 70.51591234937176, Tokens per sec: 174746.19877075168, Loss: 2.5083911418914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9189 , TFLOPS: 71.53903126989933, Tokens per sec: 177281.60016167222, Loss: 2.500685691833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9190 , TFLOPS: 68.29866785424855, Tokens per sec: 169251.6226621919, Loss: 2.5132105350494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9191 , TFLOPS: 69.65914927257522, Tokens per sec: 172623.04548620566, Loss: 2.4894354343414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9192 , TFLOPS: 71.55447402677552, Tokens per sec: 177319.86901437154, Loss: 2.4960267543792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9193 , TFLOPS: 68.86930043100142, Tokens per sec: 170665.71304775312, Loss: 2.4936821460723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9194 , TFLOPS: 68.81577854330698, Tokens per sec: 170533.0799141236, Loss: 2.504878282546997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9195 , TFLOPS: 71.50797063876736, Tokens per sec: 177204.6284961162, Loss: 2.5044217109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9196 , TFLOPS: 70.72706793869536, Tokens per sec: 175269.46558180396, Loss: 2.5037381649017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9197 , TFLOPS: 69.16457199729574, Tokens per sec: 171397.4285158786, Loss: 2.509181022644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9198 , TFLOPS: 69.84973507336288, Tokens per sec: 173095.3380953166, Loss: 2.483238697052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9199 , TFLOPS: 71.48204124062495, Tokens per sec: 177140.37259116035, Loss: 2.5052149295806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9200 , TFLOPS: 68.99229475548202, Tokens per sec: 170970.50653275935, Loss: 2.507885694503784 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/model.pt +[rank0]:[2024-08-26 21:25:51,018] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006432191992644221, 'preprocessing_with_comm': 0.001352694001980126, 'state_converting': 0.9928249100048561, : 1.0019837989966618}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200, took 4.56s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007200 + eval ppl=10.213644981384277, eval loss=2.3237245082855225 +------------------------------------------------------------------ +iteration: 9201 , TFLOPS: 64.83336122677848, Tokens per sec: 160664.21110428296, Loss: 2.494993209838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9202 , TFLOPS: 71.51596561522274, Tokens per sec: 177224.44092849203, Loss: 2.4967551231384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9203 , TFLOPS: 69.87431054428095, Tokens per sec: 173156.23881946318, Loss: 2.4671244621276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9204 , TFLOPS: 66.86301199720262, Tokens per sec: 165693.90929788374, Loss: 2.490276336669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9205 , TFLOPS: 71.5213380901398, Tokens per sec: 177237.75451316347, Loss: 2.5067460536956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9206 , TFLOPS: 70.57136504090778, Tokens per sec: 174883.6166489962, Loss: 2.4988815784454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9207 , TFLOPS: 68.95204528417084, Tokens per sec: 170870.76390900495, Loss: 2.471083164215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9208 , TFLOPS: 68.89796749845496, Tokens per sec: 170736.75319884115, Loss: 2.479619026184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9209 , TFLOPS: 71.5448931297, Tokens per sec: 177296.12649599544, Loss: 2.50405216217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9210 , TFLOPS: 68.89429157348997, Tokens per sec: 170727.64385184133, Loss: 2.5155529975891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9211 , TFLOPS: 68.44930735503488, Tokens per sec: 169624.92393945134, Loss: 2.5005054473876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9212 , TFLOPS: 71.48479632594201, Tokens per sec: 177147.19999607373, Loss: 2.514230489730835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9213 , TFLOPS: 69.64917860571475, Tokens per sec: 172598.3370753658, Loss: 2.505915641784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9214 , TFLOPS: 68.20288799155645, Tokens per sec: 169014.26961141799, Loss: 2.4980783462524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9215 , TFLOPS: 69.74476617454684, Tokens per sec: 172835.21360077165, Loss: 2.473482131958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9216 , TFLOPS: 70.6528416175926, Tokens per sec: 175085.52458140705, Loss: 2.5026421546936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9217 , TFLOPS: 68.02437784578682, Tokens per sec: 168571.90180568493, Loss: 2.486133098602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9218 , TFLOPS: 71.47814633083907, Tokens per sec: 177130.72057565118, Loss: 2.503634214401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9219 , TFLOPS: 70.43038300273345, Tokens per sec: 174534.2476279466, Loss: 2.508596897125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9220 , TFLOPS: 70.57479224202027, Tokens per sec: 174892.1096308903, Loss: 2.497549057006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9221 , TFLOPS: 68.3141993924893, Tokens per sec: 169290.11152489268, Loss: 2.4913253784179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9222 , TFLOPS: 71.4844591952305, Tokens per sec: 177146.36454903256, Loss: 2.5124051570892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9223 , TFLOPS: 68.47405271746455, Tokens per sec: 169686.24567347873, Loss: 2.4829776287078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9224 , TFLOPS: 68.74911460233173, Tokens per sec: 170367.87932474763, Loss: 2.4881503582000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9225 , TFLOPS: 71.50660444649542, Tokens per sec: 177201.24292116784, Loss: 2.498295783996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9226 , TFLOPS: 69.0495924229551, Tokens per sec: 171112.4964646168, Loss: 2.4953393936157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9227 , TFLOPS: 69.2192053089837, Tokens per sec: 171532.8158805975, Loss: 2.507502555847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9228 , TFLOPS: 69.73076131631315, Tokens per sec: 172800.5080193061, Loss: 2.5014636516571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9229 , TFLOPS: 71.47107801739571, Tokens per sec: 177113.20451629953, Loss: 2.4724862575531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9230 , TFLOPS: 67.96936572629957, Tokens per sec: 168435.5757135103, Loss: 2.487065315246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9231 , TFLOPS: 68.96279727835817, Tokens per sec: 170897.4085350311, Loss: 2.5117647647857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9232 , TFLOPS: 71.52797211071801, Tokens per sec: 177254.19434695397, Loss: 2.514620065689087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9233 , TFLOPS: 69.77405624813107, Tokens per sec: 172907.7976296814, Loss: 2.4866580963134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9234 , TFLOPS: 67.12040332001875, Tokens per sec: 166331.75334981698, Loss: 2.4948863983154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9235 , TFLOPS: 71.45019029344937, Tokens per sec: 177061.44243538775, Loss: 2.5068464279174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9236 , TFLOPS: 70.60099269342327, Tokens per sec: 174957.03723568493, Loss: 2.4866385459899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9237 , TFLOPS: 69.00151820681923, Tokens per sec: 170993.36326122002, Loss: 2.4995477199554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9238 , TFLOPS: 70.25385554242418, Tokens per sec: 174096.79313519518, Loss: 2.5034048557281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9239 , TFLOPS: 71.5495392457008, Tokens per sec: 177307.64008325737, Loss: 2.4827308654785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9240 , TFLOPS: 70.07390880260027, Tokens per sec: 173650.86529114176, Loss: 2.516927719116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9241 , TFLOPS: 67.76301626509576, Tokens per sec: 167924.21901737756, Loss: 2.494332790374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9242 , TFLOPS: 71.54751668246132, Tokens per sec: 177302.6279487462, Loss: 2.4890294075012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9243 , TFLOPS: 70.61063893356821, Tokens per sec: 174980.9416813852, Loss: 2.491746187210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9244 , TFLOPS: 68.40563355819768, Tokens per sec: 169516.69545982286, Loss: 2.5059733390808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9245 , TFLOPS: 70.52936447248317, Tokens per sec: 174779.53461369543, Loss: 2.4808425903320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9246 , TFLOPS: 71.50259176062995, Tokens per sec: 177191.29904355938, Loss: 2.478699207305908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9247 , TFLOPS: 69.74376410667927, Tokens per sec: 172832.73036620894, Loss: 2.5024189949035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9248 , TFLOPS: 68.08976160265266, Tokens per sec: 168733.92995781387, Loss: 2.501706123352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9249 , TFLOPS: 71.51232117357411, Tokens per sec: 177215.4096006183, Loss: 2.5025618076324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9250 , TFLOPS: 69.85363782019198, Tokens per sec: 173105.00953188215, Loss: 2.4860644340515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9251 , TFLOPS: 68.71931653136605, Tokens per sec: 170294.03642236616, Loss: 2.5079188346862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9252 , TFLOPS: 70.47571977865582, Tokens per sec: 174646.5971529404, Loss: 2.4973433017730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9253 , TFLOPS: 69.75083749311376, Tokens per sec: 172850.25899699147, Loss: 2.519533395767212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9254 , TFLOPS: 69.01740184413113, Tokens per sec: 171032.72466420598, Loss: 2.4896278381347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9255 , TFLOPS: 70.78953166650902, Tokens per sec: 175424.25758027387, Loss: 2.4980008602142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9256 , TFLOPS: 69.67825915817838, Tokens per sec: 172670.40188785797, Loss: 2.501434087753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9257 , TFLOPS: 69.74730556386197, Tokens per sec: 172841.5064872314, Loss: 2.482739210128784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9258 , TFLOPS: 69.98718890578205, Tokens per sec: 173435.96383384094, Loss: 2.4965322017669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9259 , TFLOPS: 71.5144119750734, Tokens per sec: 177220.59083705436, Loss: 2.4987378120422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9260 , TFLOPS: 69.58217860725874, Tokens per sec: 172432.3036985327, Loss: 2.4944639205932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9261 , TFLOPS: 67.96789865353685, Tokens per sec: 168431.94014559285, Loss: 2.4868268966674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9262 , TFLOPS: 71.59590138954447, Tokens per sec: 177422.53058291372, Loss: 2.4909327030181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9263 , TFLOPS: 69.47723365569355, Tokens per sec: 172172.23854791335, Loss: 2.4716460704803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9264 , TFLOPS: 67.73410512902187, Tokens per sec: 167852.57403736192, Loss: 2.499117136001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9265 , TFLOPS: 71.50320149214924, Tokens per sec: 177192.81002543168, Loss: 2.4963152408599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9266 , TFLOPS: 71.46778526831876, Tokens per sec: 177105.0447213606, Loss: 2.5124351978302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9267 , TFLOPS: 69.62102968575802, Tokens per sec: 172528.58095085347, Loss: 2.5190513134002686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9268 , TFLOPS: 67.9120575585175, Tokens per sec: 168293.55976073048, Loss: 2.5485050678253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9269 , TFLOPS: 71.54359661370187, Tokens per sec: 177292.91358652906, Loss: 2.497990608215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9270 , TFLOPS: 69.80039204437662, Tokens per sec: 172973.06063390532, Loss: 2.4976813793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9271 , TFLOPS: 67.83795304669235, Tokens per sec: 168109.9206171419, Loss: 2.5167665481567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9272 , TFLOPS: 71.54862727626619, Tokens per sec: 177305.38012253665, Loss: 2.4930906295776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9273 , TFLOPS: 70.64963668865435, Tokens per sec: 175077.58241444602, Loss: 2.4716103076934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9274 , TFLOPS: 69.34229165748668, Tokens per sec: 171837.8374112685, Loss: 2.4909353256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9275 , TFLOPS: 69.82310800300476, Tokens per sec: 173029.35328175523, Loss: 2.4961681365966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9276 , TFLOPS: 71.4827078154269, Tokens per sec: 177142.02443694885, Loss: 2.5083062648773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9277 , TFLOPS: 69.97985545153888, Tokens per sec: 173417.79072637967, Loss: 2.487753391265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9278 , TFLOPS: 68.1046964697895, Tokens per sec: 168770.9402037904, Loss: 2.50986647605896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9279 , TFLOPS: 71.54434780102427, Tokens per sec: 177294.77511146409, Loss: 2.5000336170196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9280 , TFLOPS: 70.5971218465405, Tokens per sec: 174947.44485070044, Loss: 2.5246033668518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9281 , TFLOPS: 68.07489135227755, Tokens per sec: 168697.07983929105, Loss: 2.519541025161743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9282 , TFLOPS: 70.61217723826145, Tokens per sec: 174984.7537698733, Loss: 2.488921642303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9283 , TFLOPS: 70.63856844217301, Tokens per sec: 175050.15408039725, Loss: 2.5165252685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9284 , TFLOPS: 68.10382352553275, Tokens per sec: 168768.77695175944, Loss: 2.5081734657287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9285 , TFLOPS: 70.41779581640616, Tokens per sec: 174503.05519363456, Loss: 2.5009565353393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9286 , TFLOPS: 71.46856011225772, Tokens per sec: 177106.96486999744, Loss: 2.504042625427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9287 , TFLOPS: 70.61459255279352, Tokens per sec: 174990.73918535264, Loss: 2.515383720397949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9288 , TFLOPS: 68.00997475337061, Tokens per sec: 168536.20935604596, Loss: 2.4955599308013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9289 , TFLOPS: 70.56695016933806, Tokens per sec: 174872.67611090819, Loss: 2.495133876800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9290 , TFLOPS: 69.7715200595539, Tokens per sec: 172901.5126750029, Loss: 2.5008275508880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9291 , TFLOPS: 68.76149786174054, Tokens per sec: 170398.56640569126, Loss: 2.489412307739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9292 , TFLOPS: 71.51043982064063, Tokens per sec: 177210.74740080102, Loss: 2.485264301300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9293 , TFLOPS: 70.42286058153901, Tokens per sec: 174515.6062395637, Loss: 2.4870970249176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9294 , TFLOPS: 68.39857755537813, Tokens per sec: 169499.20990755313, Loss: 2.502760410308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9295 , TFLOPS: 69.52141239224974, Tokens per sec: 172281.7183237892, Loss: 2.5042502880096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9296 , TFLOPS: 71.53797411912954, Tokens per sec: 177278.98042560995, Loss: 2.485781192779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9297 , TFLOPS: 68.97831113483194, Tokens per sec: 170935.85358036548, Loss: 2.507547378540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9298 , TFLOPS: 68.89219712100012, Tokens per sec: 170722.4535678485, Loss: 2.5032479763031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9299 , TFLOPS: 71.54559255101309, Tokens per sec: 177297.8597390569, Loss: 2.5043869018554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9300 , TFLOPS: 70.38171387665334, Tokens per sec: 174413.64017217327, Loss: 2.490816116333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9301 , TFLOPS: 67.93341586446981, Tokens per sec: 168346.48799569195, Loss: 2.4980547428131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9302 , TFLOPS: 71.46867657754399, Tokens per sec: 177107.2534838065, Loss: 2.487938165664673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9303 , TFLOPS: 70.7029806378354, Tokens per sec: 175209.77459683776, Loss: 2.4958016872406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9304 , TFLOPS: 68.74495106425329, Tokens per sec: 170357.56161873764, Loss: 2.5015978813171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9305 , TFLOPS: 70.78115993072015, Tokens per sec: 175403.51149676624, Loss: 2.4982032775878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9306 , TFLOPS: 71.5057313708356, Tokens per sec: 177199.0793435056, Loss: 2.5196526050567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9307 , TFLOPS: 70.59660706170308, Tokens per sec: 174946.1691571651, Loss: 2.4778103828430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9308 , TFLOPS: 67.1226332241225, Tokens per sec: 166337.2792978303, Loss: 2.4923553466796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9309 , TFLOPS: 71.4566197871149, Tokens per sec: 177077.37542895816, Loss: 2.4964704513549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9310 , TFLOPS: 70.72802598121284, Tokens per sec: 175271.83971669938, Loss: 2.517582893371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9311 , TFLOPS: 68.00162980821887, Tokens per sec: 168515.52966268832, Loss: 2.4863052368164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9312 , TFLOPS: 70.51953002915809, Tokens per sec: 174755.16377978257, Loss: 2.533888578414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9313 , TFLOPS: 71.52424331821965, Tokens per sec: 177244.95398838262, Loss: 2.5072145462036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9314 , TFLOPS: 69.0030431817222, Tokens per sec: 170997.14231701943, Loss: 2.4991493225097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9315 , TFLOPS: 68.20409184715773, Tokens per sec: 169017.2528982136, Loss: 2.4967942237854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9316 , TFLOPS: 71.52350234089145, Tokens per sec: 177243.11776493798, Loss: 2.483548641204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9317 , TFLOPS: 69.83245614161382, Tokens per sec: 173052.51899901096, Loss: 2.5256552696228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9318 , TFLOPS: 68.8614694722927, Tokens per sec: 170646.30706942157, Loss: 2.49780535697937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9319 , TFLOPS: 70.33933758946712, Tokens per sec: 174308.62706439247, Loss: 2.5026419162750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9320 , TFLOPS: 70.58430758708694, Tokens per sec: 174915.68970416734, Loss: 2.4787917137145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9321 , TFLOPS: 69.08870274475012, Tokens per sec: 171209.41615038246, Loss: 2.497476577758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9322 , TFLOPS: 68.93389887629675, Tokens per sec: 170825.79511130112, Loss: 2.479912281036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9323 , TFLOPS: 70.70490225196066, Tokens per sec: 175214.5365683234, Loss: 2.492196798324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9324 , TFLOPS: 69.71663488716975, Tokens per sec: 172765.5011717341, Loss: 2.509227991104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9325 , TFLOPS: 70.02262428452572, Tokens per sec: 173523.77660589697, Loss: 2.520740032196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9326 , TFLOPS: 70.2975200185314, Tokens per sec: 174204.998517028, Loss: 2.4944403171539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9327 , TFLOPS: 70.6247663652132, Tokens per sec: 175015.95101326655, Loss: 2.475414752960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9328 , TFLOPS: 68.02070745920994, Tokens per sec: 168562.806183421, Loss: 2.4873383045196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9329 , TFLOPS: 71.44446484934647, Tokens per sec: 177047.25415419106, Loss: 2.51694393157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9330 , TFLOPS: 69.83403230702686, Tokens per sec: 173056.42491053342, Loss: 2.5187408924102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9331 , TFLOPS: 67.74161149555037, Tokens per sec: 167871.17564051412, Loss: 2.5041112899780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9332 , TFLOPS: 71.52013643352636, Tokens per sec: 177234.77667570196, Loss: 2.5099658966064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9333 , TFLOPS: 71.56711801606122, Tokens per sec: 177351.2022126719, Loss: 2.497828483581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9334 , TFLOPS: 68.73513078659053, Tokens per sec: 170333.22588889644, Loss: 2.48319411277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9335 , TFLOPS: 68.77724369390613, Tokens per sec: 170437.5863123442, Loss: 2.5011234283447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9336 , TFLOPS: 71.5195286063673, Tokens per sec: 177233.27041304414, Loss: 2.4911327362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9337 , TFLOPS: 68.90525997374316, Tokens per sec: 170754.82475593776, Loss: 2.4989492893218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9338 , TFLOPS: 68.97100053009206, Tokens per sec: 170917.73709649045, Loss: 2.5122578144073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9339 , TFLOPS: 71.55346864024175, Tokens per sec: 177317.37755578835, Loss: 2.508690357208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9340 , TFLOPS: 70.61529736466844, Tokens per sec: 174992.4857868201, Loss: 2.5071699619293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9341 , TFLOPS: 69.52643662605911, Tokens per sec: 172294.16892863385, Loss: 2.4953932762145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9342 , TFLOPS: 69.88974155855037, Tokens per sec: 173194.47857268935, Loss: 2.484570026397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9343 , TFLOPS: 71.51830803653208, Tokens per sec: 177230.24570653556, Loss: 2.4778170585632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9344 , TFLOPS: 69.32033745170442, Tokens per sec: 171783.43247088464, Loss: 2.505687713623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9345 , TFLOPS: 69.40808800757503, Tokens per sec: 172000.88801485373, Loss: 2.4922895431518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9346 , TFLOPS: 71.4749627680026, Tokens per sec: 177122.83135624405, Loss: 2.496222734451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9347 , TFLOPS: 70.57447689546916, Tokens per sec: 174891.3281673984, Loss: 2.4958479404449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9348 , TFLOPS: 67.85154480818706, Tokens per sec: 168143.6024993819, Loss: 2.4930081367492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9349 , TFLOPS: 70.48026917836283, Tokens per sec: 174657.87106657217, Loss: 2.4873886108398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9350 , TFLOPS: 70.47561793484347, Tokens per sec: 174646.34477275438, Loss: 2.499086856842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9351 , TFLOPS: 68.88620333957523, Tokens per sec: 170707.6003172081, Loss: 2.509356737136841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9352 , TFLOPS: 69.69548680392978, Tokens per sec: 172713.0938918118, Loss: 2.5112152099609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9353 , TFLOPS: 71.47490088726121, Tokens per sec: 177122.67800895014, Loss: 2.482295513153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9354 , TFLOPS: 69.78948761770079, Tokens per sec: 172946.0382633808, Loss: 2.494915723800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9355 , TFLOPS: 68.16150577835415, Tokens per sec: 168911.71992847556, Loss: 2.499727249145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9356 , TFLOPS: 70.47984158898826, Tokens per sec: 174656.81145299913, Loss: 2.4908287525177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9357 , TFLOPS: 69.77428852689779, Tokens per sec: 172908.3732420529, Loss: 2.499973773956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9358 , TFLOPS: 68.80558551695482, Tokens per sec: 170507.8204719684, Loss: 2.527592897415161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9359 , TFLOPS: 70.57081188551027, Tokens per sec: 174882.24586898607, Loss: 2.500601291656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9360 , TFLOPS: 71.52026895065757, Tokens per sec: 177235.10506775105, Loss: 2.5074267387390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9361 , TFLOPS: 68.68936794160142, Tokens per sec: 170219.8205178199, Loss: 2.4761369228363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9362 , TFLOPS: 69.63532627176839, Tokens per sec: 172564.00946588616, Loss: 2.511258125305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9363 , TFLOPS: 70.42734222642366, Tokens per sec: 174526.71224928185, Loss: 2.5003089904785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9364 , TFLOPS: 69.83073127987977, Tokens per sec: 173048.24460162478, Loss: 2.495897054672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9365 , TFLOPS: 68.93506661889279, Tokens per sec: 170828.6889060916, Loss: 2.4922804832458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9366 , TFLOPS: 71.5149953622921, Tokens per sec: 177222.0365348477, Loss: 2.486879825592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9367 , TFLOPS: 69.62293525477537, Tokens per sec: 172533.30316079422, Loss: 2.5155515670776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9368 , TFLOPS: 67.83735353004454, Tokens per sec: 168108.43494884638, Loss: 2.4713099002838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9369 , TFLOPS: 71.50691757370805, Tokens per sec: 177202.01888489455, Loss: 2.5060923099517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9370 , TFLOPS: 70.73870660198556, Tokens per sec: 175298.30747154166, Loss: 2.493985652923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9371 , TFLOPS: 68.85423631884547, Tokens per sec: 170628.3825183817, Loss: 2.495832920074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9372 , TFLOPS: 69.70392676918368, Tokens per sec: 172734.0090554499, Loss: 2.4800212383270264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9373 , TFLOPS: 71.5122338337849, Tokens per sec: 177215.1931629994, Loss: 2.497628688812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9374 , TFLOPS: 68.84475030184376, Tokens per sec: 170604.87512328042, Loss: 2.495389699935913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9375 , TFLOPS: 68.0730805378141, Tokens per sec: 168692.59244157298, Loss: 2.5018787384033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9376 , TFLOPS: 71.48008215179627, Tokens per sec: 177135.5177532314, Loss: 2.4866433143615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9377 , TFLOPS: 70.72245626008981, Tokens per sec: 175258.0373342011, Loss: 2.4984805583953857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9378 , TFLOPS: 67.9390183606733, Tokens per sec: 168360.3715984501, Loss: 2.5087952613830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9379 , TFLOPS: 71.49038869558899, Tokens per sec: 177161.0585041093, Loss: 2.4892568588256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9380 , TFLOPS: 71.45611685844001, Tokens per sec: 177076.12911629962, Loss: 2.5012264251708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9381 , TFLOPS: 69.70634883180738, Tokens per sec: 172740.0111934436, Loss: 2.5321242809295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9382 , TFLOPS: 68.21799147664645, Tokens per sec: 169051.6977112572, Loss: 2.494969606399536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9383 , TFLOPS: 71.51598922506548, Tokens per sec: 177224.4994362832, Loss: 2.508232831954956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9384 , TFLOPS: 69.82382354152037, Tokens per sec: 173031.12646501997, Loss: 2.487253189086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9385 , TFLOPS: 68.74028345056713, Tokens per sec: 170345.9947578433, Loss: 2.473081111907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9386 , TFLOPS: 70.61324438607514, Tokens per sec: 174987.39827971163, Loss: 2.497695207595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9387 , TFLOPS: 70.62771155615805, Tokens per sec: 175023.24952086742, Loss: 2.4884417057037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9388 , TFLOPS: 69.11867574917027, Tokens per sec: 171283.69255713458, Loss: 2.5153095722198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9389 , TFLOPS: 69.70541506444079, Tokens per sec: 172737.69721504624, Loss: 2.4891269207000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9390 , TFLOPS: 71.45839385598002, Tokens per sec: 177081.77176703574, Loss: 2.5201423168182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9391 , TFLOPS: 69.18743482666973, Tokens per sec: 171454.08512561594, Loss: 2.4764435291290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9392 , TFLOPS: 69.50714968933818, Tokens per sec: 172246.3737748079, Loss: 2.511718511581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9393 , TFLOPS: 70.4537148376713, Tokens per sec: 174592.0664851364, Loss: 2.5024516582489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9394 , TFLOPS: 69.70768864759103, Tokens per sec: 172743.3314045484, Loss: 2.481475353240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9395 , TFLOPS: 68.11563744170141, Tokens per sec: 168798.05313743567, Loss: 2.499242067337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9396 , TFLOPS: 70.42378590430152, Tokens per sec: 174517.89929130193, Loss: 2.492081880569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9397 , TFLOPS: 70.65177570503332, Tokens per sec: 175082.88313266516, Loss: 2.5155365467071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9398 , TFLOPS: 67.9000168937699, Tokens per sec: 168263.7216670972, Loss: 2.51332426071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9399 , TFLOPS: 71.52804127307334, Tokens per sec: 177254.36573888935, Loss: 2.517798900604248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9400 , TFLOPS: 70.55285970548896, Tokens per sec: 174837.75838929738, Loss: 2.505750894546509 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/optimizer.pt +[rank0]:[2024-08-26 21:51:13,651] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006449788008467294, 'preprocessing_with_comm': 0.001251175010111183, 'state_converting': 0.9942408870119834, : 1.0033176119904965}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400, took 4.49s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007400 + eval ppl=11.048565864562988, eval loss=2.4023005962371826 +------------------------------------------------------------------ +iteration: 9401 , TFLOPS: 63.408935305363116, Tokens per sec: 157134.3267575469, Loss: 2.4830117225646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9402 , TFLOPS: 69.59390798907621, Tokens per sec: 172461.37039877445, Loss: 2.5034453868865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9403 , TFLOPS: 71.49900373923289, Tokens per sec: 177182.40753128426, Loss: 2.498152256011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9404 , TFLOPS: 68.46187631651088, Tokens per sec: 169656.07121057424, Loss: 2.4891107082366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9405 , TFLOPS: 68.78467154568686, Tokens per sec: 170455.99334730228, Loss: 2.498382329940796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9406 , TFLOPS: 71.40429876215076, Tokens per sec: 176947.7181654603, Loss: 2.4914205074310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9407 , TFLOPS: 69.51685059394886, Tokens per sec: 172270.41368507606, Loss: 2.5013656616210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9408 , TFLOPS: 68.05561278975382, Tokens per sec: 168649.3054376483, Loss: 2.5126686096191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9409 , TFLOPS: 71.49786202098602, Tokens per sec: 177179.57822769787, Loss: 2.472055673599243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9410 , TFLOPS: 70.51416410700898, Tokens per sec: 174741.86643359193, Loss: 2.493537187576294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9411 , TFLOPS: 68.73850781814687, Tokens per sec: 170341.5945451038, Loss: 2.499001979827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9412 , TFLOPS: 70.69115023493255, Tokens per sec: 175180.457555218, Loss: 2.5045604705810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9413 , TFLOPS: 71.46873965346663, Tokens per sec: 177107.4097928912, Loss: 2.5045297145843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9414 , TFLOPS: 69.66953845810133, Tokens per sec: 172648.79103240263, Loss: 2.4890403747558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9415 , TFLOPS: 67.24203148274849, Tokens per sec: 166633.1613355095, Loss: 2.5069031715393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9416 , TFLOPS: 71.53041745038388, Tokens per sec: 177260.2541680224, Loss: 2.5282421112060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9417 , TFLOPS: 69.7583967967971, Tokens per sec: 172868.9917842448, Loss: 2.4858834743499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9418 , TFLOPS: 68.83781344338726, Tokens per sec: 170587.68482386717, Loss: 2.4945545196533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9419 , TFLOPS: 70.34666177858686, Tokens per sec: 174326.7772118572, Loss: 2.482564926147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9420 , TFLOPS: 71.52263036951156, Tokens per sec: 177240.956923803, Loss: 2.4746692180633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9421 , TFLOPS: 69.9860885940982, Tokens per sec: 173433.237140279, Loss: 2.4946672916412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9422 , TFLOPS: 67.83312420203251, Tokens per sec: 168097.95420813365, Loss: 2.5023984909057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9423 , TFLOPS: 71.46423943081494, Tokens per sec: 177096.2577454199, Loss: 2.5200307369232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9424 , TFLOPS: 69.75371913134958, Tokens per sec: 172857.40001397848, Loss: 2.510223388671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9425 , TFLOPS: 68.69830081995846, Tokens per sec: 170241.9571744264, Loss: 2.5041418075561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9426 , TFLOPS: 70.4103500870843, Tokens per sec: 174484.60385616022, Loss: 2.4899513721466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9427 , TFLOPS: 70.53291077494039, Tokens per sec: 174788.32274184193, Loss: 2.5003890991210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9428 , TFLOPS: 68.79791317968386, Tokens per sec: 170488.80757503328, Loss: 2.4991095066070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9429 , TFLOPS: 71.00035631698306, Tokens per sec: 175946.70428840097, Loss: 2.504969358444214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9430 , TFLOPS: 69.60283553211744, Tokens per sec: 172483.49383388064, Loss: 2.4910852909088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9431 , TFLOPS: 70.19318368429612, Tokens per sec: 173946.44158719692, Loss: 2.509028434753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9432 , TFLOPS: 69.7264249282747, Tokens per sec: 172789.76197205426, Loss: 2.484861373901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9433 , TFLOPS: 71.5334236909731, Tokens per sec: 177267.7039633672, Loss: 2.4960453510284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9434 , TFLOPS: 68.68802994886661, Tokens per sec: 170216.50482443097, Loss: 2.5071468353271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9435 , TFLOPS: 68.12541547827323, Tokens per sec: 168822.2841892003, Loss: 2.4904000759124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9436 , TFLOPS: 71.50646621437393, Tokens per sec: 177200.90036674327, Loss: 2.5161662101745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9437 , TFLOPS: 69.4924941834116, Tokens per sec: 172210.05581668456, Loss: 2.5051987171173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9438 , TFLOPS: 68.20581216667178, Tokens per sec: 169021.51603947824, Loss: 2.4938650131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9439 , TFLOPS: 71.48691573056617, Tokens per sec: 177152.45211420406, Loss: 2.489459276199341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9440 , TFLOPS: 71.52969482284958, Tokens per sec: 177258.4634173884, Loss: 2.490004062652588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9441 , TFLOPS: 68.74690770651185, Tokens per sec: 170362.4103937963, Loss: 2.5097413063049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9442 , TFLOPS: 68.92203345776508, Tokens per sec: 170796.39129709586, Loss: 2.488408088684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9443 , TFLOPS: 71.51556031755531, Tokens per sec: 177223.43655622491, Loss: 2.4999523162841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9444 , TFLOPS: 68.44401747546763, Tokens per sec: 169611.81503515536, Loss: 2.493495464324951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9445 , TFLOPS: 68.76813066416112, Tokens per sec: 170415.003220753, Loss: 2.5048460960388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9446 , TFLOPS: 71.49942055204262, Tokens per sec: 177183.44043934252, Loss: 2.5121994018554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9447 , TFLOPS: 70.60175217011084, Tokens per sec: 174958.91930258, Loss: 2.4817512035369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9448 , TFLOPS: 68.89410500211403, Tokens per sec: 170727.1815074192, Loss: 2.4759442806243896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9449 , TFLOPS: 69.89996505119305, Tokens per sec: 173219.8135136686, Loss: 2.5011894702911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9450 , TFLOPS: 71.5111346176904, Tokens per sec: 177212.46918442857, Loss: 2.510215997695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9451 , TFLOPS: 70.04308639194575, Tokens per sec: 173574.483962171, Loss: 2.4843692779541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9452 , TFLOPS: 69.41969747032078, Tokens per sec: 172029.6575424256, Loss: 2.51849365234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9453 , TFLOPS: 71.49331508194813, Tokens per sec: 177168.3104118753, Loss: 2.4836273193359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9454 , TFLOPS: 69.78391554284306, Tokens per sec: 172932.23004806865, Loss: 2.4892737865448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9455 , TFLOPS: 68.0683409707566, Tokens per sec: 168680.84727229804, Loss: 2.497873306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9456 , TFLOPS: 70.52407763739022, Tokens per sec: 174766.43325394217, Loss: 2.482828140258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9457 , TFLOPS: 70.58334186914477, Tokens per sec: 174913.29654872403, Loss: 2.522202491760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9458 , TFLOPS: 68.0264226513512, Tokens per sec: 168576.96905912735, Loss: 2.4990856647491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9459 , TFLOPS: 70.44713618348763, Tokens per sec: 174575.7638553707, Loss: 2.4917242527008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9460 , TFLOPS: 71.50092040234837, Tokens per sec: 177187.15723362233, Loss: 2.477327823638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9461 , TFLOPS: 69.85574458562269, Tokens per sec: 173110.23032869885, Loss: 2.4882941246032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9462 , TFLOPS: 68.64857367961356, Tokens per sec: 170118.72784275503, Loss: 2.496964454650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9463 , TFLOPS: 70.37777408216877, Tokens per sec: 174403.87692743636, Loss: 2.4882214069366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9464 , TFLOPS: 69.8779703299304, Tokens per sec: 173165.30817146122, Loss: 2.4861183166503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9465 , TFLOPS: 68.79936104253213, Tokens per sec: 170492.39553866652, Loss: 2.497934341430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9466 , TFLOPS: 71.50522687198678, Tokens per sec: 177197.82913978308, Loss: 2.502164363861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9467 , TFLOPS: 69.45115212052899, Tokens per sec: 172107.6056306573, Loss: 2.5052363872528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9468 , TFLOPS: 70.7063684341992, Tokens per sec: 175218.16992942384, Loss: 2.4985768795013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9469 , TFLOPS: 69.01412004521525, Tokens per sec: 171024.59200497245, Loss: 2.477816343307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9470 , TFLOPS: 70.5191326968858, Tokens per sec: 174754.1791466421, Loss: 2.496068239212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9471 , TFLOPS: 69.86957237027752, Tokens per sec: 173144.49710232974, Loss: 2.504080295562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9472 , TFLOPS: 69.59348372270873, Tokens per sec: 172460.319019979, Loss: 2.5126237869262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9473 , TFLOPS: 71.53866184398414, Tokens per sec: 177280.68468355876, Loss: 2.467461585998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9474 , TFLOPS: 68.64260980685393, Tokens per sec: 170103.9487090819, Loss: 2.470613479614258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9475 , TFLOPS: 67.94080109017293, Tokens per sec: 168364.78939853405, Loss: 2.4752657413482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9476 , TFLOPS: 71.52263094921679, Tokens per sec: 177240.95836037645, Loss: 2.5022072792053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9477 , TFLOPS: 70.7632882621361, Tokens per sec: 175359.22353328814, Loss: 2.508181571960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9478 , TFLOPS: 67.97327566986371, Tokens per sec: 168445.26498437807, Loss: 2.485893964767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9479 , TFLOPS: 71.01086294488307, Tokens per sec: 175972.74086973784, Loss: 2.481048583984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9480 , TFLOPS: 71.51683015920953, Tokens per sec: 177226.58336372869, Loss: 2.5025885105133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9481 , TFLOPS: 69.63561197751946, Tokens per sec: 172564.71747621038, Loss: 2.503013849258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9482 , TFLOPS: 68.19913772595083, Tokens per sec: 169004.9760401214, Loss: 2.5064635276794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9483 , TFLOPS: 71.53368022200885, Tokens per sec: 177268.33967553303, Loss: 2.5015623569488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9484 , TFLOPS: 69.86833961477761, Tokens per sec: 173141.44219840175, Loss: 2.510502815246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9485 , TFLOPS: 67.87018628664555, Tokens per sec: 168189.79813653041, Loss: 2.4956676959991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9486 , TFLOPS: 71.48587720397529, Tokens per sec: 177149.87853090873, Loss: 2.5039868354797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9487 , TFLOPS: 71.57134103190977, Tokens per sec: 177361.6673111482, Loss: 2.4663965702056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9488 , TFLOPS: 70.0973872094242, Tokens per sec: 173709.04736961145, Loss: 2.5094213485717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9489 , TFLOPS: 68.6196774710259, Tokens per sec: 170047.11985469464, Loss: 2.4940643310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9490 , TFLOPS: 71.52411432280705, Tokens per sec: 177244.6343235402, Loss: 2.4893381595611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9491 , TFLOPS: 69.73107763575035, Tokens per sec: 172801.29189371693, Loss: 2.501326084136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9492 , TFLOPS: 69.52564006823587, Tokens per sec: 172292.19497059978, Loss: 2.4814400672912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9493 , TFLOPS: 70.40266010602367, Tokens per sec: 174465.54723597062, Loss: 2.5005125999450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9494 , TFLOPS: 69.72393724030385, Tokens per sec: 172783.59720722106, Loss: 2.5084924697875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9495 , TFLOPS: 68.75343343683154, Tokens per sec: 170378.58187239704, Loss: 2.490483283996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9496 , TFLOPS: 69.89940993076198, Tokens per sec: 173218.43786408886, Loss: 2.4958174228668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9497 , TFLOPS: 70.68619559834133, Tokens per sec: 175168.1794199467, Loss: 2.5172863006591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9498 , TFLOPS: 69.71300460275869, Tokens per sec: 172756.50492705466, Loss: 2.5054917335510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9499 , TFLOPS: 69.9482657964649, Tokens per sec: 173339.50808121855, Loss: 2.500725507736206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9500 , TFLOPS: 70.4507407162108, Tokens per sec: 174584.6962845308, Loss: 2.494622230529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9501 , TFLOPS: 70.51443691529212, Tokens per sec: 174742.54248256987, Loss: 2.490779399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9502 , TFLOPS: 67.95954863342058, Tokens per sec: 168411.2478759141, Loss: 2.4822897911071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9503 , TFLOPS: 71.52431599042697, Tokens per sec: 177245.13407811787, Loss: 2.499354362487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9504 , TFLOPS: 69.78441706289301, Tokens per sec: 172933.47286999942, Loss: 2.4880661964416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9505 , TFLOPS: 68.34552655290702, Tokens per sec: 169367.74367937935, Loss: 2.509237051010132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9506 , TFLOPS: 71.50169413973053, Tokens per sec: 177189.07464008953, Loss: 2.500591278076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9507 , TFLOPS: 71.48429220911649, Tokens per sec: 177145.95073904702, Loss: 2.5094075202941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9508 , TFLOPS: 68.68632167325244, Tokens per sec: 170212.27152927694, Loss: 2.510096549987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9509 , TFLOPS: 68.6480650762741, Tokens per sec: 170117.46746765258, Loss: 2.4702117443084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9510 , TFLOPS: 71.46349322359794, Tokens per sec: 177094.40856173463, Loss: 2.4975197315216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9511 , TFLOPS: 68.80138683843059, Tokens per sec: 170497.41568406273, Loss: 2.4995510578155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9512 , TFLOPS: 69.60327632846085, Tokens per sec: 172484.58617577734, Loss: 2.473569869995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9513 , TFLOPS: 71.52304073300947, Tokens per sec: 177241.97384975612, Loss: 2.515416145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9514 , TFLOPS: 69.70542780272756, Tokens per sec: 172737.7287819241, Loss: 2.4990031719207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9515 , TFLOPS: 68.76646993872946, Tokens per sec: 170410.88776018933, Loss: 2.501641035079956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9516 , TFLOPS: 69.79790962089011, Tokens per sec: 172966.90891504422, Loss: 2.4968202114105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9517 , TFLOPS: 70.71048440364879, Tokens per sec: 175228.36975513137, Loss: 2.518247604370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9518 , TFLOPS: 69.60342347842014, Tokens per sec: 172484.95082958636, Loss: 2.479285717010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9519 , TFLOPS: 69.14264069622814, Tokens per sec: 171343.08033590086, Loss: 2.4819915294647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9520 , TFLOPS: 71.51549814438978, Tokens per sec: 177223.2824842718, Loss: 2.495648145675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9521 , TFLOPS: 70.5519372679753, Tokens per sec: 174835.47248752366, Loss: 2.514617681503296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9522 , TFLOPS: 67.83182025343125, Tokens per sec: 168094.72287985793, Loss: 2.485983371734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9523 , TFLOPS: 71.46842199826656, Tokens per sec: 177106.62260831284, Loss: 2.4940173625946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9524 , TFLOPS: 70.67367133880555, Tokens per sec: 175137.14292515593, Loss: 2.5141806602478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9525 , TFLOPS: 68.83710657742263, Tokens per sec: 170585.93313213912, Loss: 2.523597478866577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9526 , TFLOPS: 69.98941902791549, Tokens per sec: 173441.49032215585, Loss: 2.485945224761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9527 , TFLOPS: 71.51906939760859, Tokens per sec: 177232.132443154, Loss: 2.486856698989868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9528 , TFLOPS: 70.59381282886626, Tokens per sec: 174939.2447403853, Loss: 2.5044798851013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9529 , TFLOPS: 68.17569949689639, Tokens per sec: 168946.89352659043, Loss: 2.512634754180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9530 , TFLOPS: 70.56728013099146, Tokens per sec: 174873.4937922336, Loss: 2.499598741531372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9531 , TFLOPS: 69.7631904110708, Tokens per sec: 172880.87088847524, Loss: 2.495253562927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9532 , TFLOPS: 69.66018154874277, Tokens per sec: 172625.60358026347, Loss: 2.4992101192474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9533 , TFLOPS: 70.42573199883576, Tokens per sec: 174522.7219279378, Loss: 2.486053705215454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9534 , TFLOPS: 70.55535676205382, Tokens per sec: 174843.94637053862, Loss: 2.489102363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9535 , TFLOPS: 70.10535531913943, Tokens per sec: 173728.79322324556, Loss: 2.4894654750823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9536 , TFLOPS: 68.91124184179128, Tokens per sec: 170769.6484839182, Loss: 2.5157344341278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9537 , TFLOPS: 70.48872328865342, Tokens per sec: 174678.82128316935, Loss: 2.4979429244995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9538 , TFLOPS: 70.03275103389166, Tokens per sec: 173548.8718063793, Loss: 2.5134129524230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9539 , TFLOPS: 68.92709822580017, Tokens per sec: 170808.94235021708, Loss: 2.5190680027008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9540 , TFLOPS: 71.55129890111513, Tokens per sec: 177312.0007032161, Loss: 2.486726760864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9541 , TFLOPS: 69.56264035563223, Tokens per sec: 172383.88575868608, Loss: 2.498673677444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9542 , TFLOPS: 67.89419680439104, Tokens per sec: 168249.29884448063, Loss: 2.5084848403930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9543 , TFLOPS: 71.4840381652147, Tokens per sec: 177145.32119027313, Loss: 2.4892525672912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9544 , TFLOPS: 69.67472683637858, Tokens per sec: 172661.6484053207, Loss: 2.4625754356384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9545 , TFLOPS: 68.8239265268687, Tokens per sec: 170553.27151496403, Loss: 2.5280463695526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9546 , TFLOPS: 70.73040370104228, Tokens per sec: 175277.73196836418, Loss: 2.496063232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9547 , TFLOPS: 71.56986233583002, Tokens per sec: 177358.00293937078, Loss: 2.490138292312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9548 , TFLOPS: 69.6741437271838, Tokens per sec: 172660.20339650125, Loss: 2.5038039684295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9549 , TFLOPS: 67.9835212816362, Tokens per sec: 168470.6547389976, Loss: 2.5168228149414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9550 , TFLOPS: 71.48819733532486, Tokens per sec: 177155.6280719765, Loss: 2.5092697143554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9551 , TFLOPS: 69.70102070520576, Tokens per sec: 172726.80750878324, Loss: 2.504709482192993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9552 , TFLOPS: 68.56678710671396, Tokens per sec: 169916.05170557508, Loss: 2.4905147552490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9553 , TFLOPS: 71.45892202834133, Tokens per sec: 177083.08063633007, Loss: 2.489652156829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9554 , TFLOPS: 71.48337493383681, Tokens per sec: 177143.67762985776, Loss: 2.5046026706695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9555 , TFLOPS: 67.71316791084683, Tokens per sec: 167800.68930429922, Loss: 2.5013885498046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9556 , TFLOPS: 68.85509848242988, Tokens per sec: 170630.51905471904, Loss: 2.4878363609313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9557 , TFLOPS: 71.52171949167318, Tokens per sec: 177238.69966817801, Loss: 2.5017776489257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9558 , TFLOPS: 69.77340713552056, Tokens per sec: 172906.18905712685, Loss: 2.5036580562591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9559 , TFLOPS: 67.92221459337463, Tokens per sec: 168318.73001199623, Loss: 2.4910051822662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9560 , TFLOPS: 71.51343067513771, Tokens per sec: 177218.15906771462, Loss: 2.4798660278320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9561 , TFLOPS: 70.59486590447105, Tokens per sec: 174941.8543777398, Loss: 2.4637198448181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9562 , TFLOPS: 68.8507360117974, Tokens per sec: 170619.70837192592, Loss: 2.5206406116485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9563 , TFLOPS: 69.85505356393972, Tokens per sec: 173108.5179008461, Loss: 2.4851319789886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9564 , TFLOPS: 70.73438980504639, Tokens per sec: 175287.609973192, Loss: 2.4935836791992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9565 , TFLOPS: 69.69728977922638, Tokens per sec: 172717.56186321087, Loss: 2.4942984580993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9566 , TFLOPS: 69.9615330908776, Tokens per sec: 173372.3858982864, Loss: 2.4870352745056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9567 , TFLOPS: 70.493335057662, Tokens per sec: 174690.2497548009, Loss: 2.5033087730407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9568 , TFLOPS: 70.62681942018587, Tokens per sec: 175021.03870965148, Loss: 2.502014636993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9569 , TFLOPS: 68.04883841365229, Tokens per sec: 168632.51778740954, Loss: 2.5060224533081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9570 , TFLOPS: 70.54220428011227, Tokens per sec: 174811.35307142197, Loss: 2.484706163406372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9571 , TFLOPS: 70.65206987828303, Tokens per sec: 175083.6121263838, Loss: 2.4838051795959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9572 , TFLOPS: 68.89375561934753, Tokens per sec: 170726.3156984361, Loss: 2.491267681121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9573 , TFLOPS: 71.53596851597077, Tokens per sec: 177274.01032004703, Loss: 2.478637933731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9574 , TFLOPS: 69.56535293012942, Tokens per sec: 172390.60781710484, Loss: 2.4867751598358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9575 , TFLOPS: 68.91539597186231, Tokens per sec: 170779.94287584937, Loss: 2.499588966369629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9576 , TFLOPS: 68.96393485555126, Tokens per sec: 170900.22757663004, Loss: 2.481518030166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9577 , TFLOPS: 71.40197408747223, Tokens per sec: 176941.9573655228, Loss: 2.467435121536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9578 , TFLOPS: 68.81463402417123, Tokens per sec: 170530.24366962604, Loss: 2.4914627075195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9579 , TFLOPS: 68.78968692688787, Tokens per sec: 170468.42201440822, Loss: 2.462677478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9580 , TFLOPS: 71.50179250096274, Tokens per sec: 177189.3183900583, Loss: 2.5123777389526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9581 , TFLOPS: 70.58988551726704, Tokens per sec: 174929.51242961333, Loss: 2.4782683849334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9582 , TFLOPS: 68.75682629518073, Tokens per sec: 170386.98974914054, Loss: 2.491499662399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9583 , TFLOPS: 69.92541852557405, Tokens per sec: 173282.89002711637, Loss: 2.49930739402771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9584 , TFLOPS: 70.71672666048497, Tokens per sec: 175243.83875521275, Loss: 2.5016369819641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9585 , TFLOPS: 70.48133666854646, Tokens per sec: 174660.51642484096, Loss: 2.477710723876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9586 , TFLOPS: 68.78422312132261, Tokens per sec: 170454.88210232963, Loss: 2.4879236221313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9587 , TFLOPS: 71.50826830533904, Tokens per sec: 177205.36614667156, Loss: 2.52010440826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9588 , TFLOPS: 70.68773126992902, Tokens per sec: 175171.98498330917, Loss: 2.5045087337493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9589 , TFLOPS: 68.00894746562436, Tokens per sec: 168533.66362384564, Loss: 2.504486322402954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9590 , TFLOPS: 71.54479528801915, Tokens per sec: 177295.88403353217, Loss: 2.5073342323303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9591 , TFLOPS: 70.69700033163183, Tokens per sec: 175194.9547392802, Loss: 2.482931613922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9592 , TFLOPS: 68.57272975513945, Tokens per sec: 169930.77824301022, Loss: 2.510685443878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9593 , TFLOPS: 71.51148740644327, Tokens per sec: 177213.34343382105, Loss: 2.496208429336548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9594 , TFLOPS: 71.51537489936052, Tokens per sec: 177222.97706951274, Loss: 2.505112648010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9595 , TFLOPS: 69.03867817376762, Tokens per sec: 171085.44975282642, Loss: 2.4726221561431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9596 , TFLOPS: 67.21241827683524, Tokens per sec: 166559.7765490925, Loss: 2.480772018432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9597 , TFLOPS: 71.52619688521872, Tokens per sec: 177249.7951426095, Loss: 2.4869582653045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9598 , TFLOPS: 69.00292453205358, Tokens per sec: 170996.84829007025, Loss: 2.5008504390716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9599 , TFLOPS: 68.86757771643042, Tokens per sec: 170661.44397127346, Loss: 2.505007743835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9600 , TFLOPS: 70.50160007799963, Tokens per sec: 174710.7313856647, Loss: 2.5033464431762695 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/optimizer.pt +[rank0]:[2024-08-26 22:16:35,382] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006316158003755845, 'preprocessing_with_comm': 0.0014687529910588637, 'state_converting': 0.9875874440040207, : 0.9967369100049837}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600, took 4.47s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007600 + eval ppl=10.217534065246582, eval loss=2.3241052627563477 +------------------------------------------------------------------ +iteration: 9601 , TFLOPS: 64.54763204560365, Tokens per sec: 159956.14271766404, Loss: 2.493927478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9602 , TFLOPS: 68.81563983007533, Tokens per sec: 170532.73616745506, Loss: 2.5057590007781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9603 , TFLOPS: 69.21293574454288, Tokens per sec: 171517.2792092636, Loss: 2.517369270324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9604 , TFLOPS: 70.6294462119777, Tokens per sec: 175027.54818907662, Loss: 2.4901909828186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9605 , TFLOPS: 69.84963992397148, Tokens per sec: 173095.10230464314, Loss: 2.494180917739868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9606 , TFLOPS: 69.23652508943698, Tokens per sec: 171575.73620449202, Loss: 2.4911293983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9607 , TFLOPS: 70.46056032293865, Tokens per sec: 174609.03035172136, Loss: 2.4955549240112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9608 , TFLOPS: 69.82724012082414, Tokens per sec: 173039.59312490202, Loss: 2.4946224689483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9609 , TFLOPS: 68.83817631112873, Tokens per sec: 170588.58405010373, Loss: 2.476386547088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9610 , TFLOPS: 70.91696612387088, Tokens per sec: 175740.05420368092, Loss: 2.5033912658691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9611 , TFLOPS: 69.34975223449669, Tokens per sec: 171856.32554294972, Loss: 2.492589235305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9612 , TFLOPS: 68.62194945046019, Tokens per sec: 170052.75007001206, Loss: 2.502751350402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9613 , TFLOPS: 70.65059682069283, Tokens per sec: 175079.96172740462, Loss: 2.510580062866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9614 , TFLOPS: 71.52505296493004, Tokens per sec: 177246.96038212092, Loss: 2.5003714561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9615 , TFLOPS: 68.27334594173536, Tokens per sec: 169188.87217354524, Loss: 2.491748332977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9616 , TFLOPS: 68.6713236075638, Tokens per sec: 170175.10467032721, Loss: 2.5004262924194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9617 , TFLOPS: 71.53047910833459, Tokens per sec: 177260.40696321643, Loss: 2.5152218341827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9618 , TFLOPS: 68.6979653742061, Tokens per sec: 170241.12590289972, Loss: 2.472811222076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9619 , TFLOPS: 68.88270392221699, Tokens per sec: 170698.9283755017, Loss: 2.5312604904174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9620 , TFLOPS: 71.5084641851288, Tokens per sec: 177205.8515583685, Loss: 2.515449047088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9621 , TFLOPS: 69.78925503544966, Tokens per sec: 172945.46189894146, Loss: 2.5063657760620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9622 , TFLOPS: 67.7708009943554, Tokens per sec: 167943.51043404167, Loss: 2.4991822242736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9623 , TFLOPS: 70.67841740460302, Tokens per sec: 175148.90419902437, Loss: 2.4874632358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9624 , TFLOPS: 70.73419608274511, Tokens per sec: 175287.12990798912, Loss: 2.506967306137085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9625 , TFLOPS: 69.88288362025773, Tokens per sec: 173177.48384613593, Loss: 2.508380651473999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9626 , TFLOPS: 68.91527225907761, Tokens per sec: 170779.6363019408, Loss: 2.4715113639831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9627 , TFLOPS: 71.54980122030541, Tokens per sec: 177308.28928518647, Loss: 2.4818310737609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9628 , TFLOPS: 69.76979461189167, Tokens per sec: 172897.23682562215, Loss: 2.491793155670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9629 , TFLOPS: 68.7217766844666, Tokens per sec: 170300.13295275756, Loss: 2.5244877338409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9630 , TFLOPS: 70.40194615450902, Tokens per sec: 174463.7779854691, Loss: 2.5085067749023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9631 , TFLOPS: 70.68680770900538, Tokens per sec: 175169.69629760174, Loss: 2.497304916381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9632 , TFLOPS: 68.84260195493763, Tokens per sec: 170599.55128298711, Loss: 2.48286771774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9633 , TFLOPS: 69.65724224311312, Tokens per sec: 172618.31965712193, Loss: 2.486650228500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9634 , TFLOPS: 71.53350029628278, Tokens per sec: 177267.89379976338, Loss: 2.494293689727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9635 , TFLOPS: 68.49567297220521, Tokens per sec: 169739.82304639378, Loss: 2.493101119995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9636 , TFLOPS: 68.77511432615567, Tokens per sec: 170432.30950448936, Loss: 2.5016047954559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9637 , TFLOPS: 70.43381399917848, Tokens per sec: 174542.75001509287, Loss: 2.505254030227661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9638 , TFLOPS: 69.88039301981182, Tokens per sec: 173171.3118638687, Loss: 2.508103609085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9639 , TFLOPS: 68.83157183444911, Tokens per sec: 170572.21742934844, Loss: 2.5039010047912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9640 , TFLOPS: 70.43500565667289, Tokens per sec: 174545.70307363593, Loss: 2.492429733276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9641 , TFLOPS: 70.58160866809573, Tokens per sec: 174909.00148559661, Loss: 2.4967668056488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9642 , TFLOPS: 69.14408571295372, Tokens per sec: 171346.66124652856, Loss: 2.4851174354553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9643 , TFLOPS: 69.96413885448732, Tokens per sec: 173378.8432675598, Loss: 2.500725746154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9644 , TFLOPS: 70.47067503190611, Tokens per sec: 174634.09571476994, Loss: 2.4841597080230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9645 , TFLOPS: 70.72028521738387, Tokens per sec: 175252.65725121569, Loss: 2.4931867122650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9646 , TFLOPS: 69.0490467824191, Tokens per sec: 171111.14430726127, Loss: 2.4936413764953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9647 , TFLOPS: 71.49573309459242, Tokens per sec: 177174.302513574, Loss: 2.503350019454956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9648 , TFLOPS: 68.68252793690392, Tokens per sec: 170202.8702326908, Loss: 2.520277738571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9649 , TFLOPS: 68.74800343721854, Tokens per sec: 170365.1257351922, Loss: 2.4988598823547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9650 , TFLOPS: 70.76772970318223, Tokens per sec: 175370.22991346556, Loss: 2.4905171394348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9651 , TFLOPS: 70.7453002655217, Tokens per sec: 175314.64729613625, Loss: 2.492384195327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9652 , TFLOPS: 67.9044784396522, Tokens per sec: 168274.7778692734, Loss: 2.4905405044555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9653 , TFLOPS: 70.63187807673353, Tokens per sec: 175033.5746178336, Loss: 2.4940614700317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9654 , TFLOPS: 71.55305814999244, Tokens per sec: 177316.36031573135, Loss: 2.491048574447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9655 , TFLOPS: 67.50251934821395, Tokens per sec: 167278.67896124013, Loss: 2.50966215133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9656 , TFLOPS: 68.75312351430219, Tokens per sec: 170377.8138502285, Loss: 2.4810266494750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9657 , TFLOPS: 71.56941643132676, Tokens per sec: 177356.89793889126, Loss: 2.4911491870880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9658 , TFLOPS: 70.69603274523979, Tokens per sec: 175192.55695361228, Loss: 2.4992387294769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9659 , TFLOPS: 68.10368961251505, Tokens per sec: 168768.44510054984, Loss: 2.497617721557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9660 , TFLOPS: 70.85698829878291, Tokens per sec: 175591.42254601006, Loss: 2.497946262359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9661 , TFLOPS: 70.57236017653953, Tokens per sec: 174886.08270471497, Loss: 2.487074613571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9662 , TFLOPS: 69.6936494907534, Tokens per sec: 172708.54082736396, Loss: 2.488084316253662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9663 , TFLOPS: 68.94922031495499, Tokens per sec: 170863.76332409203, Loss: 2.5039610862731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9664 , TFLOPS: 71.44615486746034, Tokens per sec: 177051.44220524805, Loss: 2.498167037963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9665 , TFLOPS: 70.26865907759442, Tokens per sec: 174133.47792609106, Loss: 2.5051627159118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9666 , TFLOPS: 68.74237472760777, Tokens per sec: 170351.17717272314, Loss: 2.5177245140075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9667 , TFLOPS: 70.48322988865809, Tokens per sec: 174665.20803283292, Loss: 2.5046424865722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9668 , TFLOPS: 69.67889576300473, Tokens per sec: 172671.9794647455, Loss: 2.490590810775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9669 , TFLOPS: 68.80035571793346, Tokens per sec: 170494.8604538836, Loss: 2.5020980834960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9670 , TFLOPS: 69.82254235275722, Tokens per sec: 173027.95153813035, Loss: 2.495351791381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9671 , TFLOPS: 70.69620720817545, Tokens per sec: 175192.98929198552, Loss: 2.500459671020508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9672 , TFLOPS: 69.73622992711745, Tokens per sec: 172814.05983929636, Loss: 2.4872734546661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9673 , TFLOPS: 69.87023768553038, Tokens per sec: 173146.14582681685, Loss: 2.483752727508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9674 , TFLOPS: 70.43282183948584, Tokens per sec: 174540.2913340791, Loss: 2.5069010257720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9675 , TFLOPS: 69.71987238960773, Tokens per sec: 172773.52405941577, Loss: 2.506676435470581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9676 , TFLOPS: 68.97696808068238, Tokens per sec: 170932.52534423323, Loss: 2.51071834564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9677 , TFLOPS: 69.97024125183262, Tokens per sec: 173393.96568041603, Loss: 2.5071516036987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9678 , TFLOPS: 70.74554657127659, Tokens per sec: 175315.2576689297, Loss: 2.481070041656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9679 , TFLOPS: 68.86294094360197, Tokens per sec: 170649.95353742188, Loss: 2.4942498207092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9680 , TFLOPS: 70.65208214458731, Tokens per sec: 175083.6425236371, Loss: 2.5051259994506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9681 , TFLOPS: 70.56013190979658, Tokens per sec: 174855.77971267112, Loss: 2.477543830871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9682 , TFLOPS: 69.14564856713874, Tokens per sec: 171350.53417136165, Loss: 2.4851059913635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9683 , TFLOPS: 68.79486959460638, Tokens per sec: 170481.26523593272, Loss: 2.500964641571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9684 , TFLOPS: 71.54142164387014, Tokens per sec: 177287.52377169335, Loss: 2.4835057258605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9685 , TFLOPS: 68.84849101806974, Tokens per sec: 170614.14503016233, Loss: 2.495198965072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9686 , TFLOPS: 68.80545427572193, Tokens per sec: 170507.49524173592, Loss: 2.4772565364837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9687 , TFLOPS: 71.54086885665771, Tokens per sec: 177286.1539040864, Loss: 2.4997639656066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9688 , TFLOPS: 70.67114027944753, Tokens per sec: 175130.8706812476, Loss: 2.501823902130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9689 , TFLOPS: 67.41588851278125, Tokens per sec: 167063.9982078636, Loss: 2.502948045730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9690 , TFLOPS: 70.71857148793038, Tokens per sec: 175248.410440847, Loss: 2.4976680278778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9691 , TFLOPS: 70.61116819996205, Tokens per sec: 174982.2532618126, Loss: 2.50805401802063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9692 , TFLOPS: 69.61039664310371, Tokens per sec: 172502.23109982978, Loss: 2.4979076385498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9693 , TFLOPS: 68.96678110074923, Tokens per sec: 170907.28088577045, Loss: 2.487248420715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9694 , TFLOPS: 71.54562091784202, Tokens per sec: 177297.93003518388, Loss: 2.5240917205810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9695 , TFLOPS: 69.68509097004939, Tokens per sec: 172687.33187026053, Loss: 2.502699851989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9696 , TFLOPS: 68.87376502084172, Tokens per sec: 170676.77679319255, Loss: 2.5068092346191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9697 , TFLOPS: 70.68091219417373, Tokens per sec: 175155.08656240438, Loss: 2.481276750564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9698 , TFLOPS: 70.68373086000712, Tokens per sec: 175162.07152684117, Loss: 2.4937119483947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9699 , TFLOPS: 68.8177107737013, Tokens per sec: 170537.8681938935, Loss: 2.4763987064361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9700 , TFLOPS: 69.61426572412772, Tokens per sec: 172511.81910882174, Loss: 2.4865822792053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9701 , TFLOPS: 71.54050281478543, Tokens per sec: 177285.24681200404, Loss: 2.4671778678894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9702 , TFLOPS: 68.9937008216061, Tokens per sec: 170973.99091950577, Loss: 2.506035327911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9703 , TFLOPS: 68.85709935252594, Tokens per sec: 170635.4774312311, Loss: 2.5069422721862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9704 , TFLOPS: 70.5439448612491, Tokens per sec: 174815.66642321987, Loss: 2.4841368198394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9705 , TFLOPS: 69.89971922704405, Tokens per sec: 173219.2043343476, Loss: 2.5120725631713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9706 , TFLOPS: 68.81406957823192, Tokens per sec: 170528.8449104096, Loss: 2.5120205879211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9707 , TFLOPS: 70.43219913731592, Tokens per sec: 174538.74820950572, Loss: 2.4948513507843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9708 , TFLOPS: 70.62676869398003, Tokens per sec: 175020.91300452544, Loss: 2.4744811058044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9709 , TFLOPS: 69.08955389415907, Tokens per sec: 171211.52539237967, Loss: 2.481837034225464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9710 , TFLOPS: 69.93113177737648, Tokens per sec: 173297.04809444948, Loss: 2.499471426010132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9711 , TFLOPS: 70.55982992529897, Tokens per sec: 174855.0313618197, Loss: 2.492711067199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9712 , TFLOPS: 70.09707096395822, Tokens per sec: 173708.26367850945, Loss: 2.4703822135925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9713 , TFLOPS: 68.76673757294131, Tokens per sec: 170411.55098724863, Loss: 2.502802848815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9714 , TFLOPS: 70.57029198131089, Tokens per sec: 174880.95748910785, Loss: 2.4850926399230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9715 , TFLOPS: 69.81818850253703, Tokens per sec: 173017.1622176685, Loss: 2.512624502182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9716 , TFLOPS: 68.85163537201922, Tokens per sec: 170621.9370856283, Loss: 2.4942781925201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9717 , TFLOPS: 70.62547349053803, Tokens per sec: 175017.70334771773, Loss: 2.4992637634277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9718 , TFLOPS: 69.55250515821896, Tokens per sec: 172358.76962301185, Loss: 2.4882380962371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9719 , TFLOPS: 68.87522071922183, Tokens per sec: 170680.38417413685, Loss: 2.483081579208374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9720 , TFLOPS: 70.72472923577176, Tokens per sec: 175263.67001832987, Loss: 2.496586561203003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9721 , TFLOPS: 71.49768546176446, Tokens per sec: 177179.14069449727, Loss: 2.503107786178589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9722 , TFLOPS: 68.00336498898965, Tokens per sec: 168519.82963178426, Loss: 2.5188069343566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9723 , TFLOPS: 68.8227922039476, Tokens per sec: 170550.46053780956, Loss: 2.5139212608337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9724 , TFLOPS: 71.53559137954171, Tokens per sec: 177273.07573443084, Loss: 2.474529981613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9725 , TFLOPS: 70.60441683158211, Tokens per sec: 174965.52262724252, Loss: 2.4944043159484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9726 , TFLOPS: 68.19378180123302, Tokens per sec: 168991.70346866685, Loss: 2.499109983444214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9727 , TFLOPS: 70.93575069526733, Tokens per sec: 175786.60444089267, Loss: 2.50193190574646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9728 , TFLOPS: 70.71778268862131, Tokens per sec: 175246.45570926572, Loss: 2.503284454345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9729 , TFLOPS: 69.02452360032315, Tokens per sec: 171050.373160576, Loss: 2.500927686691284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9730 , TFLOPS: 69.06888597968627, Tokens per sec: 171160.30802355736, Loss: 2.4872684478759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9731 , TFLOPS: 71.50087717106346, Tokens per sec: 177187.050101736, Loss: 2.4960968494415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9732 , TFLOPS: 69.64390609296542, Tokens per sec: 172585.2712079577, Loss: 2.5121872425079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9733 , TFLOPS: 67.77612635824597, Tokens per sec: 167956.7072723998, Loss: 2.484009265899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9734 , TFLOPS: 71.49152390032691, Tokens per sec: 177163.871666503, Loss: 2.503244400024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9735 , TFLOPS: 69.87798891410604, Tokens per sec: 173165.35422509562, Loss: 2.5248289108276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9736 , TFLOPS: 68.8347690474761, Tokens per sec: 170580.14047543143, Loss: 2.512118101119995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9737 , TFLOPS: 69.62541709559203, Tokens per sec: 172539.45343573525, Loss: 2.4987215995788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9738 , TFLOPS: 70.48794339989475, Tokens per sec: 174676.88863291353, Loss: 2.4882187843322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9739 , TFLOPS: 68.86061008695148, Tokens per sec: 170644.17741787693, Loss: 2.4879040718078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9740 , TFLOPS: 70.70901759748976, Tokens per sec: 175224.73484788727, Loss: 2.483492136001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9741 , TFLOPS: 69.17581014170258, Tokens per sec: 171425.27787570257, Loss: 2.5111238956451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9742 , TFLOPS: 69.89620306201543, Tokens per sec: 173210.49089006917, Loss: 2.508568525314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9743 , TFLOPS: 68.92590373447848, Tokens per sec: 170805.9822691379, Loss: 2.4988715648651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9744 , TFLOPS: 69.8001269509936, Tokens per sec: 172972.4037032997, Loss: 2.4978647232055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9745 , TFLOPS: 70.62532075309967, Tokens per sec: 175017.3248475201, Loss: 2.500063180923462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9746 , TFLOPS: 68.86108626065956, Tokens per sec: 170645.3574287804, Loss: 2.497799873352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9747 , TFLOPS: 70.69973231791865, Tokens per sec: 175201.72490225206, Loss: 2.480355978012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9748 , TFLOPS: 69.92052074978696, Tokens per sec: 173270.75279918144, Loss: 2.495054244995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9749 , TFLOPS: 69.42778065468279, Tokens per sec: 172049.68856371092, Loss: 2.5037431716918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9750 , TFLOPS: 68.7202770498512, Tokens per sec: 170296.41669298222, Loss: 2.4860715866088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9751 , TFLOPS: 70.30272704647963, Tokens per sec: 174217.9021058852, Loss: 2.496575355529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9752 , TFLOPS: 69.6042616987606, Tokens per sec: 172487.02803192762, Loss: 2.4974069595336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9753 , TFLOPS: 68.84346099831903, Tokens per sec: 170601.68008711765, Loss: 2.484927177429199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9754 , TFLOPS: 71.49532567052964, Tokens per sec: 177173.29287186492, Loss: 2.4884426593780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9755 , TFLOPS: 68.83078583041225, Tokens per sec: 170570.26962475965, Loss: 2.480973243713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9756 , TFLOPS: 68.8261746967251, Tokens per sec: 170558.8427275246, Loss: 2.476720094680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9757 , TFLOPS: 70.61236951093217, Tokens per sec: 174985.23024273192, Loss: 2.503390312194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9758 , TFLOPS: 70.56715310661029, Tokens per sec: 174873.17901182416, Loss: 2.487968921661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9759 , TFLOPS: 68.04846783156817, Tokens per sec: 168631.59944418212, Loss: 2.4722812175750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9760 , TFLOPS: 70.05386198909814, Tokens per sec: 173601.18707894464, Loss: 2.494490385055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9761 , TFLOPS: 71.53497059883398, Tokens per sec: 177271.53737145246, Loss: 2.5219671726226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9762 , TFLOPS: 69.93628679196145, Tokens per sec: 173309.82278846277, Loss: 2.509235382080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9763 , TFLOPS: 67.89629728397679, Tokens per sec: 168254.50406428106, Loss: 2.4984371662139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9764 , TFLOPS: 70.62378987398931, Tokens per sec: 175013.53116044437, Loss: 2.4929988384246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9765 , TFLOPS: 70.57076067548408, Tokens per sec: 174882.11896489994, Loss: 2.4979569911956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9766 , TFLOPS: 68.7576441324349, Tokens per sec: 170389.01643994858, Loss: 2.4914870262145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9767 , TFLOPS: 70.7566011928596, Tokens per sec: 175342.65223897947, Loss: 2.4981698989868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9768 , TFLOPS: 71.48364272654183, Tokens per sec: 177144.34124968058, Loss: 2.4858784675598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9769 , TFLOPS: 68.95606443562778, Tokens per sec: 170880.72380906134, Loss: 2.4988911151885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9770 , TFLOPS: 67.94717512646291, Tokens per sec: 168380.58496261865, Loss: 2.476930856704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9771 , TFLOPS: 71.53657379045987, Tokens per sec: 177275.51025690755, Loss: 2.4936697483062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9772 , TFLOPS: 69.64755914452678, Tokens per sec: 172594.3238721436, Loss: 2.513049364089966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9773 , TFLOPS: 69.57851670136715, Tokens per sec: 172423.2290923411, Loss: 2.4974236488342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9774 , TFLOPS: 69.8438725427494, Tokens per sec: 173080.81009864534, Loss: 2.4855875968933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9775 , TFLOPS: 70.7236834206629, Tokens per sec: 175261.07837328408, Loss: 2.506910800933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9776 , TFLOPS: 69.18790788199226, Tokens per sec: 171455.2574088165, Loss: 2.504011392593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9777 , TFLOPS: 69.86875249554495, Tokens per sec: 173142.46536242578, Loss: 2.4996681213378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9778 , TFLOPS: 69.653736629051, Tokens per sec: 172609.6323593007, Loss: 2.482703685760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9779 , TFLOPS: 70.55361666124672, Tokens per sec: 174839.63420905068, Loss: 2.502995491027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9780 , TFLOPS: 69.22627830387401, Tokens per sec: 171550.34354109038, Loss: 2.4834976196289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9781 , TFLOPS: 70.429134725698, Tokens per sec: 174531.15425994375, Loss: 2.5115256309509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9782 , TFLOPS: 69.20249741823312, Tokens per sec: 171491.41188679147, Loss: 2.4889888763427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9783 , TFLOPS: 68.68257294150047, Tokens per sec: 170202.9817590387, Loss: 2.513556957244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9784 , TFLOPS: 70.66101751487372, Tokens per sec: 175105.78535551907, Loss: 2.4976933002471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9785 , TFLOPS: 69.67387357695027, Tokens per sec: 172659.53393446308, Loss: 2.506901264190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9786 , TFLOPS: 68.80018132245422, Tokens per sec: 170494.42828267478, Loss: 2.4982259273529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9787 , TFLOPS: 70.68942298576926, Tokens per sec: 175176.17724151918, Loss: 2.4930973052978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9788 , TFLOPS: 70.59943597212794, Tokens per sec: 174953.17950882812, Loss: 2.499206066131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9789 , TFLOPS: 69.83749300812325, Tokens per sec: 173065.000909078, Loss: 2.4888174533843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9790 , TFLOPS: 68.18739953821229, Tokens per sec: 168975.88751783717, Loss: 2.5036299228668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9791 , TFLOPS: 71.50470341316955, Tokens per sec: 177196.53195117027, Loss: 2.492094039916992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9792 , TFLOPS: 69.54665610769744, Tokens per sec: 172344.27503149345, Loss: 2.512291193008423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9793 , TFLOPS: 68.99099216115741, Tokens per sec: 170967.27856053034, Loss: 2.484220266342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9794 , TFLOPS: 70.72473801256051, Tokens per sec: 175263.69176817924, Loss: 2.4895198345184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9795 , TFLOPS: 70.69073465634665, Tokens per sec: 175179.4277057024, Loss: 2.4869027137756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9796 , TFLOPS: 68.83312843240681, Tokens per sec: 170576.07485056104, Loss: 2.4854555130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9797 , TFLOPS: 68.96373162026798, Tokens per sec: 170899.7239372099, Loss: 2.4886152744293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9798 , TFLOPS: 71.5110159084384, Tokens per sec: 177212.17500982518, Loss: 2.4900524616241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9799 , TFLOPS: 69.92742018594525, Tokens per sec: 173287.85036201734, Loss: 2.49473237991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9800 , TFLOPS: 68.79456789781906, Tokens per sec: 170480.5175980591, Loss: 2.497068405151367 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/model.pt +[rank0]:[2024-08-26 22:41:59,695] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006398191006155685, 'preprocessing_with_comm': 0.0013291280047269538, 'state_converting': 1.0181726029986748, : 1.0273056070000166}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800, took 4.90s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0007800 + eval ppl=10.211738586425781, eval loss=2.323537826538086 +------------------------------------------------------------------ +iteration: 9801 , TFLOPS: 67.03146231324324, Tokens per sec: 166111.3477373666, Loss: 2.499112844467163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9802 , TFLOPS: 70.62920475604412, Tokens per sec: 175026.94983467468, Loss: 2.4814443588256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9803 , TFLOPS: 66.39111404786637, Tokens per sec: 164524.494195578, Loss: 2.5120317935943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9804 , TFLOPS: 71.3597577577659, Tokens per sec: 176837.34064999793, Loss: 2.502073049545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9805 , TFLOPS: 70.63461183639099, Tokens per sec: 175040.34917541328, Loss: 2.484841823577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9806 , TFLOPS: 68.83183465111273, Tokens per sec: 170572.86871799253, Loss: 2.5126450061798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9807 , TFLOPS: 69.33935598683658, Tokens per sec: 171830.56249600672, Loss: 2.4970345497131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9808 , TFLOPS: 71.45438268297, Tokens per sec: 177071.83163844974, Loss: 2.5069925785064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9809 , TFLOPS: 68.9661432382291, Tokens per sec: 170905.70019217915, Loss: 2.492353916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9810 , TFLOPS: 68.8756781083867, Tokens per sec: 170681.517634873, Loss: 2.490957260131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9811 , TFLOPS: 70.14642619563101, Tokens per sec: 173830.57137952224, Loss: 2.498849630355835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9812 , TFLOPS: 69.81576008877356, Tokens per sec: 173011.14434085073, Loss: 2.50158953666687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9813 , TFLOPS: 67.97148671550661, Tokens per sec: 168440.8317584116, Loss: 2.4945881366729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9814 , TFLOPS: 71.49985157376994, Tokens per sec: 177184.50855866377, Loss: 2.483333110809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9815 , TFLOPS: 68.69607245885632, Tokens per sec: 170236.4350501411, Loss: 2.5110344886779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9816 , TFLOPS: 68.05340921879498, Tokens per sec: 168643.84474606888, Loss: 2.4832582473754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9817 , TFLOPS: 71.53002909436793, Tokens per sec: 177259.29177903393, Loss: 2.4973206520080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9818 , TFLOPS: 70.38136594381857, Tokens per sec: 174412.77795628077, Loss: 2.5063095092773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9819 , TFLOPS: 68.92502756675826, Tokens per sec: 170803.81102900932, Loss: 2.5159778594970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9820 , TFLOPS: 69.08686276456903, Tokens per sec: 171204.85647680316, Loss: 2.5069174766540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9821 , TFLOPS: 71.51519676594482, Tokens per sec: 177222.5356352856, Loss: 2.462402105331421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9822 , TFLOPS: 70.19782695595362, Tokens per sec: 173957.94812586284, Loss: 2.486637592315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9823 , TFLOPS: 68.28589525910748, Tokens per sec: 169219.9707644148, Loss: 2.5087552070617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9824 , TFLOPS: 71.51668874784909, Tokens per sec: 177226.23293079974, Loss: 2.5269622802734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9825 , TFLOPS: 69.62143198581583, Tokens per sec: 172529.57789471376, Loss: 2.5022096633911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9826 , TFLOPS: 68.08164452111649, Tokens per sec: 168713.8149355969, Loss: 2.5239176750183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9827 , TFLOPS: 70.65341233383153, Tokens per sec: 175086.93887911667, Loss: 2.5027146339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9828 , TFLOPS: 71.54324621510729, Tokens per sec: 177292.04526021206, Loss: 2.494830369949341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9829 , TFLOPS: 68.87236365951655, Tokens per sec: 170673.30406545606, Loss: 2.4953696727752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9830 , TFLOPS: 68.36375972211987, Tokens per sec: 169412.9274812382, Loss: 2.4741315841674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9831 , TFLOPS: 71.56394673060682, Tokens per sec: 177343.34341796994, Loss: 2.481898546218872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9832 , TFLOPS: 69.87045829526348, Tokens per sec: 173146.6925220376, Loss: 2.51033091545105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9833 , TFLOPS: 66.99296241192297, Tokens per sec: 166015.9407407807, Loss: 2.500356912612915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9834 , TFLOPS: 71.55911477162688, Tokens per sec: 177331.36929133345, Loss: 2.485167980194092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9835 , TFLOPS: 70.5780085329119, Tokens per sec: 174900.07995402347, Loss: 2.49082350730896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9836 , TFLOPS: 68.80330478605373, Tokens per sec: 170502.1685695523, Loss: 2.487492322921753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9837 , TFLOPS: 69.00354609554701, Tokens per sec: 170998.38859287783, Loss: 2.506587028503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9838 , TFLOPS: 71.50224934407133, Tokens per sec: 177190.45049760683, Loss: 2.491142749786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9839 , TFLOPS: 69.91649010809265, Tokens per sec: 173260.76442505108, Loss: 2.485496997833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9840 , TFLOPS: 67.54265458876276, Tokens per sec: 167378.1385085818, Loss: 2.4977974891662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9841 , TFLOPS: 71.51971038440101, Tokens per sec: 177233.72087903618, Loss: 2.49682354927063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9842 , TFLOPS: 70.57071717900345, Tokens per sec: 174882.0111758294, Loss: 2.509026050567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9843 , TFLOPS: 68.04161655291868, Tokens per sec: 168614.62122095807, Loss: 2.50695538520813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9844 , TFLOPS: 70.44248238559439, Tokens per sec: 174564.2312315331, Loss: 2.4853475093841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9845 , TFLOPS: 71.45925575464877, Tokens per sec: 177083.90764688287, Loss: 2.519956111907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9846 , TFLOPS: 68.356582811307, Tokens per sec: 169395.14230564193, Loss: 2.498056411743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9847 , TFLOPS: 69.99932961231211, Tokens per sec: 173466.04984203164, Loss: 2.488614559173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9848 , TFLOPS: 70.41529669622622, Tokens per sec: 174496.8620985279, Loss: 2.4928946495056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9849 , TFLOPS: 69.75975876238859, Tokens per sec: 172872.3668850134, Loss: 2.516219139099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9850 , TFLOPS: 68.2853820015623, Tokens per sec: 169218.69885567788, Loss: 2.5122334957122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9851 , TFLOPS: 70.35145053321295, Tokens per sec: 174338.64427334553, Loss: 2.511303424835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9852 , TFLOPS: 70.70419283652002, Tokens per sec: 175212.77855871295, Loss: 2.5017757415771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9853 , TFLOPS: 68.3057659941559, Tokens per sec: 169269.212634807, Loss: 2.49495792388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9854 , TFLOPS: 70.78505219761922, Tokens per sec: 175413.1569629107, Loss: 2.4926624298095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9855 , TFLOPS: 69.64790126178069, Tokens per sec: 172595.17167638612, Loss: 2.4906234741210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9856 , TFLOPS: 69.7748314087113, Tokens per sec: 172909.71856299014, Loss: 2.4846370220184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9857 , TFLOPS: 69.78176215364371, Tokens per sec: 172926.89371241728, Loss: 2.506518840789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9858 , TFLOPS: 71.54724172000503, Tokens per sec: 177301.94656148992, Loss: 2.4735047817230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9859 , TFLOPS: 69.55542849430427, Tokens per sec: 172366.01397186329, Loss: 2.5126404762268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9860 , TFLOPS: 68.14998626102675, Tokens per sec: 168883.1732955587, Loss: 2.5093770027160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9861 , TFLOPS: 71.5115759807689, Tokens per sec: 177213.56293075727, Loss: 2.497358560562134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9862 , TFLOPS: 69.5581649797532, Tokens per sec: 172372.7952842545, Loss: 2.497873544692993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9863 , TFLOPS: 67.97157158964117, Tokens per sec: 168441.04208586668, Loss: 2.5209567546844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9864 , TFLOPS: 71.51826425294558, Tokens per sec: 177230.13720598508, Loss: 2.46712327003479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9865 , TFLOPS: 71.51176728096023, Tokens per sec: 177214.03699370482, Loss: 2.485051155090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9866 , TFLOPS: 67.25636506552011, Tokens per sec: 166668.68153259208, Loss: 2.494565725326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9867 , TFLOPS: 69.66742962553899, Tokens per sec: 172643.56511300607, Loss: 2.4758100509643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9868 , TFLOPS: 71.54572088892714, Tokens per sec: 177298.17777454547, Loss: 2.4710631370544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9869 , TFLOPS: 69.76090440935937, Tokens per sec: 172875.20592441043, Loss: 2.5093564987182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9870 , TFLOPS: 67.96406705603827, Tokens per sec: 168422.44502490608, Loss: 2.50161075592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9871 , TFLOPS: 71.45404732421223, Tokens per sec: 177071.00058250513, Loss: 2.5147101879119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9872 , TFLOPS: 70.69047430993969, Tokens per sec: 175178.78253862643, Loss: 2.507869005203247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9873 , TFLOPS: 68.05518199132509, Tokens per sec: 168648.2378716855, Loss: 2.4927914142608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9874 , TFLOPS: 69.52488001182688, Tokens per sec: 172290.31146709115, Loss: 2.494431257247925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9875 , TFLOPS: 71.471534448867, Tokens per sec: 177114.33560376547, Loss: 2.5149776935577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9876 , TFLOPS: 69.13307497951796, Tokens per sec: 171319.37543614232, Loss: 2.510399580001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9877 , TFLOPS: 68.70605836704685, Tokens per sec: 170261.1812306758, Loss: 2.4927756786346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9878 , TFLOPS: 71.52712705885632, Tokens per sec: 177252.10021535057, Loss: 2.5001747608184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9879 , TFLOPS: 69.88287560446469, Tokens per sec: 173177.46398211774, Loss: 2.47489070892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9880 , TFLOPS: 68.08661164617696, Tokens per sec: 168726.1240186708, Loss: 2.479933500289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9881 , TFLOPS: 70.40793998805738, Tokens per sec: 174478.6313652772, Loss: 2.4945616722106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9882 , TFLOPS: 70.54066438891716, Tokens per sec: 174807.53705140747, Loss: 2.499009132385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9883 , TFLOPS: 68.00312177247518, Tokens per sec: 168519.22691446892, Loss: 2.5096144676208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9884 , TFLOPS: 71.51771613242208, Tokens per sec: 177228.77890294738, Loss: 2.4861629009246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9885 , TFLOPS: 70.48501152421221, Tokens per sec: 174669.62312199944, Loss: 2.498927116394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9886 , TFLOPS: 69.01856806295329, Tokens per sec: 171035.61468291693, Loss: 2.4889936447143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9887 , TFLOPS: 69.07984763898683, Tokens per sec: 171187.47222283058, Loss: 2.493760347366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9888 , TFLOPS: 71.52674007219804, Tokens per sec: 177251.14121978122, Loss: 2.5021986961364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9889 , TFLOPS: 69.61624751369554, Tokens per sec: 172516.73020168292, Loss: 2.504972457885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9890 , TFLOPS: 67.58752661356468, Tokens per sec: 167489.33632910255, Loss: 2.492368459701538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9891 , TFLOPS: 71.53721952775877, Tokens per sec: 177277.11046506802, Loss: 2.515167236328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9892 , TFLOPS: 69.61940325731959, Tokens per sec: 172524.5504820172, Loss: 2.511363983154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9893 , TFLOPS: 67.7359232242369, Tokens per sec: 167857.0794775845, Loss: 2.5092673301696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9894 , TFLOPS: 70.64124250028429, Tokens per sec: 175056.7806909688, Loss: 2.512425661087036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9895 , TFLOPS: 71.47821489512212, Tokens per sec: 177130.8904854976, Loss: 2.489762544631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9896 , TFLOPS: 68.70731460263555, Tokens per sec: 170264.29432085037, Loss: 2.506856679916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9897 , TFLOPS: 68.30740058464033, Tokens per sec: 169273.2633300924, Loss: 2.4901585578918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9898 , TFLOPS: 71.47247009044801, Tokens per sec: 177116.65422667263, Loss: 2.487299680709839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9899 , TFLOPS: 69.52378198203574, Tokens per sec: 172287.59042831077, Loss: 2.493563652038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9900 , TFLOPS: 68.18751748116448, Tokens per sec: 168976.17979346515, Loss: 2.5058095455169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9901 , TFLOPS: 71.49735526751682, Tokens per sec: 177178.3224367777, Loss: 2.5033488273620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9902 , TFLOPS: 70.7004189187976, Tokens per sec: 175203.42637486543, Loss: 2.5006985664367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9903 , TFLOPS: 68.56502468819562, Tokens per sec: 169911.6842383402, Loss: 2.482257604598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9904 , TFLOPS: 70.86404313899746, Tokens per sec: 175608.90521721652, Loss: 2.4996118545532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9905 , TFLOPS: 71.55453037839234, Tokens per sec: 177320.00865988564, Loss: 2.481405019760132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9906 , TFLOPS: 69.73199525575266, Tokens per sec: 172803.56585716686, Loss: 2.504220485687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9907 , TFLOPS: 67.43829273374213, Tokens per sec: 167119.51833543333, Loss: 2.490156888961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9908 , TFLOPS: 71.50844543840186, Tokens per sec: 177205.805101914, Loss: 2.4895927906036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9909 , TFLOPS: 70.74846097447671, Tokens per sec: 175322.4798811076, Loss: 2.49489164352417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9910 , TFLOPS: 67.21026295516806, Tokens per sec: 166554.43542457296, Loss: 2.4916532039642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9911 , TFLOPS: 70.54816703627824, Tokens per sec: 174826.12943805303, Loss: 2.5042195320129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9912 , TFLOPS: 71.45977297333425, Tokens per sec: 177085.18937176184, Loss: 2.477766990661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9913 , TFLOPS: 68.3076159391647, Tokens per sec: 169273.79700232705, Loss: 2.5243430137634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9914 , TFLOPS: 69.07665723476383, Tokens per sec: 171179.56604971396, Loss: 2.456219434738159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9915 , TFLOPS: 71.53517739242021, Tokens per sec: 177272.0498287393, Loss: 2.4891645908355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9916 , TFLOPS: 69.70411046593262, Tokens per sec: 172734.4642762296, Loss: 2.504918336868286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9917 , TFLOPS: 68.74637360551426, Tokens per sec: 170361.0868326882, Loss: 2.4858205318450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9918 , TFLOPS: 70.52435887371, Tokens per sec: 174767.1301885233, Loss: 2.46280574798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9919 , TFLOPS: 69.72904089605916, Tokens per sec: 172796.2446283968, Loss: 2.4908370971679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9920 , TFLOPS: 69.03497190734583, Tokens per sec: 171076.26521635428, Loss: 2.497403621673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9921 , TFLOPS: 70.97411135588584, Tokens per sec: 175881.66638368097, Loss: 2.4983444213867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9922 , TFLOPS: 69.56969578953613, Tokens per sec: 172401.36990112014, Loss: 2.496917486190796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9923 , TFLOPS: 69.65547056402653, Tokens per sec: 172613.92924117882, Loss: 2.4960854053497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9924 , TFLOPS: 69.95048487131488, Tokens per sec: 173345.00719314814, Loss: 2.498291254043579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9925 , TFLOPS: 70.52534665276218, Tokens per sec: 174769.57801382747, Loss: 2.4847044944763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9926 , TFLOPS: 69.81759680259641, Tokens per sec: 173015.69592003452, Loss: 2.4845449924468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9927 , TFLOPS: 68.21433996378201, Tokens per sec: 169042.64886013183, Loss: 2.4863085746765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9928 , TFLOPS: 71.50323673028427, Tokens per sec: 177192.89734941206, Loss: 2.4938740730285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9929 , TFLOPS: 69.67322066069201, Tokens per sec: 172657.9159360504, Loss: 2.4809517860412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9930 , TFLOPS: 67.23715855431155, Tokens per sec: 166621.08568204538, Loss: 2.50555682182312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9931 , TFLOPS: 71.52060497953123, Tokens per sec: 177235.93778431663, Loss: 2.5024971961975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9932 , TFLOPS: 71.50917875358704, Tokens per sec: 177207.62233772423, Loss: 2.4876551628112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9933 , TFLOPS: 68.67021757403153, Tokens per sec: 170172.36379739456, Loss: 2.4980103969573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9934 , TFLOPS: 69.14664824164517, Tokens per sec: 171353.01147491182, Loss: 2.5246872901916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9935 , TFLOPS: 71.56476348687156, Tokens per sec: 177345.36742996713, Loss: 2.4828717708587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9936 , TFLOPS: 68.79050387484529, Tokens per sec: 170470.44650144095, Loss: 2.4955153465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9937 , TFLOPS: 68.98720467078215, Tokens per sec: 170957.89274215396, Loss: 2.4880335330963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9938 , TFLOPS: 71.41757912602775, Tokens per sec: 176980.62837010034, Loss: 2.506359577178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9939 , TFLOPS: 70.47889872091842, Tokens per sec: 174654.47492205654, Loss: 2.4890363216400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9940 , TFLOPS: 67.3119964438662, Tokens per sec: 166806.5422164359, Loss: 2.48927640914917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9941 , TFLOPS: 70.64927620288138, Tokens per sec: 175076.6890909898, Loss: 2.5133256912231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9942 , TFLOPS: 70.75287305856841, Tokens per sec: 175333.41351151798, Loss: 2.487403631210327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9943 , TFLOPS: 70.694165088406, Tokens per sec: 175187.92869423874, Loss: 2.50344181060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9944 , TFLOPS: 69.01599233768624, Tokens per sec: 171029.23175196562, Loss: 2.4836506843566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9945 , TFLOPS: 71.51635353809108, Tokens per sec: 177225.40224409298, Loss: 2.4919002056121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9946 , TFLOPS: 69.72256064146812, Tokens per sec: 172780.18584366122, Loss: 2.5033087730407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9947 , TFLOPS: 67.62148020253503, Tokens per sec: 167573.4770628677, Loss: 2.482010841369629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9948 , TFLOPS: 70.35914245850512, Tokens per sec: 174357.70571155503, Loss: 2.4861154556274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9949 , TFLOPS: 70.65776830645135, Tokens per sec: 175097.7334591186, Loss: 2.5114994049072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9950 , TFLOPS: 68.11219499246435, Tokens per sec: 168789.52236900918, Loss: 2.4908957481384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9951 , TFLOPS: 69.84052594830067, Tokens per sec: 173072.51686894355, Loss: 2.500237464904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9952 , TFLOPS: 71.53187139912202, Tokens per sec: 177263.8572131612, Loss: 2.5091171264648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9953 , TFLOPS: 69.5994847577, Tokens per sec: 172475.19024575595, Loss: 2.4903504848480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9954 , TFLOPS: 68.2238098939908, Tokens per sec: 169066.11639038814, Loss: 2.507286310195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9955 , TFLOPS: 70.42440491671691, Tokens per sec: 174519.43327225713, Loss: 2.502122163772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9956 , TFLOPS: 70.72007618393852, Tokens per sec: 175252.13924331113, Loss: 2.5016589164733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9957 , TFLOPS: 68.31392485029147, Tokens per sec: 169289.43117908313, Loss: 2.493278980255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9958 , TFLOPS: 71.4998040085284, Tokens per sec: 177184.39068675548, Loss: 2.4956207275390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9959 , TFLOPS: 69.60326235569325, Tokens per sec: 172484.55154972, Loss: 2.490835666656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9960 , TFLOPS: 67.90776938458683, Tokens per sec: 168282.9331933489, Loss: 2.492680311203003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9961 , TFLOPS: 70.73115640138172, Tokens per sec: 175279.59724272214, Loss: 2.4777939319610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9962 , TFLOPS: 71.44672442423351, Tokens per sec: 177052.85362967383, Loss: 2.4999747276306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9963 , TFLOPS: 69.51615978022002, Tokens per sec: 172268.70177255658, Loss: 2.498011589050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9964 , TFLOPS: 68.01459450311526, Tokens per sec: 168547.65760482065, Loss: 2.4772067070007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9965 , TFLOPS: 71.50945113440824, Tokens per sec: 177208.29732740438, Loss: 2.5182619094848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9966 , TFLOPS: 68.7629675553644, Tokens per sec: 170402.20846839133, Loss: 2.5054726600646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9967 , TFLOPS: 68.21934123857694, Tokens per sec: 169055.04257000954, Loss: 2.4831676483154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9968 , TFLOPS: 71.51757186766156, Tokens per sec: 177228.4213989787, Loss: 2.466871738433838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9969 , TFLOPS: 70.71308669910229, Tokens per sec: 175234.81852993165, Loss: 2.4912219047546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9970 , TFLOPS: 67.93249427447022, Tokens per sec: 168344.20419415145, Loss: 2.4869375228881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9971 , TFLOPS: 69.89932536953943, Tokens per sec: 173218.2283120642, Loss: 2.504849910736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9972 , TFLOPS: 71.52118150177162, Tokens per sec: 177237.3664699375, Loss: 2.509783983230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9973 , TFLOPS: 68.31179262564937, Tokens per sec: 169284.14729153604, Loss: 2.4930732250213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9974 , TFLOPS: 68.16284889173933, Tokens per sec: 168915.0483114002, Loss: 2.5022339820861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9975 , TFLOPS: 71.51027819937651, Tokens per sec: 177210.3468855045, Loss: 2.507395029067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9976 , TFLOPS: 70.6647531974385, Tokens per sec: 175115.04278843405, Loss: 2.5035922527313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9977 , TFLOPS: 66.2302343218616, Tokens per sec: 164125.81651217336, Loss: 2.4867987632751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9978 , TFLOPS: 71.49354860882244, Tokens per sec: 177168.88911719498, Loss: 2.505276918411255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9979 , TFLOPS: 71.48816720886751, Tokens per sec: 177155.5534152965, Loss: 2.5029048919677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9980 , TFLOPS: 68.31157122484598, Tokens per sec: 169283.59863595592, Loss: 2.4976587295532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9981 , TFLOPS: 69.06194575438629, Tokens per sec: 171143.10938073506, Loss: 2.4896786212921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9982 , TFLOPS: 71.49753654660802, Tokens per sec: 177178.77166633523, Loss: 2.5073251724243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9983 , TFLOPS: 69.82877056319178, Tokens per sec: 173043.38572968118, Loss: 2.4966259002685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9984 , TFLOPS: 68.86507114946627, Tokens per sec: 170655.2324222152, Loss: 2.484556198120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9985 , TFLOPS: 70.47499505726906, Tokens per sec: 174644.80121350937, Loss: 2.52486252784729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9986 , TFLOPS: 69.7447797264767, Tokens per sec: 172835.24718394672, Loss: 2.4882025718688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9987 , TFLOPS: 68.95999173678234, Tokens per sec: 170890.4560939503, Loss: 2.5059127807617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9988 , TFLOPS: 69.98105928898418, Tokens per sec: 173420.7739681828, Loss: 2.477301836013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9989 , TFLOPS: 70.65996668010887, Tokens per sec: 175103.18127121282, Loss: 2.500890016555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9990 , TFLOPS: 69.83938900695199, Tokens per sec: 173069.6994030371, Loss: 2.483886480331421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9991 , TFLOPS: 69.05061593400022, Tokens per sec: 171115.03283773563, Loss: 2.4995110034942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9992 , TFLOPS: 70.51184121870827, Tokens per sec: 174736.1100604955, Loss: 2.4880449771881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9993 , TFLOPS: 69.71040009994074, Tokens per sec: 172750.0506821621, Loss: 2.5026724338531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9994 , TFLOPS: 68.1432832117688, Tokens per sec: 168866.56240109567, Loss: 2.5051167011260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9995 , TFLOPS: 71.55360327301901, Tokens per sec: 177317.71119064145, Loss: 2.5114047527313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9996 , TFLOPS: 69.56349965811955, Tokens per sec: 172386.01520490967, Loss: 2.4816765785217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9997 , TFLOPS: 68.06932237457447, Tokens per sec: 168683.27929906928, Loss: 2.5087006092071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9998 , TFLOPS: 71.50271575345928, Tokens per sec: 177191.6063114494, Loss: 2.5133919715881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9999 , TFLOPS: 70.55053764225782, Tokens per sec: 174832.00406081424, Loss: 2.4874207973480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10000 , TFLOPS: 68.01465027505945, Tokens per sec: 168547.79581384215, Loss: 2.5006954669952393 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/optimizer.pt +[rank0]:[2024-08-26 23:07:25,019] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006400446000043303, 'preprocessing_with_comm': 0.0012723980034934357, 'state_converting': 0.992249143004301, : 1.001267869010917}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008000 + eval ppl=9.919724464416504, eval loss=2.294525146484375 +------------------------------------------------------------------ +iteration: 10001 , TFLOPS: 64.83903426319579, Tokens per sec: 160678.2695134616, Loss: 2.4925625324249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10002 , TFLOPS: 71.50076072642756, Tokens per sec: 177186.7615391008, Loss: 2.4907236099243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10003 , TFLOPS: 68.46631647216606, Tokens per sec: 169667.07440541135, Loss: 2.5165886878967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10004 , TFLOPS: 68.82999296309235, Tokens per sec: 170568.3048121994, Loss: 2.5074148178100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10005 , TFLOPS: 71.5254117755671, Tokens per sec: 177247.8495544058, Loss: 2.468194007873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10006 , TFLOPS: 70.63022809499208, Tokens per sec: 175029.48578131778, Loss: 2.494117498397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10007 , TFLOPS: 66.56156838787508, Tokens per sec: 164946.89882721525, Loss: 2.4821109771728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10008 , TFLOPS: 71.53039097025126, Tokens per sec: 177260.18854733673, Loss: 2.497130870819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10009 , TFLOPS: 70.64977377739743, Tokens per sec: 175077.9221354529, Loss: 2.5044913291931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10010 , TFLOPS: 68.42389896683439, Tokens per sec: 169561.95915452548, Loss: 2.5085573196411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10011 , TFLOPS: 69.81553494495297, Tokens per sec: 173010.5864096617, Loss: 2.498574733734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10012 , TFLOPS: 71.44163784150216, Tokens per sec: 177040.24851733222, Loss: 2.4708058834075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10013 , TFLOPS: 69.77537933825352, Tokens per sec: 172911.0763927552, Loss: 2.484896659851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10014 , TFLOPS: 67.25804901489212, Tokens per sec: 166672.85454463825, Loss: 2.4869790077209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10015 , TFLOPS: 71.43422444617825, Tokens per sec: 177021.87730706664, Loss: 2.490478515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10016 , TFLOPS: 70.57377690587042, Tokens per sec: 174889.59351606248, Loss: 2.502887487411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10017 , TFLOPS: 67.40683021536003, Tokens per sec: 167041.550749, Loss: 2.507244825363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10018 , TFLOPS: 70.40339860061435, Tokens per sec: 174467.3773069182, Loss: 2.497694492340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10019 , TFLOPS: 71.54805654811881, Tokens per sec: 177303.96579531656, Loss: 2.4932332038879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10020 , TFLOPS: 68.12476647934383, Tokens per sec: 168820.67589836012, Loss: 2.498701333999634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10021 , TFLOPS: 69.78858518998555, Tokens per sec: 172943.80194809186, Loss: 2.5078279972076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10022 , TFLOPS: 70.40211190427972, Tokens per sec: 174464.18873165973, Loss: 2.474050521850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10023 , TFLOPS: 69.80104762243221, Tokens per sec: 172974.6852285449, Loss: 2.4906864166259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10024 , TFLOPS: 68.16325596427797, Tokens per sec: 168916.05708199367, Loss: 2.4807779788970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10025 , TFLOPS: 70.42592132768627, Tokens per sec: 174523.19110568563, Loss: 2.5010690689086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10026 , TFLOPS: 70.73694898683107, Tokens per sec: 175293.95190757155, Loss: 2.512328624725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10027 , TFLOPS: 68.2148621998259, Tokens per sec: 169043.94301857761, Loss: 2.4742014408111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10028 , TFLOPS: 70.85198756632725, Tokens per sec: 175579.0301801088, Loss: 2.5094687938690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10029 , TFLOPS: 70.45366834456796, Tokens per sec: 174591.95127010468, Loss: 2.478562831878662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10030 , TFLOPS: 69.00203951639726, Tokens per sec: 170994.65512378135, Loss: 2.4871819019317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10031 , TFLOPS: 69.207563704132, Tokens per sec: 171503.96670134636, Loss: 2.4922382831573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10032 , TFLOPS: 70.43097516578995, Tokens per sec: 174535.71507323277, Loss: 2.5051939487457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10033 , TFLOPS: 69.92339778055464, Tokens per sec: 173277.88239835485, Loss: 2.486207962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10034 , TFLOPS: 67.98382978700336, Tokens per sec: 168471.4192492821, Loss: 2.498143434524536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10035 , TFLOPS: 71.51822028594871, Tokens per sec: 177230.02825092347, Loss: 2.4799304008483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10036 , TFLOPS: 69.62023151745431, Tokens per sec: 172526.6030018714, Loss: 2.496124029159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10037 , TFLOPS: 67.32321853961838, Tokens per sec: 166834.35180592572, Loss: 2.507077932357788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10038 , TFLOPS: 71.54377168858342, Tokens per sec: 177293.34744137165, Loss: 2.497896671295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10039 , TFLOPS: 71.54946047866648, Tokens per sec: 177307.4448898694, Loss: 2.5006580352783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10040 , TFLOPS: 68.12259900264716, Tokens per sec: 168815.3046523385, Loss: 2.4984853267669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10041 , TFLOPS: 69.13507419892974, Tokens per sec: 171324.32972207692, Loss: 2.4851622581481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10042 , TFLOPS: 71.51793751085343, Tokens per sec: 177229.3275030871, Loss: 2.4802327156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10043 , TFLOPS: 70.60132119410554, Tokens per sec: 174957.85129656293, Loss: 2.5031771659851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10044 , TFLOPS: 67.32583541685102, Tokens per sec: 166840.83671598102, Loss: 2.49015212059021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10045 , TFLOPS: 71.49566698756452, Tokens per sec: 177174.13869307653, Loss: 2.495046615600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10046 , TFLOPS: 70.74272604542867, Tokens per sec: 175308.26809517198, Loss: 2.5067527294158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10047 , TFLOPS: 68.02718985013551, Tokens per sec: 168578.87026222885, Loss: 2.47409725189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10048 , TFLOPS: 69.99138909144986, Tokens per sec: 173446.3723566144, Loss: 2.4982802867889404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10049 , TFLOPS: 71.53629755899001, Tokens per sec: 177274.82572489584, Loss: 2.493894100189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10050 , TFLOPS: 69.64767772153294, Tokens per sec: 172594.61771902724, Loss: 2.5195364952087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10051 , TFLOPS: 68.13322364839253, Tokens per sec: 168841.63369489685, Loss: 2.4924380779266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10052 , TFLOPS: 71.5333708809853, Tokens per sec: 177267.57309439994, Loss: 2.4721426963806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10053 , TFLOPS: 69.88540693409544, Tokens per sec: 173183.73689579175, Loss: 2.4811630249023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10054 , TFLOPS: 67.95313921977761, Tokens per sec: 168395.3646428516, Loss: 2.5213160514831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10055 , TFLOPS: 70.53897937201509, Tokens per sec: 174803.3613939042, Loss: 2.495274543762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10056 , TFLOPS: 70.56600182769157, Tokens per sec: 174870.32601583973, Loss: 2.4891834259033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10057 , TFLOPS: 68.12364600461089, Tokens per sec: 168817.8992385414, Loss: 2.502152919769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10058 , TFLOPS: 70.77460464443153, Tokens per sec: 175387.26677521862, Loss: 2.4889769554138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10059 , TFLOPS: 70.37495056877373, Tokens per sec: 174396.87995020775, Loss: 2.500049114227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10060 , TFLOPS: 68.95934734406671, Tokens per sec: 170888.85921781522, Loss: 2.505819320678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10061 , TFLOPS: 68.78785455406504, Tokens per sec: 170463.8811927007, Loss: 2.4754109382629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10062 , TFLOPS: 70.53447384808241, Tokens per sec: 174792.19620927455, Loss: 2.486201286315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10063 , TFLOPS: 70.61454453796775, Tokens per sec: 174990.62019932506, Loss: 2.490365743637085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10064 , TFLOPS: 67.32353363145926, Tokens per sec: 166835.13263821762, Loss: 2.4881134033203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10065 , TFLOPS: 71.52197628675462, Tokens per sec: 177239.33603467818, Loss: 2.4973561763763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10066 , TFLOPS: 70.46569170609605, Tokens per sec: 174621.74648445394, Loss: 2.4944257736206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10067 , TFLOPS: 68.23749215599254, Tokens per sec: 169100.02254285399, Loss: 2.4827864170074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10068 , TFLOPS: 69.75650032166322, Tokens per sec: 172864.29210994908, Loss: 2.4803802967071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10069 , TFLOPS: 70.61040713658745, Tokens per sec: 174980.36726293233, Loss: 2.484304904937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10070 , TFLOPS: 69.64557668961713, Tokens per sec: 172589.41113049205, Loss: 2.5079033374786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10071 , TFLOPS: 68.81156372073113, Tokens per sec: 170522.63511947962, Loss: 2.4920270442962646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10072 , TFLOPS: 71.49615329673506, Tokens per sec: 177175.34382077248, Loss: 2.4912729263305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10073 , TFLOPS: 69.49045909852224, Tokens per sec: 172205.01265214567, Loss: 2.4929757118225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10074 , TFLOPS: 67.83697270538917, Tokens per sec: 168107.49122339912, Loss: 2.4927284717559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10075 , TFLOPS: 71.51564336667711, Tokens per sec: 177223.64236109736, Loss: 2.5132744312286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10076 , TFLOPS: 70.6470555018063, Tokens per sec: 175071.18594909535, Loss: 2.4837889671325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10077 , TFLOPS: 68.04487799588624, Tokens per sec: 168622.70343590924, Loss: 2.50921630859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10078 , TFLOPS: 69.9715497835161, Tokens per sec: 173397.20836607437, Loss: 2.478682279586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10079 , TFLOPS: 71.4811164469567, Tokens per sec: 177138.08085057596, Loss: 2.526299476623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10080 , TFLOPS: 69.14556037435925, Tokens per sec: 171350.3156199388, Loss: 2.4971721172332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10081 , TFLOPS: 68.43914270719668, Tokens per sec: 169599.7348223799, Loss: 2.5033366680145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10082 , TFLOPS: 71.55520114914438, Tokens per sec: 177321.6709037007, Loss: 2.500518321990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10083 , TFLOPS: 70.61772088012643, Tokens per sec: 174998.49152509726, Loss: 2.5041263103485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10084 , TFLOPS: 67.08585300843701, Tokens per sec: 166246.13387168644, Loss: 2.516207695007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10085 , TFLOPS: 71.4645777307838, Tokens per sec: 177097.09609000955, Loss: 2.4753432273864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10086 , TFLOPS: 71.48941671366582, Tokens per sec: 177158.64982583103, Loss: 2.486544132232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10087 , TFLOPS: 69.0588624952346, Tokens per sec: 171135.46872490813, Loss: 2.4777650833129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10088 , TFLOPS: 68.08583260841482, Tokens per sec: 168724.19347727808, Loss: 2.499544620513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10089 , TFLOPS: 71.5197373395579, Tokens per sec: 177233.7876768843, Loss: 2.5198092460632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10090 , TFLOPS: 69.66476535912459, Tokens per sec: 172636.962767338, Loss: 2.526116371154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10091 , TFLOPS: 68.85298579044344, Tokens per sec: 170625.28357124442, Loss: 2.5013155937194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10092 , TFLOPS: 70.56351080789507, Tokens per sec: 174864.15299437553, Loss: 2.499758720397949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10093 , TFLOPS: 70.69590591753256, Tokens per sec: 175192.2426605825, Loss: 2.4833810329437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10094 , TFLOPS: 68.35366681230417, Tokens per sec: 169387.91613888912, Loss: 2.503655433654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10095 , TFLOPS: 70.82670984694838, Tokens per sec: 175516.38920691726, Loss: 2.4987382888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10096 , TFLOPS: 69.55197009800911, Tokens per sec: 172357.4436848701, Loss: 2.5018539428710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10097 , TFLOPS: 69.59774071780176, Tokens per sec: 172470.86832276627, Loss: 2.5013303756713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10098 , TFLOPS: 69.36542840168865, Tokens per sec: 171895.17281212375, Loss: 2.4948415756225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10099 , TFLOPS: 70.54217244532597, Tokens per sec: 174811.2741813146, Loss: 2.4887008666992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10100 , TFLOPS: 69.81665852398478, Tokens per sec: 173013.37076227498, Loss: 2.4909396171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10101 , TFLOPS: 68.28132655622382, Tokens per sec: 169208.64901538493, Loss: 2.491833209991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10102 , TFLOPS: 71.5339613625323, Tokens per sec: 177269.0363727203, Loss: 2.521054267883301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10103 , TFLOPS: 69.53072078844185, Tokens per sec: 172304.78555495775, Loss: 2.478543758392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10104 , TFLOPS: 67.4536488941876, Tokens per sec: 167157.57259263788, Loss: 2.50075626373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10105 , TFLOPS: 71.5230604440626, Tokens per sec: 177242.02269591708, Loss: 2.4884495735168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10106 , TFLOPS: 70.5143157909987, Tokens per sec: 174742.24232322784, Loss: 2.4919700622558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10107 , TFLOPS: 69.02225546233872, Tokens per sec: 171044.75246479458, Loss: 2.498913288116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10108 , TFLOPS: 68.76445264681243, Tokens per sec: 170405.88868859588, Loss: 2.495948314666748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10109 , TFLOPS: 71.50935904961334, Tokens per sec: 177208.06913113865, Loss: 2.4813754558563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10110 , TFLOPS: 69.76183567457828, Tokens per sec: 172877.51370221027, Loss: 2.4909329414367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10111 , TFLOPS: 68.21881908348077, Tokens per sec: 169053.7486121611, Loss: 2.503084182739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10112 , TFLOPS: 71.55416719446958, Tokens per sec: 177319.10865011706, Loss: 2.507138729095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10113 , TFLOPS: 70.56475344148572, Tokens per sec: 174867.23237730053, Loss: 2.479494333267212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10114 , TFLOPS: 67.83372112344567, Tokens per sec: 168099.43344515178, Loss: 2.4907567501068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10115 , TFLOPS: 69.9537063170754, Tokens per sec: 173352.99029061422, Loss: 2.516050338745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10116 , TFLOPS: 71.4910863757911, Tokens per sec: 177162.78743250592, Loss: 2.5033655166625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10117 , TFLOPS: 70.06413586555229, Tokens per sec: 173626.64687655895, Loss: 2.493089199066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10118 , TFLOPS: 68.2578875510212, Tokens per sec: 169150.56457847328, Loss: 2.48870849609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10119 , TFLOPS: 71.48027221227859, Tokens per sec: 177135.98874404348, Loss: 2.5135366916656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10120 , TFLOPS: 69.80808441143127, Tokens per sec: 172992.12316685088, Loss: 2.503307342529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10121 , TFLOPS: 67.79806400367066, Tokens per sec: 168011.07117439332, Loss: 2.4854142665863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10122 , TFLOPS: 71.48923945263623, Tokens per sec: 177158.2105534728, Loss: 2.493046283721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10123 , TFLOPS: 70.71117278105179, Tokens per sec: 175230.075630167, Loss: 2.5138094425201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10124 , TFLOPS: 68.23184251681373, Tokens per sec: 169086.0221146098, Loss: 2.4951720237731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10125 , TFLOPS: 69.96949658649055, Tokens per sec: 173392.12031766673, Loss: 2.4808149337768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10126 , TFLOPS: 71.52823427531821, Tokens per sec: 177254.84401971308, Loss: 2.5029852390289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10127 , TFLOPS: 68.75521729598746, Tokens per sec: 170383.00247189373, Loss: 2.496875286102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10128 , TFLOPS: 68.89857583019015, Tokens per sec: 170738.26071189423, Loss: 2.4905002117156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10129 , TFLOPS: 70.42048888161261, Tokens per sec: 174509.72890588178, Loss: 2.510861873626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10130 , TFLOPS: 70.7011556764883, Tokens per sec: 175205.25214158354, Loss: 2.4978220462799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10131 , TFLOPS: 67.5724154943925, Tokens per sec: 167451.88931116668, Loss: 2.516404628753662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10132 , TFLOPS: 71.56086482901165, Tokens per sec: 177335.70612631764, Loss: 2.487222909927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10133 , TFLOPS: 70.50164448430402, Tokens per sec: 174710.84142937875, Loss: 2.498527765274048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10134 , TFLOPS: 68.25166064119738, Tokens per sec: 169135.1336099791, Loss: 2.5248820781707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10135 , TFLOPS: 69.7165667598331, Tokens per sec: 172765.33234468906, Loss: 2.491363763809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10136 , TFLOPS: 70.43800176555494, Tokens per sec: 174553.12776149428, Loss: 2.506826877593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10137 , TFLOPS: 69.87672337056961, Tokens per sec: 173162.21806880124, Loss: 2.508354663848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10138 , TFLOPS: 68.23085701009066, Tokens per sec: 169083.57992038748, Loss: 2.4837145805358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10139 , TFLOPS: 71.47252387174055, Tokens per sec: 177116.78750264004, Loss: 2.5010743141174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10140 , TFLOPS: 68.69824548999657, Tokens per sec: 170241.82006068577, Loss: 2.497157335281372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10141 , TFLOPS: 68.05276257070706, Tokens per sec: 168642.24228087286, Loss: 2.473290205001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10142 , TFLOPS: 71.5180481281116, Tokens per sec: 177229.60162483837, Loss: 2.49755859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10143 , TFLOPS: 69.67783984164804, Tokens per sec: 172669.36277530427, Loss: 2.4857842922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10144 , TFLOPS: 69.03497448186543, Tokens per sec: 171076.27159629745, Loss: 2.5124619007110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10145 , TFLOPS: 70.10274935749474, Tokens per sec: 173722.33536321964, Loss: 2.486746311187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10146 , TFLOPS: 71.53148869093474, Tokens per sec: 177262.9088201143, Loss: 2.5168521404266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10147 , TFLOPS: 68.26959582037775, Tokens per sec: 169179.57895971113, Loss: 2.4774487018585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10148 , TFLOPS: 68.34969112926142, Tokens per sec: 169378.06395835153, Loss: 2.489886999130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10149 , TFLOPS: 71.51872811788328, Tokens per sec: 177231.28671439964, Loss: 2.5078048706054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10150 , TFLOPS: 70.71698690752307, Tokens per sec: 175244.48367604174, Loss: 2.4904258251190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10151 , TFLOPS: 67.05062212231105, Tokens per sec: 166158.8278548638, Loss: 2.50351881980896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10152 , TFLOPS: 71.49800292331834, Tokens per sec: 177179.92739919905, Loss: 2.4968957901000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10153 , TFLOPS: 71.494653781711, Tokens per sec: 177171.62785735764, Loss: 2.5288853645324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10154 , TFLOPS: 67.9246855846902, Tokens per sec: 168324.85340067834, Loss: 2.4710984230041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10155 , TFLOPS: 69.55210588384378, Tokens per sec: 172357.78017712646, Loss: 2.491304636001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10156 , TFLOPS: 71.53814931151882, Tokens per sec: 177279.4145716496, Loss: 2.4857017993927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10157 , TFLOPS: 69.89965735835607, Tokens per sec: 173219.05101692342, Loss: 2.4756650924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10158 , TFLOPS: 67.27423814595268, Tokens per sec: 166712.97299478634, Loss: 2.5117762088775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10159 , TFLOPS: 71.56592609078507, Tokens per sec: 177348.2484905362, Loss: 2.4936726093292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10160 , TFLOPS: 70.58790566397448, Tokens per sec: 174924.60613505554, Loss: 2.494107246398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10161 , TFLOPS: 67.79090126848176, Tokens per sec: 167993.3211275548, Loss: 2.5107173919677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10162 , TFLOPS: 69.8455553360981, Tokens per sec: 173084.9802459384, Loss: 2.476320743560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10163 , TFLOPS: 70.72181037283501, Tokens per sec: 175256.43675443335, Loss: 2.493624448776245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10164 , TFLOPS: 69.69249224478598, Tokens per sec: 172705.67304437555, Loss: 2.483156681060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10165 , TFLOPS: 69.15008522306465, Tokens per sec: 171361.52869348406, Loss: 2.491572380065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10166 , TFLOPS: 70.51232437846419, Tokens per sec: 174737.3073835951, Loss: 2.5049102306365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10167 , TFLOPS: 69.67873412546521, Tokens per sec: 172671.5789091167, Loss: 2.5103423595428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10168 , TFLOPS: 68.59390274587976, Tokens per sec: 169983.24724646215, Loss: 2.504391670227051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10169 , TFLOPS: 71.52086433656062, Tokens per sec: 177236.58049960597, Loss: 2.492164373397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10170 , TFLOPS: 69.6969962912896, Tokens per sec: 172716.83456777316, Loss: 2.5043702125549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10171 , TFLOPS: 67.83209823473666, Tokens per sec: 168095.4117481549, Loss: 2.5055248737335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10172 , TFLOPS: 71.5214285150973, Tokens per sec: 177237.97859616927, Loss: 2.4991674423217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10173 , TFLOPS: 70.54207702621287, Tokens per sec: 174811.03772224105, Loss: 2.483320474624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10174 , TFLOPS: 68.8518587291742, Tokens per sec: 170622.49058926318, Loss: 2.502385139465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10175 , TFLOPS: 69.32208035990963, Tokens per sec: 171787.75158941434, Loss: 2.4815661907196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10176 , TFLOPS: 71.54283237136693, Tokens per sec: 177291.0197098348, Loss: 2.5121612548828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10177 , TFLOPS: 69.6829896430072, Tokens per sec: 172682.12455036974, Loss: 2.5011396408081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10178 , TFLOPS: 67.45165955152535, Tokens per sec: 167152.64278237667, Loss: 2.482098340988159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10179 , TFLOPS: 71.50008008371509, Tokens per sec: 177185.07483148028, Loss: 2.498884439468384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10180 , TFLOPS: 70.52928051297384, Tokens per sec: 174779.32655278244, Loss: 2.490607500076294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10181 , TFLOPS: 68.86429807567649, Tokens per sec: 170653.31666020298, Loss: 2.500929832458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10182 , TFLOPS: 70.15100472398785, Tokens per sec: 173841.91747715703, Loss: 2.4709038734436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10183 , TFLOPS: 71.48659121475997, Tokens per sec: 177151.64792828792, Loss: 2.4961705207824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10184 , TFLOPS: 69.25047851801726, Tokens per sec: 171610.31433761102, Loss: 2.495253562927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10185 , TFLOPS: 68.27687409305673, Tokens per sec: 169197.61532117758, Loss: 2.5020077228546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10186 , TFLOPS: 71.53241312594186, Tokens per sec: 177265.19967189687, Loss: 2.4803810119628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10187 , TFLOPS: 69.64935742345233, Tokens per sec: 172598.7802054179, Loss: 2.4990620613098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10188 , TFLOPS: 67.29771542070642, Tokens per sec: 166771.1522678614, Loss: 2.494364023208618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10189 , TFLOPS: 71.52600068595441, Tokens per sec: 177249.3089392194, Loss: 2.5152394771575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10190 , TFLOPS: 70.66419693573228, Tokens per sec: 175113.66431064872, Loss: 2.5095572471618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10191 , TFLOPS: 68.87734951629713, Tokens per sec: 170685.6595677958, Loss: 2.5155816078186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10192 , TFLOPS: 69.00349843886526, Tokens per sec: 170998.27049437066, Loss: 2.503659248352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10193 , TFLOPS: 71.50863947461306, Tokens per sec: 177206.28594502024, Loss: 2.493448495864868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10194 , TFLOPS: 68.69174663005053, Tokens per sec: 170225.71516983095, Loss: 2.511476516723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10195 , TFLOPS: 67.97634577099898, Tokens per sec: 168452.87303319082, Loss: 2.4740920066833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10196 , TFLOPS: 71.50184443385544, Tokens per sec: 177189.4470854873, Loss: 2.515125274658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10197 , TFLOPS: 70.67570131164499, Tokens per sec: 175142.17342147205, Loss: 2.509917736053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10198 , TFLOPS: 67.27337182065098, Tokens per sec: 166710.8261452551, Loss: 2.5050454139709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10199 , TFLOPS: 70.58056056079657, Tokens per sec: 174906.40416025076, Loss: 2.487440586090088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10200 , TFLOPS: 71.50520724791527, Tokens per sec: 177197.7805091721, Loss: 2.5044310092926025 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/optimizer.pt +[rank0]:[2024-08-26 23:32:50,051] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006371897994540632, 'preprocessing_with_comm': 0.0012670650030486286, 'state_converting': 1.0006744199927198, : 1.0096833089919528}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200, took 4.59s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008200 + eval ppl=10.516378402709961, eval loss=2.352933883666992 +------------------------------------------------------------------ +iteration: 10201 , TFLOPS: 64.39056222600742, Tokens per sec: 159566.9064020356, Loss: 2.496950626373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10202 , TFLOPS: 68.48619362852936, Tokens per sec: 169716.33218852914, Loss: 2.4834694862365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10203 , TFLOPS: 71.50358606099743, Tokens per sec: 177193.76302940122, Loss: 2.481595277786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10204 , TFLOPS: 69.84256892964231, Tokens per sec: 173077.57960176113, Loss: 2.498408317565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10205 , TFLOPS: 68.11465102123218, Tokens per sec: 168795.6086788497, Loss: 2.5026373863220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10206 , TFLOPS: 70.41978763486371, Tokens per sec: 174507.9911391893, Loss: 2.495769500732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10207 , TFLOPS: 70.6106105404081, Tokens per sec: 174980.87132000667, Loss: 2.491800546646118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10208 , TFLOPS: 68.6152004797542, Tokens per sec: 170036.02537714207, Loss: 2.5152084827423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10209 , TFLOPS: 70.90138842544616, Tokens per sec: 175701.45095095932, Loss: 2.490063190460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10210 , TFLOPS: 69.48005683258108, Tokens per sec: 172179.23469123928, Loss: 2.5107972621917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10211 , TFLOPS: 69.04965389470063, Tokens per sec: 171112.64879837385, Loss: 2.5239553451538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10212 , TFLOPS: 69.78641610457142, Tokens per sec: 172938.4267154911, Loss: 2.4938323497772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10213 , TFLOPS: 71.46592053498809, Tokens per sec: 177100.42370674986, Loss: 2.5000686645507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10214 , TFLOPS: 68.68618736665468, Tokens per sec: 170211.93870273264, Loss: 2.4939308166503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10215 , TFLOPS: 69.75245248831403, Tokens per sec: 172854.26113300415, Loss: 2.4653708934783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10216 , TFLOPS: 71.52306628013255, Tokens per sec: 177242.03715834132, Loss: 2.5143356323242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10217 , TFLOPS: 69.492170675384, Tokens per sec: 172209.25412815486, Loss: 2.479681968688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10218 , TFLOPS: 68.00688820467624, Tokens per sec: 168528.5605483699, Loss: 2.4790875911712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10219 , TFLOPS: 71.52908926847077, Tokens per sec: 177256.9627869304, Loss: 2.5062367916107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10220 , TFLOPS: 71.51856180249605, Tokens per sec: 177230.8745665489, Loss: 2.4904863834381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10221 , TFLOPS: 68.00448544127057, Tokens per sec: 168522.60623596443, Loss: 2.5032057762145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10222 , TFLOPS: 69.15969997023775, Tokens per sec: 171385.35509613084, Loss: 2.496737003326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10223 , TFLOPS: 71.5070130808624, Tokens per sec: 177202.25556214398, Loss: 2.4842586517333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10224 , TFLOPS: 69.72058569434073, Tokens per sec: 172775.29170712113, Loss: 2.4966113567352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10225 , TFLOPS: 67.14638602497911, Tokens per sec: 166396.14135494, Loss: 2.486785411834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10226 , TFLOPS: 71.52194353268739, Tokens per sec: 177239.2548664914, Loss: 2.4804847240448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10227 , TFLOPS: 70.69972686108467, Tokens per sec: 175201.71137961635, Loss: 2.4980087280273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10228 , TFLOPS: 68.85100437412004, Tokens per sec: 170620.373403324, Loss: 2.493640184402466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10229 , TFLOPS: 69.5426489050964, Tokens per sec: 172334.3447420182, Loss: 2.4768500328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10230 , TFLOPS: 71.53406762202647, Tokens per sec: 177269.29969545233, Loss: 2.488076686859131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10231 , TFLOPS: 68.23724144827375, Tokens per sec: 169099.40126150913, Loss: 2.475131034851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10232 , TFLOPS: 68.76421949609758, Tokens per sec: 170405.31091544082, Loss: 2.4744691848754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10233 , TFLOPS: 71.5103988478017, Tokens per sec: 177210.6458655926, Loss: 2.4999823570251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10234 , TFLOPS: 69.42021627938935, Tokens per sec: 172030.94320844905, Loss: 2.489891529083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10235 , TFLOPS: 69.71479971776043, Tokens per sec: 172760.95341977722, Loss: 2.5072975158691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10236 , TFLOPS: 70.47722421507916, Tokens per sec: 174650.3253121248, Loss: 2.491729974746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10237 , TFLOPS: 70.52717731809685, Tokens per sec: 174774.1146041912, Loss: 2.5005805492401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10238 , TFLOPS: 68.93360717793976, Tokens per sec: 170825.0722506393, Loss: 2.4590466022491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10239 , TFLOPS: 70.70179439802524, Tokens per sec: 175206.83496391308, Loss: 2.4792137145996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10240 , TFLOPS: 70.41274781196105, Tokens per sec: 174490.545682536, Loss: 2.4905972480773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10241 , TFLOPS: 69.03374748017423, Tokens per sec: 171073.23095094127, Loss: 2.5149331092834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10242 , TFLOPS: 68.98009972632426, Tokens per sec: 170940.28590711302, Loss: 2.487149477005005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10243 , TFLOPS: 71.5255014571343, Tokens per sec: 177248.07179520855, Loss: 2.4991960525512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10244 , TFLOPS: 68.0219180012578, Tokens per sec: 168565.80603996778, Loss: 2.463059902191162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10245 , TFLOPS: 68.05655164199722, Tokens per sec: 168651.6320169305, Loss: 2.5096025466918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10246 , TFLOPS: 71.54810783276335, Tokens per sec: 177304.0928843152, Loss: 2.4675822257995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10247 , TFLOPS: 70.57161524514083, Tokens per sec: 174884.23668264798, Loss: 2.4828953742980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10248 , TFLOPS: 68.3213981757243, Tokens per sec: 169307.95090276044, Loss: 2.4970879554748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10249 , TFLOPS: 70.69780888908629, Tokens per sec: 175196.9584337223, Loss: 2.4840145111083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10250 , TFLOPS: 71.49643922100633, Tokens per sec: 177176.05237261386, Loss: 2.497885227203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10251 , TFLOPS: 67.85635906459214, Tokens per sec: 168155.5327570878, Loss: 2.4937660694122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10252 , TFLOPS: 69.16860886988007, Tokens per sec: 171407.43233084082, Loss: 2.4951844215393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10253 , TFLOPS: 71.50241620176038, Tokens per sec: 177190.8639893412, Loss: 2.4888272285461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10254 , TFLOPS: 68.67438996037323, Tokens per sec: 170182.7034303756, Loss: 2.501729726791382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10255 , TFLOPS: 69.18600924409887, Tokens per sec: 171450.55237496443, Loss: 2.528264045715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10256 , TFLOPS: 71.5030850011574, Tokens per sec: 177192.52134792134, Loss: 2.511202573776245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10257 , TFLOPS: 70.6348137210528, Tokens per sec: 175040.84946784464, Loss: 2.4880402088165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10258 , TFLOPS: 67.88442166652864, Tokens per sec: 168225.07497603726, Loss: 2.4992434978485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10259 , TFLOPS: 70.51783730541624, Tokens per sec: 174750.96902388133, Loss: 2.502826452255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10260 , TFLOPS: 71.47635026399809, Tokens per sec: 177126.26972416614, Loss: 2.4801952838897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10261 , TFLOPS: 68.93728267682997, Tokens per sec: 170834.18054177883, Loss: 2.484121561050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10262 , TFLOPS: 68.25414684442461, Tokens per sec: 169141.29469545407, Loss: 2.4835457801818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10263 , TFLOPS: 71.4410895452277, Tokens per sec: 177038.88977876428, Loss: 2.5170810222625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10264 , TFLOPS: 70.59819920551116, Tokens per sec: 174950.11466491065, Loss: 2.4732184410095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10265 , TFLOPS: 67.40174081036602, Tokens per sec: 167028.93864278053, Loss: 2.517509937286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10266 , TFLOPS: 70.37562559038096, Tokens per sec: 174398.55272811025, Loss: 2.51180362701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10267 , TFLOPS: 71.4351177908505, Tokens per sec: 177024.09111357454, Loss: 2.489994525909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10268 , TFLOPS: 68.76500746375623, Tokens per sec: 170407.26358610074, Loss: 2.479062080383301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10269 , TFLOPS: 69.01809694256939, Tokens per sec: 171034.4471947074, Loss: 2.4838740825653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10270 , TFLOPS: 71.47145025735767, Tokens per sec: 177114.1269679309, Loss: 2.496565341949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10271 , TFLOPS: 69.73272362338075, Tokens per sec: 172805.37083238558, Loss: 2.477060556411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10272 , TFLOPS: 68.23083896643463, Tokens per sec: 169083.53520622017, Loss: 2.4898080825805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10273 , TFLOPS: 70.54760304818879, Tokens per sec: 174824.73181343896, Loss: 2.4985833168029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10274 , TFLOPS: 70.72478643252599, Tokens per sec: 175263.8117581876, Loss: 2.512474775314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10275 , TFLOPS: 69.58475440484712, Tokens per sec: 172438.6868087042, Loss: 2.492846727371216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10276 , TFLOPS: 70.92809173709942, Tokens per sec: 175767.6246988487, Loss: 2.476362466812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10277 , TFLOPS: 70.46154028565886, Tokens per sec: 174611.45880729382, Loss: 2.502232074737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10278 , TFLOPS: 68.37261113535027, Tokens per sec: 169434.8622582868, Loss: 2.503922700881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10279 , TFLOPS: 70.35192471988101, Tokens per sec: 174339.81936014496, Loss: 2.4928200244903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10280 , TFLOPS: 70.5219591779311, Tokens per sec: 174761.1834780349, Loss: 2.500732421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10281 , TFLOPS: 69.05986985018976, Tokens per sec: 171137.96506145634, Loss: 2.488057851791382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10282 , TFLOPS: 68.8950953903339, Tokens per sec: 170729.63579852835, Loss: 2.50622820854187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10283 , TFLOPS: 71.50977565497797, Tokens per sec: 177209.1015251251, Loss: 2.5167300701141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10284 , TFLOPS: 69.60905641905215, Tokens per sec: 172498.90987699214, Loss: 2.521279811859131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10285 , TFLOPS: 67.05108087880014, Tokens per sec: 166159.96470398, Loss: 2.492685556411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10286 , TFLOPS: 71.50771616568926, Tokens per sec: 177203.99788379614, Loss: 2.4926860332489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10287 , TFLOPS: 71.48337230857403, Tokens per sec: 177143.6711241674, Loss: 2.4887044429779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10288 , TFLOPS: 68.05131444559106, Tokens per sec: 168638.65366731127, Loss: 2.5126845836639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10289 , TFLOPS: 69.04971689745948, Tokens per sec: 171112.80492615062, Loss: 2.4933505058288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10290 , TFLOPS: 71.51790937839515, Tokens per sec: 177229.25778775645, Loss: 2.494117498397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10291 , TFLOPS: 69.57107618614263, Tokens per sec: 172404.79067597433, Loss: 2.4767184257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10292 , TFLOPS: 68.16317070731192, Tokens per sec: 168915.84580584, Loss: 2.4990763664245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10293 , TFLOPS: 71.51505978543837, Tokens per sec: 177222.19618250098, Loss: 2.4849464893341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10294 , TFLOPS: 70.63992886737248, Tokens per sec: 175053.52536390472, Loss: 2.4642632007598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10295 , TFLOPS: 68.83358606802071, Tokens per sec: 170577.20892202505, Loss: 2.4933149814605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10296 , TFLOPS: 70.71625482592519, Tokens per sec: 175242.6694971968, Loss: 2.4925174713134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10297 , TFLOPS: 71.51078701950443, Tokens per sec: 177211.60779783264, Loss: 2.479323387145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10298 , TFLOPS: 68.1940585232447, Tokens per sec: 168992.38921629524, Loss: 2.507330894470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10299 , TFLOPS: 69.54218118969372, Tokens per sec: 172333.1856917273, Loss: 2.5028228759765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10300 , TFLOPS: 71.54331510734652, Tokens per sec: 177292.21598277002, Loss: 2.49685001373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10301 , TFLOPS: 68.87744824488043, Tokens per sec: 170685.90422810105, Loss: 2.5019679069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10302 , TFLOPS: 68.95276183166821, Tokens per sec: 170872.53959263765, Loss: 2.4835968017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10303 , TFLOPS: 70.45227418054766, Tokens per sec: 174588.49637808255, Loss: 2.472635269165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10304 , TFLOPS: 70.66162609634574, Tokens per sec: 175107.29348744755, Loss: 2.5041401386260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10305 , TFLOPS: 68.06325473578067, Tokens per sec: 168668.2430217327, Loss: 2.5047059059143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10306 , TFLOPS: 69.81975788264019, Tokens per sec: 173021.05131444562, Loss: 2.5056629180908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10307 , TFLOPS: 71.52593843656025, Tokens per sec: 177249.15467836335, Loss: 2.501227378845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10308 , TFLOPS: 69.62365858068297, Tokens per sec: 172535.09564207421, Loss: 2.5008668899536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10309 , TFLOPS: 68.92573109778317, Tokens per sec: 170805.5544563894, Loss: 2.475020170211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10310 , TFLOPS: 70.49486505457058, Tokens per sec: 174694.04125568346, Loss: 2.506467342376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10311 , TFLOPS: 69.84286690638186, Tokens per sec: 173078.31802094658, Loss: 2.487591028213501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10312 , TFLOPS: 68.19961608436809, Tokens per sec: 169006.16146497487, Loss: 2.4829330444335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10313 , TFLOPS: 71.52565325230412, Tokens per sec: 177248.447960361, Loss: 2.48364520072937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10314 , TFLOPS: 70.59206361138524, Tokens per sec: 174934.9099867751, Loss: 2.5178658962249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10315 , TFLOPS: 69.01760705143133, Tokens per sec: 171033.23319050134, Loss: 2.504415988922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10316 , TFLOPS: 70.7117842382447, Tokens per sec: 175231.59088844864, Loss: 2.4874725341796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10317 , TFLOPS: 71.49919729930359, Tokens per sec: 177182.8871944619, Loss: 2.508981227874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10318 , TFLOPS: 68.07518975473148, Tokens per sec: 168697.81931344364, Loss: 2.4886491298675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10319 , TFLOPS: 69.95106953730597, Tokens per sec: 173346.4560598803, Loss: 2.499967336654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10320 , TFLOPS: 71.48862792254877, Tokens per sec: 177156.69511455047, Loss: 2.5062549114227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10321 , TFLOPS: 68.59669143008225, Tokens per sec: 169990.15791311488, Loss: 2.493082284927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10322 , TFLOPS: 67.64160138546092, Tokens per sec: 167623.3395706893, Loss: 2.524972438812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10323 , TFLOPS: 71.48565834054432, Tokens per sec: 177149.33616321697, Loss: 2.499586582183838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10324 , TFLOPS: 70.6539338593275, Tokens per sec: 175088.2312767465, Loss: 2.5127646923065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10325 , TFLOPS: 68.75788520392099, Tokens per sec: 170389.61384164734, Loss: 2.507124423980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10326 , TFLOPS: 70.04327345609119, Tokens per sec: 173574.94752773023, Loss: 2.4800455570220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10327 , TFLOPS: 71.50121631079485, Tokens per sec: 177187.8905273497, Loss: 2.51303768157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10328 , TFLOPS: 69.59872986553461, Tokens per sec: 172473.31953981187, Loss: 2.500741958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10329 , TFLOPS: 68.88434258171806, Tokens per sec: 170702.9891542586, Loss: 2.484452962875366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10330 , TFLOPS: 71.50955099474031, Tokens per sec: 177208.54479230777, Loss: 2.487220048904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10331 , TFLOPS: 70.68756542754878, Tokens per sec: 175171.57400762188, Loss: 2.501156806945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10332 , TFLOPS: 67.34645027050922, Tokens per sec: 166891.92259425964, Loss: 2.4960501194000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10333 , TFLOPS: 71.50948494929877, Tokens per sec: 177208.38112442812, Loss: 2.468916416168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10334 , TFLOPS: 71.51503267917704, Tokens per sec: 177222.12901019937, Loss: 2.490675926208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10335 , TFLOPS: 69.77195882077862, Tokens per sec: 172902.5999736513, Loss: 2.505479574203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10336 , TFLOPS: 68.91039239832968, Tokens per sec: 170767.5434694464, Loss: 2.515444040298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10337 , TFLOPS: 71.49425168964876, Tokens per sec: 177170.63142893332, Loss: 2.4614923000335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10338 , TFLOPS: 69.84490087507146, Tokens per sec: 173083.35841941956, Loss: 2.4970285892486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10339 , TFLOPS: 68.89822562210462, Tokens per sec: 170737.39285767966, Loss: 2.49504017829895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10340 , TFLOPS: 70.48306830762652, Tokens per sec: 174664.80761723712, Loss: 2.4783782958984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10341 , TFLOPS: 69.63546931667523, Tokens per sec: 172564.36394692297, Loss: 2.4876561164855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10342 , TFLOPS: 68.91970884460642, Tokens per sec: 170790.63064961144, Loss: 2.466071605682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10343 , TFLOPS: 69.86424029213845, Tokens per sec: 173131.28362532423, Loss: 2.5084118843078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10344 , TFLOPS: 70.69947011153442, Tokens per sec: 175201.07512594754, Loss: 2.4995105266571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10345 , TFLOPS: 69.00578391709945, Tokens per sec: 171003.93416120118, Loss: 2.4814229011535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10346 , TFLOPS: 69.62787720994223, Tokens per sec: 172545.549870099, Loss: 2.4823596477508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10347 , TFLOPS: 70.38036698108574, Tokens per sec: 174410.30241658402, Loss: 2.487917423248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10348 , TFLOPS: 69.6565472800494, Tokens per sec: 172616.59746209354, Loss: 2.4860880374908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10349 , TFLOPS: 68.91200591486746, Tokens per sec: 170771.54194117055, Loss: 2.516441822052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10350 , TFLOPS: 71.52649210422803, Tokens per sec: 177250.5267278356, Loss: 2.507077932357788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10351 , TFLOPS: 68.94096363099771, Tokens per sec: 170843.3023516971, Loss: 2.4999465942382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10352 , TFLOPS: 67.20427475421525, Tokens per sec: 166539.5960029571, Loss: 2.479971170425415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10353 , TFLOPS: 71.53278395633582, Tokens per sec: 177266.11863046346, Loss: 2.466646909713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10354 , TFLOPS: 70.05130834801874, Tokens per sec: 173594.85887504308, Loss: 2.4688994884490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10355 , TFLOPS: 69.63969376524193, Tokens per sec: 172574.83259583273, Loss: 2.509753704071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10356 , TFLOPS: 69.13188799832265, Tokens per sec: 171316.43396598363, Loss: 2.496544122695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10357 , TFLOPS: 71.505044732899, Tokens per sec: 177197.37777905926, Loss: 2.4787137508392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10358 , TFLOPS: 69.64036884404847, Tokens per sec: 172576.50551548137, Loss: 2.5065090656280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10359 , TFLOPS: 68.77286969612541, Tokens per sec: 170426.74706400797, Loss: 2.468663454055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10360 , TFLOPS: 71.48724463136388, Tokens per sec: 177153.2671666122, Loss: 2.479114294052124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10361 , TFLOPS: 70.55678698843569, Tokens per sec: 174847.49062906535, Loss: 2.5082921981811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10362 , TFLOPS: 67.40421303643609, Tokens per sec: 167035.06509132037, Loss: 2.505267381668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10363 , TFLOPS: 70.57801858139325, Tokens per sec: 174900.1048552672, Loss: 2.4990789890289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10364 , TFLOPS: 71.52776211180357, Tokens per sec: 177253.67394651068, Loss: 2.485874891281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10365 , TFLOPS: 69.0961609912301, Tokens per sec: 171227.8985067535, Loss: 2.4980926513671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10366 , TFLOPS: 69.24617692360317, Tokens per sec: 171599.65451279315, Loss: 2.5242092609405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10367 , TFLOPS: 71.5088925053455, Tokens per sec: 177206.91298304882, Loss: 2.4997339248657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10368 , TFLOPS: 70.54128407620567, Tokens per sec: 174809.07270477255, Loss: 2.4864509105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10369 , TFLOPS: 67.46615548051496, Tokens per sec: 167188.5652912713, Loss: 2.5168356895446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10370 , TFLOPS: 71.49718219638974, Tokens per sec: 177177.89354745948, Loss: 2.497725009918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10371 , TFLOPS: 70.67890773766862, Tokens per sec: 175150.1192983756, Loss: 2.490981101989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10372 , TFLOPS: 68.09286169528606, Tokens per sec: 168741.61232886286, Loss: 2.501854181289673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10373 , TFLOPS: 69.86576479672202, Tokens per sec: 173135.06151562033, Loss: 2.4916269779205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10374 , TFLOPS: 71.50312695070359, Tokens per sec: 177192.6253035179, Loss: 2.4738311767578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10375 , TFLOPS: 69.72724285294237, Tokens per sec: 172791.78887948254, Loss: 2.4943017959594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10376 , TFLOPS: 69.69964148788554, Tokens per sec: 172723.3896563304, Loss: 2.5169191360473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10377 , TFLOPS: 70.44076243087106, Tokens per sec: 174559.96899426004, Loss: 2.5111920833587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10378 , TFLOPS: 69.90354024006413, Tokens per sec: 173228.6732255307, Loss: 2.4934306144714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10379 , TFLOPS: 68.73142914630736, Tokens per sec: 170324.05281650543, Loss: 2.485265016555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10380 , TFLOPS: 70.4300581987888, Tokens per sec: 174533.4427279917, Loss: 2.512423038482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10381 , TFLOPS: 71.41326784297942, Tokens per sec: 176969.94453577956, Loss: 2.4581141471862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10382 , TFLOPS: 67.65391035920047, Tokens per sec: 167653.8425635613, Loss: 2.5002267360687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10383 , TFLOPS: 70.48402870312059, Tokens per sec: 174667.1875830677, Loss: 2.5152900218963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10384 , TFLOPS: 70.54674828928265, Tokens per sec: 174822.61362671008, Loss: 2.5028884410858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10385 , TFLOPS: 69.00792677423999, Tokens per sec: 171009.24439725064, Loss: 2.5109643936157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10386 , TFLOPS: 68.81020143131428, Tokens per sec: 170519.25921623613, Loss: 2.501235008239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10387 , TFLOPS: 71.51821123365374, Tokens per sec: 177230.00581833936, Loss: 2.492605209350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10388 , TFLOPS: 69.50001831068982, Tokens per sec: 172228.70143293106, Loss: 2.5033578872680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10389 , TFLOPS: 68.0989695110686, Tokens per sec: 168756.74816920346, Loss: 2.485353946685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10390 , TFLOPS: 71.49092679197392, Tokens per sec: 177162.39196622735, Loss: 2.528287410736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10391 , TFLOPS: 70.71922801866098, Tokens per sec: 175250.0373963203, Loss: 2.502474546432495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10392 , TFLOPS: 67.8549235038993, Tokens per sec: 168151.9752795509, Loss: 2.5179054737091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10393 , TFLOPS: 70.0267887208364, Tokens per sec: 173534.09653782533, Loss: 2.481050729751587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10394 , TFLOPS: 71.52124839975086, Tokens per sec: 177237.53225049944, Loss: 2.4765748977661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10395 , TFLOPS: 69.55115122761919, Tokens per sec: 172355.41443383784, Loss: 2.4957501888275146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10396 , TFLOPS: 69.73782933711699, Tokens per sec: 172818.02335346476, Loss: 2.4854533672332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10397 , TFLOPS: 71.51170925823392, Tokens per sec: 177213.8932069973, Loss: 2.5001492500305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10398 , TFLOPS: 70.65762355793898, Tokens per sec: 175097.37475635958, Loss: 2.506009817123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10399 , TFLOPS: 67.99293665047016, Tokens per sec: 168493.98706015665, Loss: 2.507056951522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10400 , TFLOPS: 71.5146484509373, Tokens per sec: 177221.17685029507, Loss: 2.4991302490234375 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/optimizer.pt +[rank0]:[2024-08-26 23:58:13,104] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006367545007378794, 'preprocessing_with_comm': 0.0012478059943532571, 'state_converting': 1.0089164689998142, : 1.0179263429890852}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008400 + eval ppl=9.925741195678711, eval loss=2.2951314449310303 +------------------------------------------------------------------ +iteration: 10401 , TFLOPS: 66.95464665884653, Tokens per sec: 165920.99008382863, Loss: 2.4934396743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10402 , TFLOPS: 67.95444251624575, Tokens per sec: 168398.59435506962, Loss: 2.4831953048706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10403 , TFLOPS: 69.34798843219346, Tokens per sec: 171851.95464654343, Loss: 2.4904797077178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10404 , TFLOPS: 71.47755836267217, Tokens per sec: 177129.26352576356, Loss: 2.50001859664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10405 , TFLOPS: 68.95646675738705, Tokens per sec: 170881.72080670024, Loss: 2.510687828063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10406 , TFLOPS: 69.55753426188625, Tokens per sec: 172371.23229590087, Loss: 2.5082521438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10407 , TFLOPS: 71.55014871439168, Tokens per sec: 177309.1504138119, Loss: 2.4851772785186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10408 , TFLOPS: 70.5872423744959, Tokens per sec: 174922.96243066026, Loss: 2.485731840133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10409 , TFLOPS: 69.04841459246059, Tokens per sec: 171109.57767090265, Loss: 2.478304862976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10410 , TFLOPS: 69.85682860819422, Tokens per sec: 173112.91665604603, Loss: 2.5032145977020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10411 , TFLOPS: 70.65852738299485, Tokens per sec: 175099.61453441257, Loss: 2.496821165084839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10412 , TFLOPS: 69.69532919057157, Tokens per sec: 172712.7033085476, Loss: 2.492893934249878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10413 , TFLOPS: 69.3065382197331, Tokens per sec: 171749.23645394883, Loss: 2.5135955810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10414 , TFLOPS: 71.47517597892343, Tokens per sec: 177123.35971639297, Loss: 2.486870050430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10415 , TFLOPS: 68.9738334127117, Tokens per sec: 170924.75729168972, Loss: 2.4830234050750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10416 , TFLOPS: 68.99631789016387, Tokens per sec: 170980.47630368583, Loss: 2.501690149307251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10417 , TFLOPS: 70.43128098605902, Tokens per sec: 174536.4729295484, Loss: 2.5114331245422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10418 , TFLOPS: 70.57262359534573, Tokens per sec: 174886.73548553468, Loss: 2.487010955810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10419 , TFLOPS: 67.52004428440324, Tokens per sec: 167322.10768364827, Loss: 2.4689011573791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10420 , TFLOPS: 71.53421953091421, Tokens per sec: 177269.67614241032, Loss: 2.5018563270568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10421 , TFLOPS: 70.44164489589626, Tokens per sec: 174562.15583980404, Loss: 2.5211026668548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10422 , TFLOPS: 68.08104953188788, Tokens per sec: 168712.34048674494, Loss: 2.486140251159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10423 , TFLOPS: 69.92155833075152, Tokens per sec: 173273.3240391106, Loss: 2.5015437602996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10424 , TFLOPS: 71.55978744680841, Tokens per sec: 177333.03625453459, Loss: 2.4815046787261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10425 , TFLOPS: 68.70524241624533, Tokens per sec: 170259.15921470523, Loss: 2.4902918338775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10426 , TFLOPS: 68.54208384887708, Tokens per sec: 169854.83431136678, Loss: 2.4868013858795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10427 , TFLOPS: 71.54315116219725, Tokens per sec: 177291.80970863008, Loss: 2.493964672088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10428 , TFLOPS: 69.65632693763384, Tokens per sec: 172616.05142931506, Loss: 2.479979991912842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10429 , TFLOPS: 69.555553830756, Tokens per sec: 172366.32456939764, Loss: 2.5054593086242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10430 , TFLOPS: 70.10888451026554, Tokens per sec: 173737.53894762852, Loss: 2.4920144081115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10431 , TFLOPS: 71.55916543369949, Tokens per sec: 177331.49483753028, Loss: 2.4935505390167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10432 , TFLOPS: 68.95023611128299, Tokens per sec: 170866.280579292, Loss: 2.4932878017425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10433 , TFLOPS: 70.5309808428195, Tokens per sec: 174783.54015744678, Loss: 2.478562355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10434 , TFLOPS: 71.53687611260162, Tokens per sec: 177276.25944447832, Loss: 2.490011692047119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10435 , TFLOPS: 68.84818335370794, Tokens per sec: 170613.38260398174, Loss: 2.5117576122283936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10436 , TFLOPS: 67.98396148311716, Tokens per sec: 168471.74560675962, Loss: 2.484623670578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10437 , TFLOPS: 71.50044066206338, Tokens per sec: 177185.96838434838, Loss: 2.5003559589385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10438 , TFLOPS: 70.54855371562095, Tokens per sec: 174827.0876720605, Loss: 2.4908053874969482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10439 , TFLOPS: 68.01967941890634, Tokens per sec: 168560.25858630065, Loss: 2.4989852905273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10440 , TFLOPS: 70.42949719381416, Tokens per sec: 174532.05249586471, Loss: 2.4894402027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10441 , TFLOPS: 71.50491665681204, Tokens per sec: 177197.06039240706, Loss: 2.5119051933288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10442 , TFLOPS: 68.93056704708249, Tokens per sec: 170817.5384714769, Loss: 2.507838726043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10443 , TFLOPS: 68.31097360253762, Tokens per sec: 169282.11766204226, Loss: 2.5233969688415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10444 , TFLOPS: 71.46224036149852, Tokens per sec: 177091.30383143813, Loss: 2.4765734672546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10445 , TFLOPS: 70.61130456497456, Tokens per sec: 174982.59118933548, Loss: 2.492633581161499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10446 , TFLOPS: 67.47982874575045, Tokens per sec: 167222.44914875936, Loss: 2.49786376953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10447 , TFLOPS: 70.5467244877436, Tokens per sec: 174822.55464387435, Loss: 2.5073904991149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10448 , TFLOPS: 70.70823485655043, Tokens per sec: 175222.79512961354, Loss: 2.4851932525634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10449 , TFLOPS: 69.68264847406137, Tokens per sec: 172681.27909613907, Loss: 2.493832588195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10450 , TFLOPS: 68.79757828915803, Tokens per sec: 170487.977679419, Loss: 2.493978977203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10451 , TFLOPS: 71.53725664559742, Tokens per sec: 177277.2024471611, Loss: 2.508136749267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10452 , TFLOPS: 68.78268588274737, Tokens per sec: 170451.07265579942, Loss: 2.498887300491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10453 , TFLOPS: 70.06307497667669, Tokens per sec: 173624.01787705865, Loss: 2.478288412094116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10454 , TFLOPS: 70.22977561944904, Tokens per sec: 174037.12043344032, Loss: 2.4987621307373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10455 , TFLOPS: 69.13612528971842, Tokens per sec: 171326.9344408385, Loss: 2.4871037006378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10456 , TFLOPS: 68.7818845081038, Tokens per sec: 170449.0867611539, Loss: 2.4999914169311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10457 , TFLOPS: 71.49910234401449, Tokens per sec: 177182.6518847953, Loss: 2.5213677883148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10458 , TFLOPS: 69.66010124076587, Tokens per sec: 172625.40456825006, Loss: 2.485278844833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10459 , TFLOPS: 68.7960733332188, Tokens per sec: 170484.248232817, Loss: 2.5042953491210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10460 , TFLOPS: 70.85876082733765, Tokens per sec: 175595.81506702883, Loss: 2.507219076156616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10461 , TFLOPS: 71.47972510830778, Tokens per sec: 177134.63296013532, Loss: 2.480980396270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10462 , TFLOPS: 67.45432096376645, Tokens per sec: 167159.23805508885, Loss: 2.49189829826355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10463 , TFLOPS: 68.92429548030591, Tokens per sec: 170801.99683813442, Loss: 2.492056369781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10464 , TFLOPS: 71.5105134405343, Tokens per sec: 177210.92983900744, Loss: 2.4807348251342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10465 , TFLOPS: 69.68268712241786, Tokens per sec: 172681.3748710239, Loss: 2.5098183155059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10466 , TFLOPS: 67.40521106575638, Tokens per sec: 167037.53831791802, Loss: 2.5076303482055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10467 , TFLOPS: 71.51603766205271, Tokens per sec: 177224.61946847328, Loss: 2.518080472946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10468 , TFLOPS: 71.50226451475953, Tokens per sec: 177190.4880922434, Loss: 2.4895429611206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10469 , TFLOPS: 68.13471604388967, Tokens per sec: 168845.33201533972, Loss: 2.4803214073181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10470 , TFLOPS: 69.66259730903643, Tokens per sec: 172631.5901003891, Loss: 2.495783805847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10471 , TFLOPS: 71.5253800427231, Tokens per sec: 177247.77091692272, Loss: 2.4982762336730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10472 , TFLOPS: 68.89289800090052, Tokens per sec: 170724.19042544995, Loss: 2.4781761169433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10473 , TFLOPS: 69.35182563521126, Tokens per sec: 171861.46365832433, Loss: 2.46783447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10474 , TFLOPS: 71.503676508526, Tokens per sec: 177193.98716834062, Loss: 2.491976737976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10475 , TFLOPS: 69.88107055972392, Tokens per sec: 173172.99088240808, Loss: 2.5058672428131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10476 , TFLOPS: 69.03124672324306, Tokens per sec: 171067.0337997847, Loss: 2.508054494857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10477 , TFLOPS: 70.48156158548257, Tokens per sec: 174661.07379378527, Loss: 2.488028049468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10478 , TFLOPS: 70.76385358517734, Tokens per sec: 175360.6244660598, Loss: 2.469391345977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10479 , TFLOPS: 69.73881629780195, Tokens per sec: 172820.46915076478, Loss: 2.4860923290252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10480 , TFLOPS: 69.19968950714986, Tokens per sec: 171484.45357381017, Loss: 2.480071783065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10481 , TFLOPS: 71.56584955129267, Tokens per sec: 177348.0588172425, Loss: 2.4881749153137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10482 , TFLOPS: 68.87504430439719, Tokens per sec: 170679.9469987677, Loss: 2.4961259365081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10483 , TFLOPS: 68.82159384626965, Tokens per sec: 170547.49087547365, Loss: 2.4883570671081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10484 , TFLOPS: 70.39246462653232, Tokens per sec: 174440.28171466626, Loss: 2.4906933307647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10485 , TFLOPS: 70.70243997633419, Tokens per sec: 175208.43477807898, Loss: 2.516056537628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10486 , TFLOPS: 67.36249242116371, Tokens per sec: 166931.6768107702, Loss: 2.4943060874938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10487 , TFLOPS: 70.49677308209525, Tokens per sec: 174698.76955807614, Loss: 2.478303909301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10488 , TFLOPS: 71.48431270950782, Tokens per sec: 177146.00154127504, Loss: 2.498208522796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10489 , TFLOPS: 69.63978389164629, Tokens per sec: 172575.05593899096, Loss: 2.4515578746795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10490 , TFLOPS: 69.13817479462107, Tokens per sec: 171332.01333975882, Loss: 2.4949474334716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10491 , TFLOPS: 70.19684825580579, Tokens per sec: 173955.5227990838, Loss: 2.487903594970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10492 , TFLOPS: 69.68486983265159, Tokens per sec: 172686.7838674285, Loss: 2.4809889793395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10493 , TFLOPS: 69.83946373140667, Tokens per sec: 173069.88457846738, Loss: 2.4864730834960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10494 , TFLOPS: 71.56026459847664, Tokens per sec: 177334.21868893103, Loss: 2.485416889190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10495 , TFLOPS: 69.60256624562193, Tokens per sec: 172482.82651228042, Loss: 2.5119431018829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10496 , TFLOPS: 69.19437911815677, Tokens per sec: 171471.29384489768, Loss: 2.509632110595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10497 , TFLOPS: 69.93287205931394, Tokens per sec: 173301.36070479837, Loss: 2.495905637741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10498 , TFLOPS: 70.74158838893153, Tokens per sec: 175305.44885704885, Loss: 2.4905781745910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10499 , TFLOPS: 68.73248166028942, Tokens per sec: 170326.66106209694, Loss: 2.517362356185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10500 , TFLOPS: 69.7532512742946, Tokens per sec: 172856.2406126575, Loss: 2.5161349773406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10501 , TFLOPS: 71.4989129848498, Tokens per sec: 177182.18263192556, Loss: 2.4857966899871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10502 , TFLOPS: 68.65198235073031, Tokens per sec: 170127.17490527846, Loss: 2.504075050354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10503 , TFLOPS: 68.93414531400995, Tokens per sec: 170826.4058111018, Loss: 2.5064728260040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10504 , TFLOPS: 71.53038134474787, Tokens per sec: 177260.164694279, Loss: 2.512132167816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10505 , TFLOPS: 70.7328094768036, Tokens per sec: 175283.6937457188, Loss: 2.4836299419403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10506 , TFLOPS: 67.08859223748915, Tokens per sec: 166252.92198302862, Loss: 2.5019609928131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10507 , TFLOPS: 70.98595205297433, Tokens per sec: 175911.00893542668, Loss: 2.5040996074676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10508 , TFLOPS: 71.46773943388595, Tokens per sec: 177104.931138587, Loss: 2.4811809062957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10509 , TFLOPS: 69.56821534997421, Tokens per sec: 172397.70120880185, Loss: 2.529477834701538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10510 , TFLOPS: 68.61226356650846, Tokens per sec: 170028.74738259145, Loss: 2.4888267517089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10511 , TFLOPS: 71.55751773600915, Tokens per sec: 177327.4116611447, Loss: 2.4984307289123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10512 , TFLOPS: 70.74675828256996, Tokens per sec: 175318.26042299563, Loss: 2.4785680770874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10513 , TFLOPS: 68.92084260519604, Tokens per sec: 170793.44023324657, Loss: 2.4934322834014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10514 , TFLOPS: 70.525196922314, Tokens per sec: 174769.2069652827, Loss: 2.4809608459472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10515 , TFLOPS: 70.57715880439679, Tokens per sec: 174897.9742331577, Loss: 2.487016201019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10516 , TFLOPS: 68.87511645978277, Tokens per sec: 170680.12580776177, Loss: 2.497509241104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10517 , TFLOPS: 68.75800926305762, Tokens per sec: 170389.92127385412, Loss: 2.494699478149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10518 , TFLOPS: 71.52599091665746, Tokens per sec: 177249.2847298254, Loss: 2.487734794616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10519 , TFLOPS: 69.21016424363907, Tokens per sec: 171510.41112471794, Loss: 2.492377758026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10520 , TFLOPS: 68.93332416112511, Tokens per sec: 170824.37090379573, Loss: 2.488328218460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10521 , TFLOPS: 70.52238546548018, Tokens per sec: 174762.23986554096, Loss: 2.4909074306488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10522 , TFLOPS: 69.75113926407666, Tokens per sec: 172851.00681868035, Loss: 2.4956772327423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10523 , TFLOPS: 68.40202401328281, Tokens per sec: 169507.7506098996, Loss: 2.4577040672302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10524 , TFLOPS: 70.45644435846958, Tokens per sec: 174598.8305383561, Loss: 2.510139226913452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10525 , TFLOPS: 70.72684705491369, Tokens per sec: 175268.9182074606, Loss: 2.494325876235962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10526 , TFLOPS: 68.77380367817875, Tokens per sec: 170429.06157442278, Loss: 2.528989553451538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10527 , TFLOPS: 70.02792607539764, Tokens per sec: 173536.91502771806, Loss: 2.5074806213378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10528 , TFLOPS: 70.50590945491336, Tokens per sec: 174721.410496374, Loss: 2.49410080909729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10529 , TFLOPS: 69.68426450465695, Tokens per sec: 172685.2837979754, Loss: 2.493809938430786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10530 , TFLOPS: 68.95146488738749, Tokens per sec: 170869.32562183985, Loss: 2.496978998184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10531 , TFLOPS: 71.49561964987502, Tokens per sec: 177174.02138506726, Loss: 2.5151169300079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10532 , TFLOPS: 69.71708927200464, Tokens per sec: 172766.6271874096, Loss: 2.498265027999878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10533 , TFLOPS: 68.71297426914002, Tokens per sec: 170278.31959791222, Loss: 2.500718832015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10534 , TFLOPS: 71.51225551897855, Tokens per sec: 177215.2469012981, Loss: 2.493164300918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10535 , TFLOPS: 71.5272090526271, Tokens per sec: 177252.30340494693, Loss: 2.490631341934204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10536 , TFLOPS: 68.089904594461, Tokens per sec: 168734.28430726682, Loss: 2.482915163040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10537 , TFLOPS: 69.80116069496184, Tokens per sec: 172974.96543472933, Loss: 2.494232654571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10538 , TFLOPS: 71.549307209264, Tokens per sec: 177307.06507140608, Loss: 2.493537425994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10539 , TFLOPS: 68.0744287642005, Tokens per sec: 168695.93349507786, Loss: 2.47662353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10540 , TFLOPS: 69.78999252973031, Tokens per sec: 172947.28949101063, Loss: 2.4732439517974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10541 , TFLOPS: 71.53774623704291, Tokens per sec: 177278.415708696, Loss: 2.4912021160125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10542 , TFLOPS: 69.73767258860232, Tokens per sec: 172817.63491337816, Loss: 2.4944205284118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10543 , TFLOPS: 67.90241369983073, Tokens per sec: 168269.6612165459, Loss: 2.4978480339050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10544 , TFLOPS: 70.90835161388924, Tokens per sec: 175718.7064989798, Loss: 2.5023505687713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10545 , TFLOPS: 70.7595875105718, Tokens per sec: 175350.05266324038, Loss: 2.4924001693725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10546 , TFLOPS: 69.74848743741478, Tokens per sec: 172844.43530008936, Loss: 2.493469715118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10547 , TFLOPS: 69.24468524319055, Tokens per sec: 171595.95796440853, Loss: 2.5110020637512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10548 , TFLOPS: 71.52569361449896, Tokens per sec: 177248.54798232607, Loss: 2.509455680847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10549 , TFLOPS: 69.63187780229455, Tokens per sec: 172555.4637786497, Loss: 2.493116855621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10550 , TFLOPS: 69.01600145142994, Tokens per sec: 171029.25433682645, Loss: 2.507028341293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10551 , TFLOPS: 70.4224328735875, Tokens per sec: 174514.54633214395, Loss: 2.491842269897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10552 , TFLOPS: 70.66929634091196, Tokens per sec: 175126.3011984298, Loss: 2.5133297443389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10553 , TFLOPS: 68.77328810746893, Tokens per sec: 170427.78393340902, Loss: 2.474269151687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10554 , TFLOPS: 70.57861784944568, Tokens per sec: 174901.58990751594, Loss: 2.49891996383667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10555 , TFLOPS: 71.50530502013565, Tokens per sec: 177198.0227995047, Loss: 2.504091262817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10556 , TFLOPS: 68.08271611837354, Tokens per sec: 168716.47047164605, Loss: 2.5021955966949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10557 , TFLOPS: 69.76574136529982, Tokens per sec: 172887.19243406557, Loss: 2.4944074153900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10558 , TFLOPS: 70.48726080714249, Tokens per sec: 174675.19709287965, Loss: 2.5007829666137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10559 , TFLOPS: 69.82142688980306, Tokens per sec: 173025.18729805152, Loss: 2.4905309677124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10560 , TFLOPS: 69.01627378462904, Tokens per sec: 171029.92920849376, Loss: 2.485236167907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10561 , TFLOPS: 70.53120680973153, Tokens per sec: 174784.10012834708, Loss: 2.4788269996643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10562 , TFLOPS: 70.63857854562191, Tokens per sec: 175050.1791178566, Loss: 2.4819154739379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10563 , TFLOPS: 69.16474405245907, Tokens per sec: 171397.85488752677, Loss: 2.5025267601013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10564 , TFLOPS: 70.08639243273038, Tokens per sec: 173681.80110178955, Loss: 2.509050130844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10565 , TFLOPS: 69.72393110662226, Tokens per sec: 172783.58200728236, Loss: 2.4806764125823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10566 , TFLOPS: 69.76386092733765, Tokens per sec: 172882.53250164798, Loss: 2.4717633724212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10567 , TFLOPS: 69.05142895925235, Tokens per sec: 171117.04760387255, Loss: 2.4893605709075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10568 , TFLOPS: 71.53824605233079, Tokens per sec: 177279.65430603854, Loss: 2.498025894165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10569 , TFLOPS: 68.22190414449007, Tokens per sec: 169061.39373318964, Loss: 2.4700984954833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10570 , TFLOPS: 68.88582059669172, Tokens per sec: 170706.65183818014, Loss: 2.4986014366149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10571 , TFLOPS: 71.51944090115177, Tokens per sec: 177233.0530698585, Loss: 2.519669532775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10572 , TFLOPS: 70.70527812103542, Tokens per sec: 175215.46801329608, Loss: 2.466121196746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10573 , TFLOPS: 67.95187925783405, Tokens per sec: 168392.2423183592, Loss: 2.5025975704193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10574 , TFLOPS: 71.53952912153642, Tokens per sec: 177282.83389287183, Loss: 2.4893975257873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10575 , TFLOPS: 71.50804673560945, Tokens per sec: 177204.81707247373, Loss: 2.4910569190979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10576 , TFLOPS: 67.9478107610535, Tokens per sec: 168382.1601351551, Loss: 2.477372407913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10577 , TFLOPS: 69.25479662744188, Tokens per sec: 171621.01508844426, Loss: 2.479806661605835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10578 , TFLOPS: 71.5252496131753, Tokens per sec: 177247.44769813528, Loss: 2.4777743816375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10579 , TFLOPS: 70.69981958217669, Tokens per sec: 175201.9411526964, Loss: 2.4879324436187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10580 , TFLOPS: 68.0549568971842, Tokens per sec: 168647.68006360825, Loss: 2.4930155277252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10581 , TFLOPS: 71.53579002859995, Tokens per sec: 177273.56800868016, Loss: 2.4805054664611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10582 , TFLOPS: 70.58706625612935, Tokens per sec: 174922.5259899469, Loss: 2.500528335571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10583 , TFLOPS: 68.90260161841702, Tokens per sec: 170748.2370585976, Loss: 2.491689682006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10584 , TFLOPS: 69.5838928671819, Tokens per sec: 172436.5518234636, Loss: 2.4868314266204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10585 , TFLOPS: 71.53077601529364, Tokens per sec: 177261.14273136796, Loss: 2.4965872764587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10586 , TFLOPS: 69.29330858064318, Tokens per sec: 171716.45195091894, Loss: 2.501197576522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10587 , TFLOPS: 68.89764688408845, Tokens per sec: 170735.9586811223, Loss: 2.48984694480896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10588 , TFLOPS: 70.52534157032875, Tokens per sec: 174769.56541899755, Loss: 2.4814088344573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10589 , TFLOPS: 69.59723327749981, Tokens per sec: 172469.61082979888, Loss: 2.48960542678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10590 , TFLOPS: 69.04534989280893, Tokens per sec: 171101.9830075613, Loss: 2.4761085510253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10591 , TFLOPS: 70.38921870333847, Tokens per sec: 174432.23795942398, Loss: 2.4969565868377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10592 , TFLOPS: 70.64453618955663, Tokens per sec: 175064.94281581548, Loss: 2.48354434967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10593 , TFLOPS: 69.83986428506243, Tokens per sec: 173070.877194551, Loss: 2.470034122467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10594 , TFLOPS: 70.77556768406706, Tokens per sec: 175389.65329352304, Loss: 2.483273983001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10595 , TFLOPS: 70.61933009341888, Tokens per sec: 175002.47933290544, Loss: 2.5067687034606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10596 , TFLOPS: 68.98332411828231, Tokens per sec: 170948.27630558124, Loss: 2.498852491378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10597 , TFLOPS: 68.96519368475339, Tokens per sec: 170903.34709406443, Loss: 2.499927282333374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10598 , TFLOPS: 70.46173514304843, Tokens per sec: 174611.94168537052, Loss: 2.497427225112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10599 , TFLOPS: 70.65801421056493, Tokens per sec: 175098.34283660064, Loss: 2.4798803329467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10600 , TFLOPS: 68.02255614492309, Tokens per sec: 168567.3874302677, Loss: 2.480802297592163 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/optimizer.pt +[rank0]:[2024-08-27 00:23:35,363] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006391718998202123, 'preprocessing_with_comm': 0.001251688998308964, 'state_converting': 0.9922620769939385, : 1.0012823479919462}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008600 + eval ppl=9.525167465209961, eval loss=2.2539374828338623 +------------------------------------------------------------------ +iteration: 10601 , TFLOPS: 66.75310140568838, Tokens per sec: 165421.5387444603, Loss: 2.498196601867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10602 , TFLOPS: 69.32473519054983, Tokens per sec: 171794.33055219447, Loss: 2.489309072494507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10603 , TFLOPS: 68.96790606074497, Tokens per sec: 170910.06866056443, Loss: 2.487783432006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10604 , TFLOPS: 70.09556659327507, Tokens per sec: 173704.5356822365, Loss: 2.492107391357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10605 , TFLOPS: 71.50812276515158, Tokens per sec: 177205.00548205443, Loss: 2.4826433658599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10606 , TFLOPS: 68.91045790092454, Tokens per sec: 170767.70579209217, Loss: 2.50494647026062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10607 , TFLOPS: 68.27414456115804, Tokens per sec: 169190.85124044985, Loss: 2.5137810707092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10608 , TFLOPS: 71.5170242891461, Tokens per sec: 177227.06443909663, Loss: 2.4793167114257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10609 , TFLOPS: 69.75117337522785, Tokens per sec: 172851.09134987064, Loss: 2.512702226638794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10610 , TFLOPS: 67.7501749674306, Tokens per sec: 167892.39686717757, Loss: 2.477437973022461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10611 , TFLOPS: 70.66466413336283, Tokens per sec: 175114.82207784333, Loss: 2.5050220489501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10612 , TFLOPS: 70.58994300438371, Tokens per sec: 174929.6548890212, Loss: 2.4976794719696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10613 , TFLOPS: 67.84559576939961, Tokens per sec: 168128.8601259259, Loss: 2.4999852180480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10614 , TFLOPS: 70.5637379990541, Tokens per sec: 174864.71599909477, Loss: 2.505842924118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10615 , TFLOPS: 71.54535947668198, Tokens per sec: 177297.28215518914, Loss: 2.4641380310058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10616 , TFLOPS: 67.00372831803446, Tokens per sec: 166042.61984208744, Loss: 2.489697217941284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10617 , TFLOPS: 68.7243586319117, Tokens per sec: 170306.5313029515, Loss: 2.4933950901031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10618 , TFLOPS: 71.52959022875014, Tokens per sec: 177258.20422168798, Loss: 2.497788906097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10619 , TFLOPS: 69.79508575640723, Tokens per sec: 172959.91106778124, Loss: 2.4767589569091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10620 , TFLOPS: 68.41005587593733, Tokens per sec: 169527.65445034023, Loss: 2.489686965942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10621 , TFLOPS: 70.32513006915552, Tokens per sec: 174273.41926397273, Loss: 2.4964747428894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10622 , TFLOPS: 70.6108427624543, Tokens per sec: 174981.4467918185, Loss: 2.5110292434692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10623 , TFLOPS: 69.12162021485598, Tokens per sec: 171290.98926746348, Loss: 2.4722156524658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10624 , TFLOPS: 68.88059201816856, Tokens per sec: 170693.69484462426, Loss: 2.4807651042938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10625 , TFLOPS: 70.54927972508246, Tokens per sec: 174828.8868034827, Loss: 2.485217571258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10626 , TFLOPS: 70.6657602118196, Tokens per sec: 175117.5382810022, Loss: 2.498203754425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10627 , TFLOPS: 68.68925652634735, Tokens per sec: 170219.54441854692, Loss: 2.479288339614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10628 , TFLOPS: 70.3927604910569, Tokens per sec: 174441.01489955036, Loss: 2.5159754753112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10629 , TFLOPS: 69.84766572361053, Tokens per sec: 173090.2100186726, Loss: 2.488826036453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10630 , TFLOPS: 68.77430086184849, Tokens per sec: 170430.29365032568, Loss: 2.4684925079345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10631 , TFLOPS: 70.51462400716586, Tokens per sec: 174743.00611684285, Loss: 2.4904656410217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10632 , TFLOPS: 69.54898115254173, Tokens per sec: 172350.0367487424, Loss: 2.503434658050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10633 , TFLOPS: 68.59730318122932, Tokens per sec: 169991.67389984726, Loss: 2.49617862701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10634 , TFLOPS: 70.58081814116927, Tokens per sec: 174907.04247278915, Loss: 2.4903547763824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10635 , TFLOPS: 71.5374447678295, Tokens per sec: 177277.66863477556, Loss: 2.4911129474639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10636 , TFLOPS: 68.81158827975179, Tokens per sec: 170522.69597943823, Loss: 2.4935286045074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10637 , TFLOPS: 69.18368325336013, Tokens per sec: 171444.78831368484, Loss: 2.4895691871643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10638 , TFLOPS: 71.53699809759654, Tokens per sec: 177276.56173673348, Loss: 2.502288341522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10639 , TFLOPS: 69.50987281317944, Tokens per sec: 172253.12197566338, Loss: 2.515894889831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10640 , TFLOPS: 68.38008254938636, Tokens per sec: 169453.3772453124, Loss: 2.4815518856048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10641 , TFLOPS: 71.03343095228888, Tokens per sec: 176028.66687816067, Loss: 2.490464210510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10642 , TFLOPS: 70.66318839089269, Tokens per sec: 175111.16502543582, Loss: 2.4964911937713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10643 , TFLOPS: 67.88571691413154, Tokens per sec: 168228.28474228055, Loss: 2.5153684616088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10644 , TFLOPS: 69.84330845415613, Tokens per sec: 173079.41222497172, Loss: 2.4917964935302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10645 , TFLOPS: 71.5275081936411, Tokens per sec: 177253.04470933267, Loss: 2.484027624130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10646 , TFLOPS: 70.0757001716065, Tokens per sec: 173655.30450087477, Loss: 2.5013020038604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10647 , TFLOPS: 67.92068829098271, Tokens per sec: 168314.94766653294, Loss: 2.503284215927124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10648 , TFLOPS: 71.5283303612838, Tokens per sec: 177255.08213132055, Loss: 2.5073888301849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10649 , TFLOPS: 69.0222248137575, Tokens per sec: 171044.67651423407, Loss: 2.4993648529052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10650 , TFLOPS: 68.72175808582857, Tokens per sec: 170300.0868632837, Loss: 2.497955799102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10651 , TFLOPS: 69.56261409687147, Tokens per sec: 172383.82068658428, Loss: 2.462571620941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10652 , TFLOPS: 70.6926752855917, Tokens per sec: 175184.2367987496, Loss: 2.498149871826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10653 , TFLOPS: 69.69584437479335, Tokens per sec: 172713.9799918014, Loss: 2.4990384578704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10654 , TFLOPS: 68.90337595593051, Tokens per sec: 170750.15595225635, Loss: 2.4775795936584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10655 , TFLOPS: 71.5345650857272, Tokens per sec: 177270.53246530288, Loss: 2.499255895614624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10656 , TFLOPS: 69.77978971896057, Tokens per sec: 172922.00580199083, Loss: 2.5062367916107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10657 , TFLOPS: 69.09575953558, Tokens per sec: 171226.90365542858, Loss: 2.502565860748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10658 , TFLOPS: 69.85586972542515, Tokens per sec: 173110.5404389146, Loss: 2.4993107318878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10659 , TFLOPS: 70.74134138283449, Tokens per sec: 175304.8367487304, Loss: 2.491791009902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10660 , TFLOPS: 68.70884372141674, Tokens per sec: 170268.0836456365, Loss: 2.478499174118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10661 , TFLOPS: 69.68885342987367, Tokens per sec: 172696.6556601743, Loss: 2.4928629398345947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10662 , TFLOPS: 71.49569267909608, Tokens per sec: 177174.20235952188, Loss: 2.4894158840179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10663 , TFLOPS: 68.45756951287349, Tokens per sec: 169645.39847672722, Loss: 2.4949207305908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10664 , TFLOPS: 69.80250725933435, Tokens per sec: 172978.30236958072, Loss: 2.4953224658966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10665 , TFLOPS: 70.53145032089928, Tokens per sec: 174784.70357584575, Loss: 2.4963908195495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10666 , TFLOPS: 69.83597315467733, Tokens per sec: 173061.23454481334, Loss: 2.4925644397735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10667 , TFLOPS: 68.85408427835719, Tokens per sec: 170628.00574530315, Loss: 2.4887661933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10668 , TFLOPS: 71.51249212550155, Tokens per sec: 177215.83323832645, Loss: 2.5066680908203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10669 , TFLOPS: 69.55743637036849, Tokens per sec: 172370.98970993626, Loss: 2.4847867488861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10670 , TFLOPS: 68.51149622834164, Tokens per sec: 169779.0348181754, Loss: 2.4774770736694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10671 , TFLOPS: 70.64759386477364, Tokens per sec: 175072.52007183398, Loss: 2.478774070739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10672 , TFLOPS: 70.67361481900194, Tokens per sec: 175137.00286285643, Loss: 2.4945473670959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10673 , TFLOPS: 69.56774652267576, Tokens per sec: 172396.5394031107, Loss: 2.5006632804870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10674 , TFLOPS: 68.96491914038458, Tokens per sec: 170902.66674287495, Loss: 2.5056793689727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10675 , TFLOPS: 71.47813369878735, Tokens per sec: 177130.6892720355, Loss: 2.487666606903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10676 , TFLOPS: 69.47289582328001, Tokens per sec: 172161.48892134093, Loss: 2.5045480728149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10677 , TFLOPS: 68.96533052204111, Tokens per sec: 170903.68619193727, Loss: 2.476928949356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10678 , TFLOPS: 70.60134098417174, Tokens per sec: 174957.90033852705, Loss: 2.487774610519409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10679 , TFLOPS: 70.30959446331971, Tokens per sec: 174234.92032132333, Loss: 2.5097134113311768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10680 , TFLOPS: 67.74527435251717, Tokens per sec: 167880.25260357483, Loss: 2.500037431716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10681 , TFLOPS: 70.64594662710611, Tokens per sec: 175068.43803543563, Loss: 2.4855217933654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10682 , TFLOPS: 71.4761788867727, Tokens per sec: 177125.84503252286, Loss: 2.516556978225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10683 , TFLOPS: 68.0894055577352, Tokens per sec: 168733.04763928681, Loss: 2.50944185256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10684 , TFLOPS: 68.77768711325824, Tokens per sec: 170438.68515434544, Loss: 2.5055413246154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10685 , TFLOPS: 71.44674278195852, Tokens per sec: 177052.8991221387, Loss: 2.5040838718414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10686 , TFLOPS: 69.77768036855358, Tokens per sec: 172916.77859931818, Loss: 2.5007739067077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10687 , TFLOPS: 68.66314938853111, Tokens per sec: 170154.8480550996, Loss: 2.4918932914733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10688 , TFLOPS: 71.51227338890928, Tokens per sec: 177215.29118495496, Loss: 2.491905689239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10689 , TFLOPS: 70.57383732071946, Tokens per sec: 174889.74323071365, Loss: 2.4809882640838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10690 , TFLOPS: 68.83133907844767, Tokens per sec: 170571.6406343367, Loss: 2.488837242126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10691 , TFLOPS: 69.21237735848617, Tokens per sec: 171515.89546710375, Loss: 2.5064361095428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10692 , TFLOPS: 71.53450790534046, Tokens per sec: 177270.3907660056, Loss: 2.496166706085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10693 , TFLOPS: 70.09641747935697, Tokens per sec: 173706.64427168027, Loss: 2.511507749557495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10694 , TFLOPS: 68.77936213186094, Tokens per sec: 170442.83603496142, Loss: 2.486464500427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10695 , TFLOPS: 70.44167047282522, Tokens per sec: 174562.2192222515, Loss: 2.5073118209838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10696 , TFLOPS: 69.71560146954936, Tokens per sec: 172762.94024903045, Loss: 2.4858956336975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10697 , TFLOPS: 68.22279188976954, Tokens per sec: 169063.5936637854, Loss: 2.499579668045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10698 , TFLOPS: 69.58750775945416, Tokens per sec: 172445.50992472752, Loss: 2.491292715072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10699 , TFLOPS: 70.71210943593712, Tokens per sec: 175232.39676415393, Loss: 2.4964473247528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10700 , TFLOPS: 68.25795770631689, Tokens per sec: 169150.73843102422, Loss: 2.49349308013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10701 , TFLOPS: 70.70501235123501, Tokens per sec: 175214.80940645369, Loss: 2.5136663913726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10702 , TFLOPS: 70.46235401627243, Tokens per sec: 174613.4753213941, Loss: 2.5092763900756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10703 , TFLOPS: 68.84743578066667, Tokens per sec: 170611.53003563354, Loss: 2.5008394718170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10704 , TFLOPS: 69.11149602945652, Tokens per sec: 171265.9004207725, Loss: 2.4763689041137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10705 , TFLOPS: 71.474145493465, Tokens per sec: 177120.80605990964, Loss: 2.484652280807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10706 , TFLOPS: 69.70626572827638, Tokens per sec: 172739.8052537392, Loss: 2.482893943786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10707 , TFLOPS: 68.98700484903567, Tokens per sec: 170957.397561854, Loss: 2.4789321422576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10708 , TFLOPS: 71.00603684485984, Tokens per sec: 175960.78126224148, Loss: 2.5025031566619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10709 , TFLOPS: 70.5825376968413, Tokens per sec: 174911.30372116916, Loss: 2.4878127574920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10710 , TFLOPS: 68.59374100194717, Tokens per sec: 169982.84642717964, Loss: 2.501465082168579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10711 , TFLOPS: 69.83224791162893, Tokens per sec: 173052.00298216997, Loss: 2.4783294200897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10712 , TFLOPS: 71.5336124888221, Tokens per sec: 177268.17182523484, Loss: 2.529296636581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10713 , TFLOPS: 68.78401874551929, Tokens per sec: 170454.3756365752, Loss: 2.498842716217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10714 , TFLOPS: 68.67714415067238, Tokens per sec: 170189.528617336, Loss: 2.4830660820007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10715 , TFLOPS: 71.50931821322241, Tokens per sec: 177207.9679340634, Loss: 2.4967803955078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10716 , TFLOPS: 69.71892022849019, Tokens per sec: 172771.16449927734, Loss: 2.4953737258911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10717 , TFLOPS: 67.75489982783887, Tokens per sec: 167904.10559175562, Loss: 2.508030414581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10718 , TFLOPS: 70.61923363657426, Tokens per sec: 175002.24030221888, Loss: 2.4839718341827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10719 , TFLOPS: 70.70034240672072, Tokens per sec: 175203.2367695104, Loss: 2.492460250854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10720 , TFLOPS: 69.61072415415683, Tokens per sec: 172503.04270829773, Loss: 2.4789810180664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10721 , TFLOPS: 69.71639288461604, Tokens per sec: 172764.90146274725, Loss: 2.498034715652466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10722 , TFLOPS: 71.50647048311266, Tokens per sec: 177200.91094514812, Loss: 2.501389503479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10723 , TFLOPS: 69.7669456984542, Tokens per sec: 172890.17690429036, Loss: 2.488250970840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10724 , TFLOPS: 69.1363076127726, Tokens per sec: 171327.3862574513, Loss: 2.5019032955169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10725 , TFLOPS: 70.42821516476194, Tokens per sec: 174528.87548664617, Loss: 2.500396728515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10726 , TFLOPS: 70.65953967070801, Tokens per sec: 175102.1230948785, Loss: 2.4764676094055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10727 , TFLOPS: 68.82178467712578, Tokens per sec: 170547.96377535685, Loss: 2.4995129108428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10728 , TFLOPS: 69.5801531351355, Tokens per sec: 172427.28435548712, Loss: 2.4881484508514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10729 , TFLOPS: 71.53725192869575, Tokens per sec: 177277.19075815915, Loss: 2.4868288040161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10730 , TFLOPS: 68.225708927234, Tokens per sec: 169070.82240396054, Loss: 2.5108792781829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10731 , TFLOPS: 69.7252780627623, Tokens per sec: 172786.91991297685, Loss: 2.5132994651794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10732 , TFLOPS: 70.58018163176257, Tokens per sec: 174905.46513236203, Loss: 2.501349687576294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10733 , TFLOPS: 69.87065668384622, Tokens per sec: 173147.1841508, Loss: 2.4725749492645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10734 , TFLOPS: 68.79146095867928, Tokens per sec: 170472.8182606131, Loss: 2.498544931411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10735 , TFLOPS: 70.44216977622283, Tokens per sec: 174563.4565510736, Loss: 2.4922492504119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10736 , TFLOPS: 69.59832997798804, Tokens per sec: 172472.32857442024, Loss: 2.4932548999786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10737 , TFLOPS: 68.75825048887882, Tokens per sec: 170390.51905801226, Loss: 2.4908995628356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10738 , TFLOPS: 70.65211099547206, Tokens per sec: 175083.7140193077, Loss: 2.503385305404663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10739 , TFLOPS: 70.51643480552725, Tokens per sec: 174747.4934746583, Loss: 2.4867794513702393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10740 , TFLOPS: 70.68301235725455, Tokens per sec: 175160.29099787073, Loss: 2.4887800216674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10741 , TFLOPS: 69.15413053948268, Tokens per sec: 171371.55343319633, Loss: 2.5245161056518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10742 , TFLOPS: 71.50580580190936, Tokens per sec: 177199.26379190563, Loss: 2.4908061027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10743 , TFLOPS: 68.71127549829392, Tokens per sec: 170274.1098566205, Loss: 2.4748616218566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10744 , TFLOPS: 68.70590421626608, Tokens per sec: 170260.79922805994, Loss: 2.498487710952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10745 , TFLOPS: 70.63075153043944, Tokens per sec: 175030.78291201754, Loss: 2.4623682498931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10746 , TFLOPS: 70.59914540023891, Tokens per sec: 174952.45943967797, Loss: 2.492386817932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10747 , TFLOPS: 67.94399447225837, Tokens per sec: 168372.70295112184, Loss: 2.497194766998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10748 , TFLOPS: 70.74934734633358, Tokens per sec: 175324.67640821083, Loss: 2.5177528858184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10749 , TFLOPS: 71.47857595736414, Tokens per sec: 177131.7852375076, Loss: 2.486346483230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10750 , TFLOPS: 67.76989635475093, Tokens per sec: 167941.26863744753, Loss: 2.5209336280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10751 , TFLOPS: 69.23229656756816, Tokens per sec: 171565.25746149095, Loss: 2.503545045852661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10752 , TFLOPS: 71.44641147685651, Tokens per sec: 177052.07811159964, Loss: 2.4854989051818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10753 , TFLOPS: 69.8599404575773, Tokens per sec: 173120.62816161578, Loss: 2.5053508281707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10754 , TFLOPS: 68.62869595928797, Tokens per sec: 170069.46866207657, Loss: 2.4830222129821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10755 , TFLOPS: 70.87768144928606, Tokens per sec: 175642.70245249555, Loss: 2.486482858657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10756 , TFLOPS: 70.56796600507131, Tokens per sec: 174875.19346375877, Loss: 2.522599697113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10757 , TFLOPS: 69.13775822458676, Tokens per sec: 171330.98103332488, Loss: 2.522925615310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10758 , TFLOPS: 69.02780725287548, Tokens per sec: 171058.51041332487, Loss: 2.494020462036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10759 , TFLOPS: 70.57471818421395, Tokens per sec: 174891.9261074881, Loss: 2.4862658977508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10760 , TFLOPS: 70.64681696112403, Tokens per sec: 175070.59481900724, Loss: 2.474736452102661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10761 , TFLOPS: 67.98846816184185, Tokens per sec: 168482.91365309103, Loss: 2.482431650161743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10762 , TFLOPS: 71.51507916371406, Tokens per sec: 177222.24420400284, Loss: 2.4924912452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10763 , TFLOPS: 69.7213466295138, Tokens per sec: 172777.17738830397, Loss: 2.4945573806762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10764 , TFLOPS: 68.72418518346551, Tokens per sec: 170306.10147859494, Loss: 2.475266695022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10765 , TFLOPS: 69.76158554482215, Tokens per sec: 172876.8938531205, Loss: 2.4851255416870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10766 , TFLOPS: 70.70008434985529, Tokens per sec: 175202.59727617056, Loss: 2.4727916717529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10767 , TFLOPS: 68.90841467235963, Tokens per sec: 170762.6424466281, Loss: 2.501812696456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10768 , TFLOPS: 70.64354703896855, Tokens per sec: 175062.4915916943, Loss: 2.5056028366088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10769 , TFLOPS: 69.48528851425313, Tokens per sec: 172192.1993747399, Loss: 2.4828081130981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10770 , TFLOPS: 69.75560796494963, Tokens per sec: 172862.0807517116, Loss: 2.4923994541168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10771 , TFLOPS: 68.13301860738585, Tokens per sec: 168841.1255806952, Loss: 2.5153284072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10772 , TFLOPS: 70.40503398494816, Tokens per sec: 174471.42996944988, Loss: 2.4864344596862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10773 , TFLOPS: 70.5650262555378, Tokens per sec: 174867.90844057454, Loss: 2.5089223384857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10774 , TFLOPS: 69.09996101415724, Tokens per sec: 171237.31538217404, Loss: 2.509190082550049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10775 , TFLOPS: 70.06863002756974, Tokens per sec: 173637.78390511102, Loss: 2.4940690994262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10776 , TFLOPS: 69.25770590797794, Tokens per sec: 171628.2246060963, Loss: 2.507594585418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10777 , TFLOPS: 68.79823986653761, Tokens per sec: 170489.61714104438, Loss: 2.477977752685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10778 , TFLOPS: 69.72160646048022, Tokens per sec: 172777.82127806146, Loss: 2.4950881004333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10779 , TFLOPS: 71.47725094479104, Tokens per sec: 177128.50171038928, Loss: 2.5200586318969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10780 , TFLOPS: 68.93077076003823, Tokens per sec: 170818.04329462204, Loss: 2.513118267059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10781 , TFLOPS: 68.80697704391935, Tokens per sec: 170511.26882907588, Loss: 2.4965972900390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10782 , TFLOPS: 71.51763512650115, Tokens per sec: 177228.57816135176, Loss: 2.491185188293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10783 , TFLOPS: 69.77929659780082, Tokens per sec: 172920.78379343517, Loss: 2.488830089569092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10784 , TFLOPS: 67.85327059009558, Tokens per sec: 168147.8791770618, Loss: 2.4913089275360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10785 , TFLOPS: 70.60123740241218, Tokens per sec: 174957.64365151638, Loss: 2.4964776039123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10786 , TFLOPS: 70.5827610018599, Tokens per sec: 174911.8570956043, Loss: 2.496319532394409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10787 , TFLOPS: 68.71827669338222, Tokens per sec: 170291.45958929465, Loss: 2.5077919960021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10788 , TFLOPS: 69.96518525663659, Tokens per sec: 173381.43636735642, Loss: 2.4979701042175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10789 , TFLOPS: 71.50286695938998, Tokens per sec: 177191.98101640237, Loss: 2.491436243057251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10790 , TFLOPS: 69.80545990499019, Tokens per sec: 172985.61935077666, Loss: 2.4942760467529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10791 , TFLOPS: 68.51773247764504, Tokens per sec: 169794.4889309288, Loss: 2.4956369400024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10792 , TFLOPS: 70.99849517808849, Tokens per sec: 175942.09218119885, Loss: 2.4930508136749268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10793 , TFLOPS: 70.55830877675662, Tokens per sec: 174851.2617881634, Loss: 2.505556106567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10794 , TFLOPS: 68.79237619220974, Tokens per sec: 170475.0863101229, Loss: 2.4769201278686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10795 , TFLOPS: 69.598500455954, Tokens per sec: 172472.75103759955, Loss: 2.5128023624420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10796 , TFLOPS: 71.5070797784891, Tokens per sec: 177202.42084621033, Loss: 2.481760263442993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10797 , TFLOPS: 68.27375728457157, Tokens per sec: 169189.89152640654, Loss: 2.4759633541107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10798 , TFLOPS: 69.90422776315847, Tokens per sec: 173230.3769834954, Loss: 2.5010874271392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10799 , TFLOPS: 70.38146370751221, Tokens per sec: 174413.0202254832, Loss: 2.488208293914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10800 , TFLOPS: 69.74158685560393, Tokens per sec: 172827.33489820024, Loss: 2.489945888519287 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/optimizer.pt +[rank0]:[2024-08-27 00:48:59,319] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006382595995091833, 'preprocessing_with_comm': 0.0012857679976150393, 'state_converting': 0.9825106990028871, : 0.9915541570080677}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800, took 4.38s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0008800 + eval ppl=10.258781433105469, eval loss=2.328134059906006 +------------------------------------------------------------------ +iteration: 10801 , TFLOPS: 63.148607031808794, Tokens per sec: 156489.20461815255, Loss: 2.516817569732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10802 , TFLOPS: 69.99294259792049, Tokens per sec: 173450.22211678152, Loss: 2.5073065757751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10803 , TFLOPS: 69.74966273503651, Tokens per sec: 172847.34781706575, Loss: 2.495061159133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10804 , TFLOPS: 68.9187305959111, Tokens per sec: 170788.20644158145, Loss: 2.493175745010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10805 , TFLOPS: 69.59303873268905, Tokens per sec: 172459.21628569058, Loss: 2.4936940670013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10806 , TFLOPS: 70.60298302591461, Tokens per sec: 174961.96949885174, Loss: 2.509511709213257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10807 , TFLOPS: 68.9729352423651, Tokens per sec: 170922.5315266292, Loss: 2.4977810382843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10808 , TFLOPS: 69.87803563737359, Tokens per sec: 173165.47001049962, Loss: 2.50801944732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10809 , TFLOPS: 71.47647621149157, Tokens per sec: 177126.58183592925, Loss: 2.500732898712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10810 , TFLOPS: 67.39336994327424, Tokens per sec: 167008.194712, Loss: 2.4812204837799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10811 , TFLOPS: 68.83871837332397, Tokens per sec: 170589.9273399366, Loss: 2.498441696166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10812 , TFLOPS: 71.5136437678575, Tokens per sec: 177218.68713494838, Loss: 2.488112449645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10813 , TFLOPS: 68.7548789191579, Tokens per sec: 170382.1639368348, Loss: 2.499335527420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10814 , TFLOPS: 68.59872174273556, Tokens per sec: 169995.1892515272, Loss: 2.4830493927001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10815 , TFLOPS: 70.97928507031968, Tokens per sec: 175894.4874179797, Loss: 2.4969520568847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10816 , TFLOPS: 70.7091072547955, Tokens per sec: 175224.9570285674, Loss: 2.476006507873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10817 , TFLOPS: 69.67635072478144, Tokens per sec: 172665.67257966907, Loss: 2.502129077911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10818 , TFLOPS: 68.95393391338274, Tokens per sec: 170875.44414024174, Loss: 2.4764933586120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10819 , TFLOPS: 70.67665289656307, Tokens per sec: 175144.53155372478, Loss: 2.4898462295532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10820 , TFLOPS: 69.51987279047643, Tokens per sec: 172277.90302099375, Loss: 2.5025460720062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10821 , TFLOPS: 67.96407233438529, Tokens per sec: 168422.45810523146, Loss: 2.4759268760681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10822 , TFLOPS: 71.51109417536233, Tokens per sec: 177212.3689638845, Loss: 2.495495319366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10823 , TFLOPS: 69.61177514087461, Tokens per sec: 172505.64716916013, Loss: 2.460218906402588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10824 , TFLOPS: 68.60317844033449, Tokens per sec: 170006.23343912308, Loss: 2.485335111618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10825 , TFLOPS: 70.66370487350893, Tokens per sec: 175112.44492625416, Loss: 2.48659610748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10826 , TFLOPS: 70.68706723569758, Tokens per sec: 175170.3394333343, Loss: 2.474195718765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10827 , TFLOPS: 69.79963291659217, Tokens per sec: 172971.1794316306, Loss: 2.5049829483032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10828 , TFLOPS: 68.9636513110853, Tokens per sec: 170899.52492220842, Loss: 2.4988491535186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10829 , TFLOPS: 71.50062959484367, Tokens per sec: 177186.43658059058, Loss: 2.4821107387542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10830 , TFLOPS: 69.67606076014555, Tokens per sec: 172664.95401535908, Loss: 2.484729766845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10831 , TFLOPS: 68.2228678893721, Tokens per sec: 169063.78199917258, Loss: 2.5044469833374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10832 , TFLOPS: 70.33922358892879, Tokens per sec: 174308.34455850025, Loss: 2.498922824859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10833 , TFLOPS: 70.71734636560699, Tokens per sec: 175245.3744527713, Loss: 2.4944379329681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10834 , TFLOPS: 68.20992684896926, Tokens per sec: 169031.71267548107, Loss: 2.504206895828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10835 , TFLOPS: 69.38955014196495, Tokens per sec: 171954.94914175745, Loss: 2.5056333541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10836 , TFLOPS: 70.55585560028146, Tokens per sec: 174845.18254661828, Loss: 2.477334976196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10837 , TFLOPS: 69.83373527143647, Tokens per sec: 173055.68882361925, Loss: 2.465955972671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10838 , TFLOPS: 68.80286429109087, Tokens per sec: 170501.07697450975, Loss: 2.513707160949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10839 , TFLOPS: 70.43969091176128, Tokens per sec: 174557.3136518684, Loss: 2.5049893856048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10840 , TFLOPS: 70.13213530726267, Tokens per sec: 173795.15698387419, Loss: 2.4862170219421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10841 , TFLOPS: 68.1128788382097, Tokens per sec: 168791.21701409796, Loss: 2.4924376010894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10842 , TFLOPS: 71.52708428874928, Tokens per sec: 177251.99422631378, Loss: 2.4800071716308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10843 , TFLOPS: 67.90838140901242, Tokens per sec: 168284.4498572956, Loss: 2.501112699508667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10844 , TFLOPS: 69.8158890074032, Tokens per sec: 173011.46381541656, Loss: 2.5225775241851807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10845 , TFLOPS: 69.86687461420259, Tokens per sec: 173137.8117655937, Loss: 2.4851739406585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10846 , TFLOPS: 70.73848541856641, Tokens per sec: 175297.75935466366, Loss: 2.4976038932800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10847 , TFLOPS: 68.76898890402157, Tokens per sec: 170417.1300336701, Loss: 2.489694833755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10848 , TFLOPS: 68.96906335674844, Tokens per sec: 170912.9365675477, Loss: 2.4888665676116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10849 , TFLOPS: 71.50047997229176, Tokens per sec: 177186.06579942466, Loss: 2.512627601623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10850 , TFLOPS: 68.09082227250994, Tokens per sec: 168736.55841456263, Loss: 2.4980502128601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10851 , TFLOPS: 68.85322959925479, Tokens per sec: 170625.88775633668, Loss: 2.482269763946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10852 , TFLOPS: 70.95258539514784, Tokens per sec: 175828.32268169185, Loss: 2.4822824001312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10853 , TFLOPS: 70.66929181313604, Tokens per sec: 175126.28997810234, Loss: 2.503237247467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10854 , TFLOPS: 66.97911427141898, Tokens per sec: 165981.6235231132, Loss: 2.4945526123046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10855 , TFLOPS: 70.68827197060546, Tokens per sec: 175173.3248991485, Loss: 2.4897243976593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10856 , TFLOPS: 71.52136893067862, Tokens per sec: 177237.83093941602, Loss: 2.490813732147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10857 , TFLOPS: 68.16486436328562, Tokens per sec: 168920.0428719144, Loss: 2.498277425765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10858 , TFLOPS: 68.70867412317521, Tokens per sec: 170267.66336251123, Loss: 2.491140127182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10859 , TFLOPS: 71.46914973644031, Tokens per sec: 177108.42602367437, Loss: 2.490645170211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10860 , TFLOPS: 69.87874261779038, Tokens per sec: 173167.22198585275, Loss: 2.48703932762146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10861 , TFLOPS: 66.99604565973523, Tokens per sec: 166023.58136850726, Loss: 2.500458240509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10862 , TFLOPS: 71.53005103210162, Tokens per sec: 177259.34614315463, Loss: 2.4869027137756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10863 , TFLOPS: 69.78269529112133, Tokens per sec: 172929.20612988036, Loss: 2.469783067703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10864 , TFLOPS: 70.66941940370846, Tokens per sec: 175126.60616159602, Loss: 2.4802298545837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10865 , TFLOPS: 67.43042862801117, Tokens per sec: 167100.03021512923, Loss: 2.48443603515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10866 , TFLOPS: 70.07899379860304, Tokens per sec: 173663.46647139514, Loss: 2.496227502822876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10867 , TFLOPS: 70.62212450976833, Tokens per sec: 175009.40420444973, Loss: 2.493361473083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10868 , TFLOPS: 68.29193230264333, Tokens per sec: 169234.9312233319, Loss: 2.4843156337738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10869 , TFLOPS: 69.88063910466936, Tokens per sec: 173171.92168925414, Loss: 2.4923136234283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10870 , TFLOPS: 69.72269254929114, Tokens per sec: 172780.5127257775, Loss: 2.493227481842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10871 , TFLOPS: 68.51561719779454, Tokens per sec: 169789.24703444194, Loss: 2.4811275005340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10872 , TFLOPS: 69.54610883196956, Tokens per sec: 172342.91882195233, Loss: 2.4922406673431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10873 , TFLOPS: 70.67658577644059, Tokens per sec: 175144.3652226674, Loss: 2.4900083541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10874 , TFLOPS: 67.84709485795096, Tokens per sec: 168132.57503249435, Loss: 2.499349355697632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10875 , TFLOPS: 70.6673927724556, Tokens per sec: 175121.5839460996, Loss: 2.447014570236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10876 , TFLOPS: 70.43950276462361, Tokens per sec: 174556.84740253518, Loss: 2.4916505813598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10877 , TFLOPS: 68.93486196521256, Tokens per sec: 170828.1817517275, Loss: 2.475985288619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10878 , TFLOPS: 68.21557961837647, Tokens per sec: 169045.7208607761, Loss: 2.507784843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10879 , TFLOPS: 71.45253502333313, Tokens per sec: 177067.25293433302, Loss: 2.501899003982544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10880 , TFLOPS: 68.83320075940792, Tokens per sec: 170576.25408483727, Loss: 2.512695550918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10881 , TFLOPS: 68.14627823859578, Tokens per sec: 168873.98440750234, Loss: 2.509335994720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10882 , TFLOPS: 71.51638285763295, Tokens per sec: 177225.47490114754, Loss: 2.4964194297790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10883 , TFLOPS: 70.51527170373262, Tokens per sec: 174744.61118028496, Loss: 2.4984138011932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10884 , TFLOPS: 69.47152929251824, Tokens per sec: 172158.10250757748, Loss: 2.477351665496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10885 , TFLOPS: 68.85015158005551, Tokens per sec: 170618.26008568905, Loss: 2.479743242263794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10886 , TFLOPS: 70.71575222463132, Tokens per sec: 175241.42399582444, Loss: 2.4929115772247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10887 , TFLOPS: 69.54311361239095, Tokens per sec: 172335.49633788588, Loss: 2.487027645111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10888 , TFLOPS: 68.04679857461491, Tokens per sec: 168627.4628415681, Loss: 2.506498098373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10889 , TFLOPS: 71.4830019613758, Tokens per sec: 177142.7533630131, Loss: 2.487962007522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10890 , TFLOPS: 69.75861044147724, Tokens per sec: 172869.5212192971, Loss: 2.4859256744384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10891 , TFLOPS: 67.42036791530282, Tokens per sec: 167075.0986607589, Loss: 2.483224868774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10892 , TFLOPS: 70.5729530079992, Tokens per sec: 174887.55180637777, Loss: 2.4913249015808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10893 , TFLOPS: 70.68344928537617, Tokens per sec: 175161.3737538877, Loss: 2.4922778606414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10894 , TFLOPS: 68.58258443611851, Tokens per sec: 169955.19922805903, Loss: 2.480876922607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10895 , TFLOPS: 69.48750455811948, Tokens per sec: 172197.69097555813, Loss: 2.523549795150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10896 , TFLOPS: 71.5365975009786, Tokens per sec: 177275.5690141849, Loss: 2.4939069747924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10897 , TFLOPS: 68.92907699749485, Tokens per sec: 170813.8459644562, Loss: 2.4916176795959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10898 , TFLOPS: 67.93927369074977, Tokens per sec: 168361.0043345065, Loss: 2.5225069522857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10899 , TFLOPS: 71.52724252657461, Tokens per sec: 177252.3863570763, Loss: 2.503916025161743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10900 , TFLOPS: 70.71180528980109, Tokens per sec: 175231.64305652436, Loss: 2.5023343563079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10901 , TFLOPS: 68.24299783679636, Tokens per sec: 169113.6662263867, Loss: 2.481442451477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10902 , TFLOPS: 69.9752437534568, Tokens per sec: 173406.3624305118, Loss: 2.498915910720825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10903 , TFLOPS: 70.5260950232066, Tokens per sec: 174771.43255822852, Loss: 2.493217706680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10904 , TFLOPS: 69.77475700649386, Tokens per sec: 172909.53418609925, Loss: 2.475681781768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10905 , TFLOPS: 68.83972317383743, Tokens per sec: 170592.4173462969, Loss: 2.4878368377685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10906 , TFLOPS: 70.49096431483814, Tokens per sec: 174684.37479292447, Loss: 2.51163387298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10907 , TFLOPS: 70.06877486737864, Tokens per sec: 173638.14283411286, Loss: 2.4886045455932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10908 , TFLOPS: 68.15216028829619, Tokens per sec: 168888.56077462123, Loss: 2.5067806243896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10909 , TFLOPS: 70.4515705941054, Tokens per sec: 174586.7528133722, Loss: 2.487934112548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10910 , TFLOPS: 69.26528316132787, Tokens per sec: 171647.00187460083, Loss: 2.519247531890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10911 , TFLOPS: 69.58081532752921, Tokens per sec: 172428.92534118524, Loss: 2.505744695663452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10912 , TFLOPS: 70.0132940967438, Tokens per sec: 173500.65537276838, Loss: 2.4983863830566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10913 , TFLOPS: 69.69788536894244, Tokens per sec: 172719.0378001369, Loss: 2.49916672706604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10914 , TFLOPS: 69.6757519084374, Tokens per sec: 172664.1886468035, Loss: 2.4884374141693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10915 , TFLOPS: 69.1573311668665, Tokens per sec: 171379.48494043163, Loss: 2.465578079223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10916 , TFLOPS: 71.53194445432194, Tokens per sec: 177264.0382519943, Loss: 2.4977478981018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10917 , TFLOPS: 68.79178712849061, Tokens per sec: 170473.62654533616, Loss: 2.485280990600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10918 , TFLOPS: 69.15195201107994, Tokens per sec: 171366.15479982962, Loss: 2.4904708862304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10919 , TFLOPS: 70.10580244789774, Tokens per sec: 173729.90125756394, Loss: 2.497121810913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10920 , TFLOPS: 70.69412218555868, Tokens per sec: 175187.82237625695, Loss: 2.4889440536499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10921 , TFLOPS: 67.69837568389518, Tokens per sec: 167764.03253641468, Loss: 2.5009055137634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10922 , TFLOPS: 70.69719611571756, Tokens per sec: 175195.43991381204, Loss: 2.49794602394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10923 , TFLOPS: 71.49165545198606, Tokens per sec: 177164.197666006, Loss: 2.493582248687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10924 , TFLOPS: 68.35838291491763, Tokens per sec: 169399.60316068682, Loss: 2.5124809741973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10925 , TFLOPS: 68.94738134564606, Tokens per sec: 170859.20615556533, Loss: 2.472841501235962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10926 , TFLOPS: 71.56500906972349, Tokens per sec: 177345.97601132738, Loss: 2.5020430088043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10927 , TFLOPS: 69.73427867429143, Tokens per sec: 172809.2244198452, Loss: 2.5030930042266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10928 , TFLOPS: 67.03402310031937, Tokens per sec: 166117.69364983524, Loss: 2.493879795074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10929 , TFLOPS: 71.54809727413337, Tokens per sec: 177304.06671886696, Loss: 2.4847021102905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10930 , TFLOPS: 70.64856397200379, Tokens per sec: 175074.92410441642, Loss: 2.5041370391845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10931 , TFLOPS: 69.21274073870114, Tokens per sec: 171516.79596330613, Loss: 2.492833137512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10932 , TFLOPS: 68.55871670555379, Tokens per sec: 169896.0523624721, Loss: 2.4927401542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10933 , TFLOPS: 70.66999577401995, Tokens per sec: 175128.03447072033, Loss: 2.510000228881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10934 , TFLOPS: 70.60932940499082, Tokens per sec: 174977.69652531395, Loss: 2.5219340324401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10935 , TFLOPS: 67.26886427073289, Tokens per sec: 166699.65594001045, Loss: 2.4969186782836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10936 , TFLOPS: 71.54119721971568, Tokens per sec: 177286.96762391622, Loss: 2.4722747802734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10937 , TFLOPS: 69.76871508170383, Tokens per sec: 172894.5616308966, Loss: 2.48710036277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10938 , TFLOPS: 68.3635553304588, Tokens per sec: 169412.42097618646, Loss: 2.503183364868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10939 , TFLOPS: 68.96667252113839, Tokens per sec: 170907.01181353373, Loss: 2.4802987575531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10940 , TFLOPS: 70.69403694595938, Tokens per sec: 175187.61114313998, Loss: 2.4999349117279053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10941 , TFLOPS: 67.97753475614985, Tokens per sec: 168455.81946937222, Loss: 2.5084519386291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10942 , TFLOPS: 70.68890785437468, Tokens per sec: 175174.90068917698, Loss: 2.502039670944214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10943 , TFLOPS: 70.34508822205663, Tokens per sec: 174322.87776543372, Loss: 2.4820170402526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10944 , TFLOPS: 68.87188345029196, Tokens per sec: 170672.11405409814, Loss: 2.4693820476531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10945 , TFLOPS: 68.85757907789618, Tokens per sec: 170636.66624354463, Loss: 2.4839439392089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10946 , TFLOPS: 70.5371921786779, Tokens per sec: 174798.932531939, Loss: 2.499640464782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10947 , TFLOPS: 69.13573129630228, Tokens per sec: 171325.95808175133, Loss: 2.4964518547058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10948 , TFLOPS: 68.02413051079779, Tokens per sec: 168571.2888823359, Loss: 2.459254026412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10949 , TFLOPS: 71.49328344501161, Tokens per sec: 177168.23201206155, Loss: 2.4919474124908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10950 , TFLOPS: 69.68032256305219, Tokens per sec: 172675.51523243808, Loss: 2.4828975200653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10951 , TFLOPS: 69.76480340436888, Tokens per sec: 172884.86806355373, Loss: 2.4929275512695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10952 , TFLOPS: 68.77979682182752, Tokens per sec: 170443.9132445837, Loss: 2.4925124645233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10953 , TFLOPS: 71.52686241903346, Tokens per sec: 177251.444408717, Loss: 2.5051448345184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10954 , TFLOPS: 68.94174935236211, Tokens per sec: 170845.2494557924, Loss: 2.4747602939605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10955 , TFLOPS: 68.04497783298392, Tokens per sec: 168622.95084323524, Loss: 2.462954521179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10956 , TFLOPS: 71.50749591303924, Tokens per sec: 177203.4520734666, Loss: 2.4818789958953857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10957 , TFLOPS: 69.86956553777216, Tokens per sec: 173144.4801706288, Loss: 2.507277488708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10958 , TFLOPS: 67.70329221634334, Tokens per sec: 167776.216245422, Loss: 2.5042221546173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10959 , TFLOPS: 70.65094695859025, Tokens per sec: 175080.82940768532, Loss: 2.4932632446289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10960 , TFLOPS: 70.58157510408195, Tokens per sec: 174908.91831027312, Loss: 2.501213788986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10961 , TFLOPS: 67.6941757409081, Tokens per sec: 167753.62461503077, Loss: 2.4875922203063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10962 , TFLOPS: 69.96388282466789, Tokens per sec: 173378.20879746327, Loss: 2.4920151233673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10963 , TFLOPS: 71.53692913059534, Tokens per sec: 177276.3908289072, Loss: 2.473288059234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10964 , TFLOPS: 69.11874871360929, Tokens per sec: 171283.87337105218, Loss: 2.507401704788208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10965 , TFLOPS: 68.66892008587989, Tokens per sec: 170169.1484788247, Loss: 2.5045502185821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10966 , TFLOPS: 71.51869552129646, Tokens per sec: 177231.2059364667, Loss: 2.499920129776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10967 , TFLOPS: 70.56653503391745, Tokens per sec: 174871.64735960498, Loss: 2.4844350814819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10968 , TFLOPS: 67.1851832444916, Tokens per sec: 166492.28513875848, Loss: 2.514643669128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10969 , TFLOPS: 70.9749108791369, Tokens per sec: 175883.64769037184, Loss: 2.5040102005004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10970 , TFLOPS: 70.71104660746825, Tokens per sec: 175229.76295812783, Loss: 2.5089175701141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10971 , TFLOPS: 69.72323004651231, Tokens per sec: 172781.8447031018, Loss: 2.5032835006713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10972 , TFLOPS: 67.90985869205414, Tokens per sec: 168288.11072740916, Loss: 2.5042965412139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10973 , TFLOPS: 71.51079176201189, Tokens per sec: 177211.61955028857, Loss: 2.49725604057312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10974 , TFLOPS: 69.71911219376526, Tokens per sec: 172771.64021037566, Loss: 2.475287437438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10975 , TFLOPS: 67.94284618407453, Tokens per sec: 168369.8573665079, Loss: 2.499870777130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10976 , TFLOPS: 70.47738833991254, Tokens per sec: 174650.7320315418, Loss: 2.4901273250579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10977 , TFLOPS: 69.74689528432947, Tokens per sec: 172840.48976935382, Loss: 2.4908714294433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10978 , TFLOPS: 69.77992260653957, Tokens per sec: 172922.33511205032, Loss: 2.502488851547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10979 , TFLOPS: 69.27636367235893, Tokens per sec: 171674.4606015548, Loss: 2.4913647174835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10980 , TFLOPS: 69.62670679149471, Tokens per sec: 172542.64944425382, Loss: 2.493039608001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10981 , TFLOPS: 69.73567502504841, Tokens per sec: 172812.68473084178, Loss: 2.515064001083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10982 , TFLOPS: 69.09610729094567, Tokens per sec: 171227.76543153313, Loss: 2.490548610687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10983 , TFLOPS: 70.38062094111167, Tokens per sec: 174410.93175750427, Loss: 2.4716315269470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10984 , TFLOPS: 69.82662251819556, Tokens per sec: 173038.06263755387, Loss: 2.5062434673309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10985 , TFLOPS: 69.0332275097223, Tokens per sec: 171071.9424068819, Loss: 2.463468313217163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10986 , TFLOPS: 71.02726909373986, Tokens per sec: 176013.39711389143, Loss: 2.478396415710449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10987 , TFLOPS: 69.66638066120179, Tokens per sec: 172640.9656638255, Loss: 2.5085294246673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10988 , TFLOPS: 68.0107210763882, Tokens per sec: 168538.0588266978, Loss: 2.519939661026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10989 , TFLOPS: 70.60873193196883, Tokens per sec: 174976.21592134837, Loss: 2.5021815299987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10990 , TFLOPS: 71.49572249353031, Tokens per sec: 177174.2762429742, Loss: 2.5058536529541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10991 , TFLOPS: 68.14878565652697, Tokens per sec: 168880.19806535073, Loss: 2.4756765365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10992 , TFLOPS: 68.86477733695614, Tokens per sec: 170654.50432244895, Loss: 2.494173288345337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10993 , TFLOPS: 71.51232019807215, Tokens per sec: 177215.407183217, Loss: 2.488731622695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10994 , TFLOPS: 69.7211367111745, Tokens per sec: 172776.65718753473, Loss: 2.467711925506592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10995 , TFLOPS: 67.00337378875983, Tokens per sec: 166041.74127949026, Loss: 2.500499963760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10996 , TFLOPS: 71.54763043685459, Tokens per sec: 177302.9098446638, Loss: 2.4960005283355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10997 , TFLOPS: 69.75794897829839, Tokens per sec: 172867.88204067375, Loss: 2.4943127632141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10998 , TFLOPS: 69.73083279597745, Tokens per sec: 172800.68515378836, Loss: 2.4987385272979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10999 , TFLOPS: 69.05887136810064, Tokens per sec: 171135.4907128476, Loss: 2.5106124877929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11000 , TFLOPS: 70.53997678046078, Tokens per sec: 174805.83308190617, Loss: 2.4917778968811035 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/model.pt +[rank0]:[2024-08-27 01:14:26,516] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006392741997842677, 'preprocessing_with_comm': 0.001288247003685683, 'state_converting': 1.0052679969958263, : 1.014333257000544}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000, took 4.38s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009000 + eval ppl=10.58038330078125, eval loss=2.359001636505127 +------------------------------------------------------------------ +iteration: 11001 , TFLOPS: 66.05574879275555, Tokens per sec: 163693.4221498808, Loss: 2.4893782138824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11002 , TFLOPS: 68.57906304011722, Tokens per sec: 169946.47282085405, Loss: 2.4715206623077393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11003 , TFLOPS: 71.4984528336855, Tokens per sec: 177181.0423266506, Loss: 2.475102424621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11004 , TFLOPS: 69.75945991323945, Tokens per sec: 172871.6263039009, Loss: 2.4876527786254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11005 , TFLOPS: 69.03311280694157, Tokens per sec: 171071.6581607557, Loss: 2.505795478820801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11006 , TFLOPS: 69.71361479971517, Tokens per sec: 172758.01706233158, Loss: 2.491340398788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11007 , TFLOPS: 70.61309512036287, Tokens per sec: 174987.02838283367, Loss: 2.477400779724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11008 , TFLOPS: 67.97434676447854, Tokens per sec: 168447.91927482444, Loss: 2.4746267795562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11009 , TFLOPS: 69.5251299353321, Tokens per sec: 172290.93080506846, Loss: 2.480354070663452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11010 , TFLOPS: 70.55477476635606, Tokens per sec: 174842.50412108743, Loss: 2.4804940223693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11011 , TFLOPS: 69.62988603856343, Tokens per sec: 172550.52796871308, Loss: 2.4811835289001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11012 , TFLOPS: 68.82322638747193, Tokens per sec: 170551.53649241212, Loss: 2.4862589836120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11013 , TFLOPS: 70.42232207876394, Tokens per sec: 174514.27177036612, Loss: 2.5021629333496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11014 , TFLOPS: 70.06071284603256, Tokens per sec: 173618.16425711234, Loss: 2.5036659240722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11015 , TFLOPS: 67.95813740209731, Tokens per sec: 168407.75068923508, Loss: 2.491774320602417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11016 , TFLOPS: 71.5155248257651, Tokens per sec: 177223.348603659, Loss: 2.503852605819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11017 , TFLOPS: 68.5450796468084, Tokens per sec: 169862.258228655, Loss: 2.4999823570251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11018 , TFLOPS: 69.7226062573542, Tokens per sec: 172780.29888485192, Loss: 2.4876620769500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11019 , TFLOPS: 69.62103125129254, Tokens per sec: 172528.5848304205, Loss: 2.488041400909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11020 , TFLOPS: 69.62167731214055, Tokens per sec: 172530.18584037133, Loss: 2.496494770050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11021 , TFLOPS: 69.12367979797293, Tokens per sec: 171296.09314130904, Loss: 2.484022378921509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11022 , TFLOPS: 69.59529946515597, Tokens per sec: 172464.81862978384, Loss: 2.4892890453338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11023 , TFLOPS: 71.48934019291228, Tokens per sec: 177158.4601989743, Loss: 2.485877752304077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11024 , TFLOPS: 67.80266611407062, Tokens per sec: 168022.47571092873, Loss: 2.4834046363830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11025 , TFLOPS: 69.51403970492912, Tokens per sec: 172263.44799244014, Loss: 2.4915246963500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11026 , TFLOPS: 70.72522729733515, Tokens per sec: 175264.90426975, Loss: 2.484179973602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11027 , TFLOPS: 70.70940519214845, Tokens per sec: 175225.69535014857, Loss: 2.4948313236236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11028 , TFLOPS: 67.17667003413808, Tokens per sec: 166471.18846569496, Loss: 2.504946708679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11029 , TFLOPS: 70.64437551771603, Tokens per sec: 175064.54465329502, Loss: 2.4756550788879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11030 , TFLOPS: 70.59630398368986, Tokens per sec: 174945.41809646154, Loss: 2.4815149307250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11031 , TFLOPS: 68.7150399389692, Tokens per sec: 170283.43855530123, Loss: 2.5042450428009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11032 , TFLOPS: 67.8935336238857, Tokens per sec: 168247.65541013304, Loss: 2.4798412322998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11033 , TFLOPS: 71.45796254293955, Tokens per sec: 177080.7029258084, Loss: 2.4893202781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11034 , TFLOPS: 69.79461014001285, Tokens per sec: 172958.73243796252, Loss: 2.4812591075897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11035 , TFLOPS: 67.13526984259055, Tokens per sec: 166368.59423043256, Loss: 2.501455783843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11036 , TFLOPS: 71.47224612774777, Tokens per sec: 177116.09922242988, Loss: 2.474240303039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11037 , TFLOPS: 70.51273527010733, Tokens per sec: 174738.32561835024, Loss: 2.506239414215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11038 , TFLOPS: 69.06957058166131, Tokens per sec: 171162.0045426666, Loss: 2.508692741394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11039 , TFLOPS: 69.46036080491656, Tokens per sec: 172130.42576499004, Loss: 2.506434917449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11040 , TFLOPS: 70.69580753329345, Tokens per sec: 175191.9988536001, Loss: 2.4778244495391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11041 , TFLOPS: 69.8844822740935, Tokens per sec: 173181.44548644722, Loss: 2.4759509563446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11042 , TFLOPS: 69.1740839801309, Tokens per sec: 171421.00025717556, Loss: 2.4715967178344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11043 , TFLOPS: 70.35690867498649, Tokens per sec: 174352.1701499243, Loss: 2.5037643909454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11044 , TFLOPS: 69.68777310065296, Tokens per sec: 172693.97848535725, Loss: 2.508314609527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11045 , TFLOPS: 69.02957547024708, Tokens per sec: 171062.89225075694, Loss: 2.4837794303894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11046 , TFLOPS: 69.67066521833476, Tokens per sec: 172651.58326838395, Loss: 2.488095998764038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11047 , TFLOPS: 70.74556373845626, Tokens per sec: 175315.30021109202, Loss: 2.492643356323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11048 , TFLOPS: 69.04778286420324, Tokens per sec: 171108.01217869032, Loss: 2.4855706691741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11049 , TFLOPS: 69.68540350002964, Tokens per sec: 172688.1063539796, Loss: 2.5143394470214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11050 , TFLOPS: 70.38384877987565, Tokens per sec: 174418.93069753717, Loss: 2.498032808303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11051 , TFLOPS: 69.0716120333164, Tokens per sec: 171167.06348475238, Loss: 2.489590644836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11052 , TFLOPS: 68.81234332917475, Tokens per sec: 170524.5670750839, Loss: 2.497427463531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11053 , TFLOPS: 71.48068203199593, Tokens per sec: 177137.00432244843, Loss: 2.4832534790039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11054 , TFLOPS: 69.70806056969481, Tokens per sec: 172744.25306849217, Loss: 2.4796440601348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11055 , TFLOPS: 68.207057864124, Tokens per sec: 169024.60301498973, Loss: 2.4818015098571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11056 , TFLOPS: 71.03033571877843, Tokens per sec: 176020.99654855364, Loss: 2.503737449645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11057 , TFLOPS: 70.55150257561937, Tokens per sec: 174834.39527198032, Loss: 2.4990313053131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11058 , TFLOPS: 68.73280323771016, Tokens per sec: 170327.45796637022, Loss: 2.506270408630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11059 , TFLOPS: 69.10210207610936, Tokens per sec: 171242.62116954886, Loss: 2.4831595420837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11060 , TFLOPS: 70.70982303943302, Tokens per sec: 175226.73082174937, Loss: 2.474533796310425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11061 , TFLOPS: 69.56019882294326, Tokens per sec: 172377.83537172579, Loss: 2.506491184234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11062 , TFLOPS: 68.34730674263125, Tokens per sec: 169372.15518562007, Loss: 2.4951343536376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11063 , TFLOPS: 71.54304823030272, Tokens per sec: 177291.55463205656, Loss: 2.51802921295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11064 , TFLOPS: 69.73425325014146, Tokens per sec: 172809.16141600092, Loss: 2.514641046524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11065 , TFLOPS: 68.96197381688691, Tokens per sec: 170895.36790679357, Loss: 2.506136178970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11066 , TFLOPS: 69.90099666013965, Tokens per sec: 173222.3699542794, Loss: 2.487537145614624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11067 , TFLOPS: 70.68609869424778, Tokens per sec: 175167.939280928, Loss: 2.475905656814575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11068 , TFLOPS: 67.27480961082932, Tokens per sec: 166714.3891477027, Loss: 2.4829840660095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11069 , TFLOPS: 69.64077029576285, Tokens per sec: 172577.50035705316, Loss: 2.5082530975341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11070 , TFLOPS: 71.51057032169723, Tokens per sec: 177211.07079679516, Loss: 2.4869792461395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11071 , TFLOPS: 68.73543206045464, Tokens per sec: 170333.97247871978, Loss: 2.4764018058776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11072 , TFLOPS: 68.62758979514557, Tokens per sec: 170066.72746547766, Loss: 2.5015878677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11073 , TFLOPS: 70.65359327145993, Tokens per sec: 175087.3872624918, Loss: 2.472973346710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11074 , TFLOPS: 70.61263136383081, Tokens per sec: 174985.87914306024, Loss: 2.5068631172180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11075 , TFLOPS: 67.42327653062725, Tokens per sec: 167082.3065299433, Loss: 2.495419979095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11076 , TFLOPS: 69.69076774964707, Tokens per sec: 172701.39955545246, Loss: 2.5134215354919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11077 , TFLOPS: 70.57020585507033, Tokens per sec: 174880.74405879615, Loss: 2.478558301925659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11078 , TFLOPS: 69.79628743170595, Tokens per sec: 172962.88895154698, Loss: 2.4780187606811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11079 , TFLOPS: 68.8315321379158, Tokens per sec: 170572.119056966, Loss: 2.4875118732452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11080 , TFLOPS: 69.76540016696914, Tokens per sec: 172886.34690701604, Loss: 2.489515542984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11081 , TFLOPS: 70.59870580463459, Tokens per sec: 174951.370073345, Loss: 2.499467134475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11082 , TFLOPS: 68.07837620900301, Tokens per sec: 168705.7156981454, Loss: 2.489179849624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11083 , TFLOPS: 70.4496179905168, Tokens per sec: 174581.9140465826, Loss: 2.5134050846099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11084 , TFLOPS: 70.61875615997278, Tokens per sec: 175001.05706260202, Loss: 2.491297721862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11085 , TFLOPS: 69.29601806716326, Tokens per sec: 171723.1663569903, Loss: 2.4705610275268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11086 , TFLOPS: 69.58918421267187, Tokens per sec: 172449.66436047756, Loss: 2.487177848815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11087 , TFLOPS: 70.47287599976748, Tokens per sec: 174639.54995558702, Loss: 2.486128807067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11088 , TFLOPS: 69.65848646819505, Tokens per sec: 172621.40298393767, Loss: 2.5056710243225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11089 , TFLOPS: 69.17047885408009, Tokens per sec: 171412.0663576842, Loss: 2.499129295349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11090 , TFLOPS: 71.50424570802559, Tokens per sec: 177195.39770740314, Loss: 2.50109601020813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11091 , TFLOPS: 68.82816695163424, Tokens per sec: 170563.7797546544, Loss: 2.4902923107147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11092 , TFLOPS: 69.17645528711559, Tokens per sec: 171426.87661710463, Loss: 2.490832567214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11093 , TFLOPS: 70.60000723807667, Tokens per sec: 174954.59516877923, Loss: 2.494297504425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11094 , TFLOPS: 69.61924348884091, Tokens per sec: 172524.15455812705, Loss: 2.497218132019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11095 , TFLOPS: 67.79133676534053, Tokens per sec: 167994.4003367448, Loss: 2.4803965091705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11096 , TFLOPS: 70.7358108169501, Tokens per sec: 175291.13139722677, Loss: 2.4803478717803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11097 , TFLOPS: 70.61289099587863, Tokens per sec: 174986.52253987556, Loss: 2.487852096557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11098 , TFLOPS: 68.87867146192589, Tokens per sec: 170688.93549468822, Loss: 2.4857242107391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11099 , TFLOPS: 68.7616534765337, Tokens per sec: 170398.95203629113, Loss: 2.5023200511932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11100 , TFLOPS: 71.53260181227182, Tokens per sec: 177265.66725740806, Loss: 2.490450382232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11101 , TFLOPS: 69.9341305290334, Tokens per sec: 173304.47933139506, Loss: 2.50319504737854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11102 , TFLOPS: 67.10403671128746, Tokens per sec: 166291.1950904501, Loss: 2.496246337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11103 , TFLOPS: 71.49790770100937, Tokens per sec: 177179.6914278278, Loss: 2.506347417831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11104 , TFLOPS: 70.57001576827034, Tokens per sec: 174880.27300276596, Loss: 2.504821300506592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11105 , TFLOPS: 68.10850182306537, Tokens per sec: 168780.37028840228, Loss: 2.475186347961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11106 , TFLOPS: 69.59112318280808, Tokens per sec: 172454.46934207025, Loss: 2.497340202331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11107 , TFLOPS: 70.65760248223266, Tokens per sec: 175097.32252843765, Loss: 2.4736077785491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11108 , TFLOPS: 69.74676570153812, Tokens per sec: 172840.16864892215, Loss: 2.5005991458892822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11109 , TFLOPS: 68.62340560643165, Tokens per sec: 170056.35858491814, Loss: 2.4901793003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11110 , TFLOPS: 71.52755675254325, Tokens per sec: 177253.16504364143, Loss: 2.4919846057891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11111 , TFLOPS: 69.81013308015838, Tokens per sec: 172997.19999363614, Loss: 2.493373394012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11112 , TFLOPS: 68.90342300803167, Tokens per sec: 170750.2725525462, Loss: 2.5050015449523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11113 , TFLOPS: 70.5551266817644, Tokens per sec: 174843.376206236, Loss: 2.4953372478485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11114 , TFLOPS: 70.6711041209836, Tokens per sec: 175130.7810765908, Loss: 2.499661445617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11115 , TFLOPS: 68.76276680687681, Tokens per sec: 170401.7109915248, Loss: 2.4910037517547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11116 , TFLOPS: 69.83643705547368, Tokens per sec: 173062.38414208955, Loss: 2.522491693496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11117 , TFLOPS: 70.43223742214032, Tokens per sec: 174538.84308351803, Loss: 2.4925718307495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11118 , TFLOPS: 69.35180147033863, Tokens per sec: 171861.403775108, Loss: 2.493067741394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11119 , TFLOPS: 68.83093849295373, Tokens per sec: 170570.6479393546, Loss: 2.5277857780456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11120 , TFLOPS: 70.50220191686242, Tokens per sec: 174712.22280866472, Loss: 2.506758213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11121 , TFLOPS: 70.62170488153629, Tokens per sec: 175008.36431946486, Loss: 2.5051865577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11122 , TFLOPS: 68.11969736650626, Tokens per sec: 168808.11407833974, Loss: 2.489776134490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11123 , TFLOPS: 70.96136360467416, Tokens per sec: 175850.0760518977, Loss: 2.5102109909057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11124 , TFLOPS: 69.67754309710547, Tokens per sec: 172668.62740963866, Loss: 2.489650011062622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11125 , TFLOPS: 69.76473757303818, Tokens per sec: 172884.70492626436, Loss: 2.5020291805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11126 , TFLOPS: 69.28188707951162, Tokens per sec: 171688.14821293272, Loss: 2.5078442096710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11127 , TFLOPS: 70.56265053788525, Tokens per sec: 174862.0211505246, Loss: 2.488384246826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11128 , TFLOPS: 69.60646139780334, Tokens per sec: 172492.4791284724, Loss: 2.5019049644470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11129 , TFLOPS: 68.18782999222252, Tokens per sec: 168976.95423029293, Loss: 2.4968833923339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11130 , TFLOPS: 71.48272511077612, Tokens per sec: 177142.06729672942, Loss: 2.5053532123565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11131 , TFLOPS: 69.93826873076235, Tokens per sec: 173314.73425113995, Loss: 2.4727301597595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11132 , TFLOPS: 68.82494691964688, Tokens per sec: 170555.80016067388, Loss: 2.4756507873535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11133 , TFLOPS: 69.71719739970217, Tokens per sec: 172766.89513975533, Loss: 2.490720748901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11134 , TFLOPS: 70.55510835630767, Tokens per sec: 174843.33079373548, Loss: 2.5318002700805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11135 , TFLOPS: 68.77157248880623, Tokens per sec: 170423.53244137185, Loss: 2.5053510665893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11136 , TFLOPS: 69.70094055087293, Tokens per sec: 172726.60887751676, Loss: 2.5040669441223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11137 , TFLOPS: 71.52930716094984, Tokens per sec: 177257.50274849636, Loss: 2.4814155101776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11138 , TFLOPS: 69.0393672727575, Tokens per sec: 171087.15741603385, Loss: 2.504546880722046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11139 , TFLOPS: 68.65556809411196, Tokens per sec: 170136.06077238044, Loss: 2.492629051208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11140 , TFLOPS: 70.94252430615457, Tokens per sec: 175803.39019484608, Loss: 2.4968390464782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11141 , TFLOPS: 70.61049784367782, Tokens per sec: 174980.59204509447, Loss: 2.4769933223724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11142 , TFLOPS: 67.12577292197251, Tokens per sec: 166345.0598149689, Loss: 2.489441156387329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11143 , TFLOPS: 70.71269439852917, Tokens per sec: 175233.84636589597, Loss: 2.473125457763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11144 , TFLOPS: 70.7386505545317, Tokens per sec: 175298.1685797769, Loss: 2.4876365661621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11145 , TFLOPS: 69.71470128196539, Tokens per sec: 172760.70948503353, Loss: 2.501603841781616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11146 , TFLOPS: 67.92718271812834, Tokens per sec: 168331.04157242004, Loss: 2.5193047523498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11147 , TFLOPS: 70.58379443830295, Tokens per sec: 174914.41806495257, Loss: 2.488591194152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11148 , TFLOPS: 70.56470948201613, Tokens per sec: 174867.12344089238, Loss: 2.5114784240722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11149 , TFLOPS: 67.90006358686503, Tokens per sec: 168263.8373777306, Loss: 2.508723020553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11150 , TFLOPS: 70.36303373756205, Tokens per sec: 174367.3487297179, Loss: 2.4997787475585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11151 , TFLOPS: 69.874720701209, Tokens per sec: 173157.25523351342, Loss: 2.4896769523620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11152 , TFLOPS: 69.6254316175544, Tokens per sec: 172539.48942275773, Loss: 2.501117706298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11153 , TFLOPS: 69.73400863627126, Tokens per sec: 172808.55523588412, Loss: 2.4878807067871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11154 , TFLOPS: 69.58081250201629, Tokens per sec: 172428.91833925297, Loss: 2.5057663917541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11155 , TFLOPS: 69.08263360956155, Tokens per sec: 171194.37616481475, Loss: 2.500849485397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11156 , TFLOPS: 69.61092236096106, Tokens per sec: 172503.5338865929, Loss: 2.5061287879943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11157 , TFLOPS: 70.47811186281109, Tokens per sec: 174652.52500098702, Loss: 2.496044874191284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11158 , TFLOPS: 69.1100405204868, Tokens per sec: 171262.293509208, Loss: 2.4752490520477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11159 , TFLOPS: 69.57042640182347, Tokens per sec: 172403.1804388518, Loss: 2.50657057762146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11160 , TFLOPS: 70.61489206759636, Tokens per sec: 174991.48141602834, Loss: 2.5041379928588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11161 , TFLOPS: 69.59218009079389, Tokens per sec: 172457.08847648712, Loss: 2.4740560054779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11162 , TFLOPS: 68.97558479346056, Tokens per sec: 170929.09740611588, Loss: 2.4900929927825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11163 , TFLOPS: 69.84729235644144, Tokens per sec: 173089.28477369781, Loss: 2.4843835830688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11164 , TFLOPS: 70.62760108490315, Tokens per sec: 175022.97576092836, Loss: 2.506718397140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11165 , TFLOPS: 68.77374677036929, Tokens per sec: 170428.92055060196, Loss: 2.495553493499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11166 , TFLOPS: 68.9127867516061, Tokens per sec: 170773.47694062514, Loss: 2.5093142986297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11167 , TFLOPS: 71.52048214219764, Tokens per sec: 177235.63337987234, Loss: 2.5002989768981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11168 , TFLOPS: 69.76628117211509, Tokens per sec: 172888.53013481843, Loss: 2.502124071121216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11169 , TFLOPS: 68.4229665941041, Tokens per sec: 169559.64863219048, Loss: 2.5055906772613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11170 , TFLOPS: 70.87591003294285, Tokens per sec: 175638.3126876595, Loss: 2.4869022369384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11171 , TFLOPS: 70.76870334919172, Tokens per sec: 175372.64271553303, Loss: 2.5047903060913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11172 , TFLOPS: 68.94899599325308, Tokens per sec: 170863.20743020362, Loss: 2.4800314903259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11173 , TFLOPS: 69.68689508612992, Tokens per sec: 172691.8026686479, Loss: 2.486341714859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11174 , TFLOPS: 70.4531364956909, Tokens per sec: 174590.63328999936, Loss: 2.464991569519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11175 , TFLOPS: 69.70641670051691, Tokens per sec: 172740.1793795822, Loss: 2.5122621059417725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11176 , TFLOPS: 68.23023466621898, Tokens per sec: 169082.03768371663, Loss: 2.518354654312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11177 , TFLOPS: 71.5462615395799, Tokens per sec: 177299.51756642066, Loss: 2.489943742752075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11178 , TFLOPS: 69.79985794875539, Tokens per sec: 172971.73708612038, Loss: 2.4914560317993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11179 , TFLOPS: 68.2276048835403, Tokens per sec: 169075.52079254438, Loss: 2.4712250232696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11180 , TFLOPS: 70.71659644318628, Tokens per sec: 175243.516062402, Loss: 2.494441270828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11181 , TFLOPS: 70.51195337539204, Tokens per sec: 174736.38799711296, Loss: 2.5155527591705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11182 , TFLOPS: 69.05389516137235, Tokens per sec: 171123.1591244005, Loss: 2.4817519187927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11183 , TFLOPS: 68.49255176881272, Tokens per sec: 169732.08836055823, Loss: 2.499082326889038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11184 , TFLOPS: 71.51307449603566, Tokens per sec: 177217.27641666337, Loss: 2.5146076679229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11185 , TFLOPS: 69.1359935969287, Tokens per sec: 171326.6080915984, Loss: 2.489793539047241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11186 , TFLOPS: 68.98508896977161, Tokens per sec: 170952.64980198612, Loss: 2.473344087600708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11187 , TFLOPS: 70.53626939908096, Tokens per sec: 174796.64578244527, Loss: 2.4598536491394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11188 , TFLOPS: 70.62955192116092, Tokens per sec: 175027.8101480775, Loss: 2.458678722381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11189 , TFLOPS: 68.41264709938054, Tokens per sec: 169534.07578747932, Loss: 2.483283758163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11190 , TFLOPS: 70.54992392189901, Tokens per sec: 174830.48319415824, Loss: 2.4879493713378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11191 , TFLOPS: 69.61551045546003, Tokens per sec: 172514.90369018164, Loss: 2.4836435317993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11192 , TFLOPS: 69.68001063820685, Tokens per sec: 172674.74224830998, Loss: 2.505559206008911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11193 , TFLOPS: 69.1574390739553, Tokens per sec: 171379.75234608463, Loss: 2.4864091873168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11194 , TFLOPS: 69.633385222442, Tokens per sec: 172559.19933183113, Loss: 2.4991908073425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11195 , TFLOPS: 70.63509909258728, Tokens per sec: 175041.55664994338, Loss: 2.5015203952789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11196 , TFLOPS: 68.44153730142597, Tokens per sec: 169605.6688906664, Loss: 2.495548963546753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11197 , TFLOPS: 71.50443053730737, Tokens per sec: 177195.855734724, Loss: 2.4878032207489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11198 , TFLOPS: 68.7292042564778, Tokens per sec: 170318.5392944159, Loss: 2.510340690612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11199 , TFLOPS: 69.83482697517576, Tokens per sec: 173058.39418574632, Loss: 2.5024051666259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11200 , TFLOPS: 69.89564981906638, Tokens per sec: 173209.11989309665, Loss: 2.495731830596924 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/optimizer.pt +[rank0]:[2024-08-27 01:39:52,406] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006318904997897334, 'preprocessing_with_comm': 0.0013782129972241819, 'state_converting': 0.9698477890051436, : 0.9789033470005961}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200, took 4.20s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009200 + eval ppl=9.87175178527832, eval loss=2.289677381515503 +------------------------------------------------------------------ +iteration: 11201 , TFLOPS: 64.40187207823277, Tokens per sec: 159594.93346173308, Loss: 2.500561237335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11202 , TFLOPS: 70.45147856994426, Tokens per sec: 174586.5247673636, Loss: 2.492511034011841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11203 , TFLOPS: 69.96323484435628, Tokens per sec: 173376.60303087017, Loss: 2.481489896774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11204 , TFLOPS: 70.7388549246733, Tokens per sec: 175298.67503150098, Loss: 2.490725517272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11205 , TFLOPS: 69.69820169300512, Tokens per sec: 172719.8216860102, Loss: 2.495321750640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11206 , TFLOPS: 68.75485599888448, Tokens per sec: 170382.10713787237, Loss: 2.480545997619629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11207 , TFLOPS: 71.48340237326582, Tokens per sec: 177143.7456277855, Loss: 2.4984312057495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11208 , TFLOPS: 70.56058054195348, Tokens per sec: 174856.89147257683, Loss: 2.4961516857147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11209 , TFLOPS: 69.63298072372281, Tokens per sec: 172558.19693944574, Loss: 2.513467311859131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11210 , TFLOPS: 69.55408515925636, Tokens per sec: 172362.685039634, Loss: 2.486126184463501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11211 , TFLOPS: 70.6972819859377, Tokens per sec: 175195.6527096769, Loss: 2.49120831489563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11212 , TFLOPS: 69.60536671950443, Tokens per sec: 172489.76639505892, Loss: 2.4975829124450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11213 , TFLOPS: 70.44030081312413, Tokens per sec: 174558.82505463174, Loss: 2.496469497680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11214 , TFLOPS: 71.46188280156545, Tokens per sec: 177090.4177585355, Loss: 2.492258071899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11215 , TFLOPS: 70.01255291253639, Tokens per sec: 173498.81863665418, Loss: 2.5105228424072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11216 , TFLOPS: 69.54310977583454, Tokens per sec: 172335.4868304764, Loss: 2.4938769340515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11217 , TFLOPS: 69.80189378440149, Tokens per sec: 172976.78211111724, Loss: 2.492554187774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11218 , TFLOPS: 70.57340080606197, Tokens per sec: 174888.66149930633, Loss: 2.4875473976135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11219 , TFLOPS: 69.61477589357995, Tokens per sec: 172513.08336492415, Loss: 2.500300645828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11220 , TFLOPS: 69.43365279725278, Tokens per sec: 172064.24037986924, Loss: 2.496410846710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11221 , TFLOPS: 70.69990902738888, Tokens per sec: 175202.16280778547, Loss: 2.5023646354675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11222 , TFLOPS: 69.86912917194039, Tokens per sec: 173143.398808028, Loss: 2.5063467025756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11223 , TFLOPS: 69.01700388549801, Tokens per sec: 171031.73847887438, Loss: 2.496546506881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11224 , TFLOPS: 69.56318111885903, Tokens per sec: 172385.2258295321, Loss: 2.5019030570983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11225 , TFLOPS: 71.47355294031973, Tokens per sec: 177119.33764794053, Loss: 2.4948883056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11226 , TFLOPS: 68.78582675775863, Tokens per sec: 170458.85609007324, Loss: 2.4969708919525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11227 , TFLOPS: 69.8988973342412, Tokens per sec: 173217.16759344318, Loss: 2.5136992931365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11228 , TFLOPS: 70.72330469169913, Tokens per sec: 175260.13984119144, Loss: 2.509357213973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11229 , TFLOPS: 69.03046048146517, Tokens per sec: 171065.08540604738, Loss: 2.532977342605591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11230 , TFLOPS: 69.6343187278716, Tokens per sec: 172561.5126611198, Loss: 2.478881359100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11231 , TFLOPS: 69.55208743603777, Tokens per sec: 172357.73446143095, Loss: 2.5047850608825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11232 , TFLOPS: 69.7295258132404, Tokens per sec: 172797.44630658993, Loss: 2.4948480129241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11233 , TFLOPS: 70.56746108858252, Tokens per sec: 174873.94222507835, Loss: 2.4884860515594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11234 , TFLOPS: 71.53083850501561, Tokens per sec: 177261.2975877828, Loss: 2.495431900024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11235 , TFLOPS: 69.58499777162854, Tokens per sec: 172439.28989839763, Loss: 2.494234085083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11236 , TFLOPS: 70.06945234647883, Tokens per sec: 173639.82170195365, Loss: 2.5126984119415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11237 , TFLOPS: 69.57905512374487, Tokens per sec: 172424.56336230517, Loss: 2.501931667327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11238 , TFLOPS: 69.61459899257308, Tokens per sec: 172512.64498474187, Loss: 2.486654281616211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11239 , TFLOPS: 69.73174549603776, Tokens per sec: 172802.94692508006, Loss: 2.480567693710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11240 , TFLOPS: 69.72499746092574, Tokens per sec: 172786.224550715, Loss: 2.4752564430236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11241 , TFLOPS: 70.74214737248926, Tokens per sec: 175306.83407988198, Loss: 2.497349500656128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11242 , TFLOPS: 68.7490627704007, Tokens per sec: 170367.75087951278, Loss: 2.4857094287872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11243 , TFLOPS: 68.82531099383074, Tokens per sec: 170556.70237660763, Loss: 2.490175724029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11244 , TFLOPS: 70.57251060587802, Tokens per sec: 174886.45548518686, Loss: 2.4911327362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11245 , TFLOPS: 71.45053080375297, Tokens per sec: 177062.2862574304, Loss: 2.4899280071258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11246 , TFLOPS: 67.83087070579893, Tokens per sec: 168092.36979622373, Loss: 2.4821362495422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11247 , TFLOPS: 70.74283756708857, Tokens per sec: 175308.54445813032, Loss: 2.479050397872925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11248 , TFLOPS: 70.68158792684208, Tokens per sec: 175156.76110239467, Loss: 2.482499599456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11249 , TFLOPS: 68.90977946312701, Tokens per sec: 170766.0245484938, Loss: 2.5204830169677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11250 , TFLOPS: 69.68173880147124, Tokens per sec: 172679.02482725203, Loss: 2.4952392578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11251 , TFLOPS: 70.62770927608148, Tokens per sec: 175023.2438705865, Loss: 2.4848310947418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11252 , TFLOPS: 70.56787618781432, Tokens per sec: 174874.97088670172, Loss: 2.4736483097076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11253 , TFLOPS: 69.87666745407925, Tokens per sec: 173162.0795015784, Loss: 2.504237651824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11254 , TFLOPS: 71.51205272164152, Tokens per sec: 177214.74434715696, Loss: 2.5010602474212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11255 , TFLOPS: 70.69704337026525, Tokens per sec: 175195.0613937549, Loss: 2.5004329681396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11256 , TFLOPS: 69.68175165935932, Tokens per sec: 172679.0566905151, Loss: 2.5057363510131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11257 , TFLOPS: 69.62940382481365, Tokens per sec: 172549.33298992092, Loss: 2.509305715560913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11258 , TFLOPS: 70.59981916554051, Tokens per sec: 174954.1291043167, Loss: 2.5022337436676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11259 , TFLOPS: 69.62851787606157, Tokens per sec: 172547.13751131794, Loss: 2.5012617111206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11260 , TFLOPS: 69.68530118498866, Tokens per sec: 172687.85280603715, Loss: 2.49835467338562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11261 , TFLOPS: 69.54308704867952, Tokens per sec: 172335.43051008272, Loss: 2.501401662826538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11262 , TFLOPS: 69.90546440686818, Tokens per sec: 173233.4415228356, Loss: 2.513475179672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11263 , TFLOPS: 69.79675755003538, Tokens per sec: 172964.0539565551, Loss: 2.487208366394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11264 , TFLOPS: 69.4178683705484, Tokens per sec: 172025.12483169767, Loss: 2.4916152954101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11265 , TFLOPS: 70.90390024357355, Tokens per sec: 175707.67551297857, Loss: 2.4867911338806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11266 , TFLOPS: 69.73289246418797, Tokens per sec: 172805.78923850524, Loss: 2.4990663528442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11267 , TFLOPS: 69.88695152340968, Tokens per sec: 173187.564558263, Loss: 2.493291139602661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11268 , TFLOPS: 69.54518783026731, Tokens per sec: 172340.63647827742, Loss: 2.488448143005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11269 , TFLOPS: 69.14665593593827, Tokens per sec: 171353.03054221772, Loss: 2.4983808994293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11270 , TFLOPS: 69.78178930474087, Tokens per sec: 172926.96099582696, Loss: 2.486335039138794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11271 , TFLOPS: 70.65313612969663, Tokens per sec: 175086.25441484404, Loss: 2.493192672729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11272 , TFLOPS: 70.37791213872882, Tokens per sec: 174404.2190468003, Loss: 2.487135648727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11273 , TFLOPS: 68.9956680170921, Tokens per sec: 170978.86584662457, Loss: 2.4681880474090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11274 , TFLOPS: 71.50657684435457, Tokens per sec: 177201.1745200222, Loss: 2.4901010990142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11275 , TFLOPS: 70.72685713605935, Tokens per sec: 175268.94318965008, Loss: 2.484468698501587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11276 , TFLOPS: 68.77671395522657, Tokens per sec: 170436.27356154076, Loss: 2.480830192565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11277 , TFLOPS: 69.75207313571514, Tokens per sec: 172853.32105547492, Loss: 2.4728546142578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11278 , TFLOPS: 70.57240773880932, Tokens per sec: 174886.2005692589, Loss: 2.503094434738159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11279 , TFLOPS: 68.86289217336758, Tokens per sec: 170649.8326794086, Loss: 2.489532709121704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11280 , TFLOPS: 69.68726061323179, Tokens per sec: 172692.70848507242, Loss: 2.498189687728882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11281 , TFLOPS: 71.5002133569748, Tokens per sec: 177185.40509729914, Loss: 2.509756565093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11282 , TFLOPS: 69.19602285479613, Tokens per sec: 171475.36720536253, Loss: 2.5103535652160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11283 , TFLOPS: 69.5020072409858, Tokens per sec: 172233.6302213033, Loss: 2.4912493228912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11284 , TFLOPS: 69.88978328790002, Tokens per sec: 173194.58198261465, Loss: 2.5111217498779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11285 , TFLOPS: 70.74699226135924, Tokens per sec: 175318.84024821038, Loss: 2.504889488220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11286 , TFLOPS: 68.90884742554591, Tokens per sec: 170763.71485669544, Loss: 2.506268262863159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11287 , TFLOPS: 69.50897588642927, Tokens per sec: 172250.89929237185, Loss: 2.4816343784332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11288 , TFLOPS: 70.5014345634269, Tokens per sec: 174710.32122232055, Loss: 2.484851121902466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11289 , TFLOPS: 69.76617224848762, Tokens per sec: 172888.26021007064, Loss: 2.5101265907287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11290 , TFLOPS: 68.70776541328767, Tokens per sec: 170265.41147930725, Loss: 2.484909772872925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11291 , TFLOPS: 70.70013337794424, Tokens per sec: 175202.71877317587, Loss: 2.4859466552734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11292 , TFLOPS: 71.53007877795234, Tokens per sec: 177259.4149004292, Loss: 2.4712653160095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11293 , TFLOPS: 68.53017804545341, Tokens per sec: 169825.33041895044, Loss: 2.5077476501464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11294 , TFLOPS: 70.45037969793695, Tokens per sec: 174583.80164147876, Loss: 2.4806430339813232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11295 , TFLOPS: 70.59894080088385, Tokens per sec: 174951.95241993765, Loss: 2.4855802059173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11296 , TFLOPS: 69.78103876724502, Tokens per sec: 172925.10108123368, Loss: 2.5031659603118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11297 , TFLOPS: 69.66626289133005, Tokens per sec: 172640.6738171099, Loss: 2.4752371311187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11298 , TFLOPS: 69.71965117836942, Tokens per sec: 172772.97587359883, Loss: 2.498457431793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11299 , TFLOPS: 69.64190044048324, Tokens per sec: 172580.30098016604, Loss: 2.4860949516296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11300 , TFLOPS: 69.5383280175779, Tokens per sec: 172323.6371067623, Loss: 2.4972803592681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11301 , TFLOPS: 70.53568456055905, Tokens per sec: 174795.19648816265, Loss: 2.4863805770874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11302 , TFLOPS: 69.68270398864645, Tokens per sec: 172681.41666739635, Loss: 2.501588821411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11303 , TFLOPS: 70.21437383263168, Tokens per sec: 173998.95310906, Loss: 2.5209505558013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11304 , TFLOPS: 69.69024462289006, Tokens per sec: 172700.10318972144, Loss: 2.4807777404785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11305 , TFLOPS: 69.63532335012972, Tokens per sec: 172564.00222574372, Loss: 2.510578155517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11306 , TFLOPS: 69.75893056158588, Tokens per sec: 172870.31451219047, Loss: 2.478076696395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11307 , TFLOPS: 69.52754083888111, Tokens per sec: 172296.90528964586, Loss: 2.494483232498169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11308 , TFLOPS: 70.5233505964767, Tokens per sec: 174764.6315664683, Loss: 2.515242338180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11309 , TFLOPS: 68.8391745527008, Tokens per sec: 170591.05780268466, Loss: 2.4947853088378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11310 , TFLOPS: 68.85647460371018, Tokens per sec: 170633.9292348439, Loss: 2.524941921234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11311 , TFLOPS: 70.72229567751853, Tokens per sec: 175257.6393928998, Loss: 2.4810898303985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11312 , TFLOPS: 70.41890389073946, Tokens per sec: 174505.80112389693, Loss: 2.4851083755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11313 , TFLOPS: 69.76321664690819, Tokens per sec: 172880.9359037704, Loss: 2.4689321517944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11314 , TFLOPS: 70.70415882536426, Tokens per sec: 175212.6942753223, Loss: 2.4937143325805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11315 , TFLOPS: 70.58258912155262, Tokens per sec: 174911.43115726864, Loss: 2.479301929473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11316 , TFLOPS: 69.56290411435995, Tokens per sec: 172384.53938186882, Loss: 2.498405694961548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11317 , TFLOPS: 69.65011635555189, Tokens per sec: 172600.6609227639, Loss: 2.4656169414520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11318 , TFLOPS: 70.70170637363914, Tokens per sec: 175206.6168297876, Loss: 2.5192654132843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11319 , TFLOPS: 70.6828839722187, Tokens per sec: 175159.97284560907, Loss: 2.503819465637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11320 , TFLOPS: 68.14165447545136, Tokens per sec: 168862.5262130809, Loss: 2.4655356407165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11321 , TFLOPS: 71.54305444666568, Tokens per sec: 177291.57003688876, Loss: 2.494048833847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11322 , TFLOPS: 70.53576990732381, Tokens per sec: 174795.40798684745, Loss: 2.506146192550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11323 , TFLOPS: 68.13926163107972, Tokens per sec: 168856.59648113442, Loss: 2.5025618076324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11324 , TFLOPS: 69.74012965508317, Tokens per sec: 172823.7237947859, Loss: 2.484118700027466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11325 , TFLOPS: 70.62661258769103, Tokens per sec: 175020.52615594477, Loss: 2.50178599357605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11326 , TFLOPS: 69.85250264559383, Tokens per sec: 173102.19644417815, Loss: 2.4878151416778564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11327 , TFLOPS: 68.69652942386796, Tokens per sec: 170237.56745977877, Loss: 2.508558750152588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11328 , TFLOPS: 70.62332989593934, Tokens per sec: 175012.39128416576, Loss: 2.481691598892212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11329 , TFLOPS: 69.77628646401799, Tokens per sec: 172913.32435032772, Loss: 2.4942145347595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11330 , TFLOPS: 69.68319391120707, Tokens per sec: 172682.630749471, Loss: 2.4898738861083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11331 , TFLOPS: 69.95251307902682, Tokens per sec: 173350.03331528383, Loss: 2.513045310974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11332 , TFLOPS: 71.01184526411825, Tokens per sec: 175975.175165014, Loss: 2.481154441833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11333 , TFLOPS: 69.7093239029963, Tokens per sec: 172747.38374758093, Loss: 2.4790587425231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11334 , TFLOPS: 70.4188001504047, Tokens per sec: 174505.5440439195, Loss: 2.482006311416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11335 , TFLOPS: 69.6407393702979, Tokens per sec: 172577.4237203443, Loss: 2.5164437294006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11336 , TFLOPS: 69.59565220286007, Tokens per sec: 172465.69275267184, Loss: 2.491312026977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11337 , TFLOPS: 69.70890008221888, Tokens per sec: 172746.33347300673, Loss: 2.514382839202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11338 , TFLOPS: 69.61750641360625, Tokens per sec: 172519.84989434094, Loss: 2.473891496658325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11339 , TFLOPS: 71.53661291804966, Tokens per sec: 177275.60721938527, Loss: 2.5090646743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11340 , TFLOPS: 68.2853919890443, Tokens per sec: 169218.72360575848, Loss: 2.4660775661468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11341 , TFLOPS: 71.46067948644775, Tokens per sec: 177087.43581111773, Loss: 2.501314878463745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11342 , TFLOPS: 69.666314575537, Tokens per sec: 172640.80189626827, Loss: 2.4884445667266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11343 , TFLOPS: 68.93114938972202, Tokens per sec: 170818.981580688, Loss: 2.482731342315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11344 , TFLOPS: 69.58829328694245, Tokens per sec: 172447.45654837645, Loss: 2.5109527111053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11345 , TFLOPS: 70.72494049784345, Tokens per sec: 175264.19354901597, Loss: 2.478501558303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11346 , TFLOPS: 68.76638947197634, Tokens per sec: 170410.68835471093, Loss: 2.491079092025757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11347 , TFLOPS: 69.67709373015333, Tokens per sec: 172667.5138288294, Loss: 2.4893200397491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11348 , TFLOPS: 71.4813392877989, Tokens per sec: 177138.63307473093, Loss: 2.507113218307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11349 , TFLOPS: 69.94971436092969, Tokens per sec: 173343.0977835349, Loss: 2.5049195289611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11350 , TFLOPS: 69.51425172770386, Tokens per sec: 172263.9734082322, Loss: 2.4832754135131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11351 , TFLOPS: 69.89560528689695, Tokens per sec: 173209.00953747518, Loss: 2.505577325820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11352 , TFLOPS: 70.79173848368461, Tokens per sec: 175429.72631633596, Loss: 2.4985463619232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11353 , TFLOPS: 69.46268185151844, Tokens per sec: 172136.17757415387, Loss: 2.4899587631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11354 , TFLOPS: 70.45697114718388, Tokens per sec: 174600.13597882068, Loss: 2.489133596420288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11355 , TFLOPS: 70.57213539457042, Tokens per sec: 174885.52567023368, Loss: 2.507295846939087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11356 , TFLOPS: 70.51180769300079, Tokens per sec: 174736.02698009918, Loss: 2.4843368530273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11357 , TFLOPS: 68.09322968700998, Tokens per sec: 168742.52425289236, Loss: 2.489089250564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11358 , TFLOPS: 70.70221737667383, Tokens per sec: 175207.8831515992, Loss: 2.491952419281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11359 , TFLOPS: 71.53501853971936, Tokens per sec: 177271.65617424756, Loss: 2.4909849166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11360 , TFLOPS: 68.05214002495856, Tokens per sec: 168640.69954392902, Loss: 2.4738128185272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11361 , TFLOPS: 71.50593081030472, Tokens per sec: 177199.57357648015, Loss: 2.489675998687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11362 , TFLOPS: 70.58485852640156, Tokens per sec: 174917.05499247994, Loss: 2.481221914291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11363 , TFLOPS: 68.84069160834517, Tokens per sec: 170594.81723368893, Loss: 2.493579864501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11364 , TFLOPS: 68.68639373680251, Tokens per sec: 170212.4501106925, Loss: 2.500488519668579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11365 , TFLOPS: 70.69724568738287, Tokens per sec: 175195.5627578594, Loss: 2.490731716156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11366 , TFLOPS: 69.86193938497999, Tokens per sec: 173125.5817239197, Loss: 2.509291887283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11367 , TFLOPS: 69.75337794949917, Tokens per sec: 172856.55452776887, Loss: 2.4940598011016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11368 , TFLOPS: 70.44859923158022, Tokens per sec: 174579.3894497122, Loss: 2.4784791469573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11369 , TFLOPS: 69.72877965966372, Tokens per sec: 172795.5972558313, Loss: 2.4948830604553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11370 , TFLOPS: 69.6906771976803, Tokens per sec: 172701.17515770372, Loss: 2.4933109283447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11371 , TFLOPS: 69.5150770586006, Tokens per sec: 172266.018669112, Loss: 2.4768362045288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11372 , TFLOPS: 69.71567893133859, Tokens per sec: 172763.1322078773, Loss: 2.4789915084838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11373 , TFLOPS: 69.85764614765706, Tokens per sec: 173114.94260889437, Loss: 2.485884666442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11374 , TFLOPS: 70.5227645633413, Tokens per sec: 174763.17931180188, Loss: 2.497511386871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11375 , TFLOPS: 69.5553186837118, Tokens per sec: 172365.74184911852, Loss: 2.4892425537109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11376 , TFLOPS: 70.25478691723693, Tokens per sec: 174099.1011845806, Loss: 2.488795280456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11377 , TFLOPS: 68.75314326390497, Tokens per sec: 170377.86279191982, Loss: 2.495842695236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11378 , TFLOPS: 71.46929650721783, Tokens per sec: 177108.7897378292, Loss: 2.4916231632232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11379 , TFLOPS: 69.36253912537562, Tokens per sec: 171888.01286714, Loss: 2.482823371887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11380 , TFLOPS: 68.81265084353593, Tokens per sec: 170525.32912954642, Loss: 2.4905145168304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11381 , TFLOPS: 70.96281415293534, Tokens per sec: 175853.67067027997, Loss: 2.495604991912842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11382 , TFLOPS: 70.64754067673742, Tokens per sec: 175072.38826602107, Loss: 2.5172359943389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11383 , TFLOPS: 67.46948237834954, Tokens per sec: 167196.80971059346, Loss: 2.4900383949279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11384 , TFLOPS: 69.66910049819073, Tokens per sec: 172647.7057194989, Loss: 2.4888243675231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11385 , TFLOPS: 70.67203308272862, Tokens per sec: 175133.08314612854, Loss: 2.5051558017730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11386 , TFLOPS: 70.37858348417289, Tokens per sec: 174405.88271476555, Loss: 2.4630346298217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11387 , TFLOPS: 68.94722975561416, Tokens per sec: 170858.83049876723, Loss: 2.489175319671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11388 , TFLOPS: 71.54001867642474, Tokens per sec: 177284.047063814, Loss: 2.5036237239837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11389 , TFLOPS: 70.58887597730576, Tokens per sec: 174927.0106783793, Loss: 2.4848971366882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11390 , TFLOPS: 68.33466245948777, Tokens per sec: 169340.8212591078, Loss: 2.4800400733947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11391 , TFLOPS: 69.51531382288262, Tokens per sec: 172266.60539708458, Loss: 2.502398729324341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11392 , TFLOPS: 70.71841213222343, Tokens per sec: 175248.01553985075, Loss: 2.5223278999328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11393 , TFLOPS: 69.71756114250577, Tokens per sec: 172767.7965344922, Loss: 2.5336389541625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11394 , TFLOPS: 70.30829731888046, Tokens per sec: 174231.70585451063, Loss: 2.4937219619750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11395 , TFLOPS: 70.65992761910745, Tokens per sec: 175103.08447374834, Loss: 2.500621795654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11396 , TFLOPS: 70.48066870918238, Tokens per sec: 174658.86114795494, Loss: 2.4798409938812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11397 , TFLOPS: 68.35960650338161, Tokens per sec: 169402.63534769, Loss: 2.4974782466888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11398 , TFLOPS: 70.86356344626367, Tokens per sec: 175607.71648577965, Loss: 2.4878671169281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11399 , TFLOPS: 70.64320243226334, Tokens per sec: 175061.6376183172, Loss: 2.4845638275146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11400 , TFLOPS: 69.78296183417439, Tokens per sec: 172929.86665292797, Loss: 2.4834163188934326 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/model.pt +[rank0]:[2024-08-27 02:05:13,477] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006468445993959904, 'preprocessing_with_comm': 0.0012728119909297675, 'state_converting': 0.9793174489896046, : 0.9884915760048898}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400, took 4.31s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009400 + eval ppl=9.714609146118164, eval loss=2.2736308574676514 +------------------------------------------------------------------ +iteration: 11401 , TFLOPS: 66.57336402373916, Tokens per sec: 164976.12971228297, Loss: 2.5069992542266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11402 , TFLOPS: 70.5773679627461, Tokens per sec: 174898.492550588, Loss: 2.4522664546966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11403 , TFLOPS: 69.07838098867201, Tokens per sec: 171183.83770178558, Loss: 2.4840450286865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11404 , TFLOPS: 69.58064597866898, Tokens per sec: 172428.5056760543, Loss: 2.490243911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11405 , TFLOPS: 69.54286632737548, Tokens per sec: 172334.88353837683, Loss: 2.485278367996216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11406 , TFLOPS: 70.67719440904376, Tokens per sec: 175145.87348130447, Loss: 2.5051023960113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11407 , TFLOPS: 68.23148137903263, Tokens per sec: 169085.1271754056, Loss: 2.4990663528442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11408 , TFLOPS: 70.42201920272836, Tokens per sec: 174513.5212101854, Loss: 2.494659423828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11409 , TFLOPS: 70.55716640903624, Tokens per sec: 174848.43087511012, Loss: 2.493224620819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11410 , TFLOPS: 68.87451092552598, Tokens per sec: 170678.6252271683, Loss: 2.5087263584136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11411 , TFLOPS: 69.48536551059932, Tokens per sec: 172192.39018016763, Loss: 2.4970269203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11412 , TFLOPS: 69.48974315279305, Tokens per sec: 172203.23845976076, Loss: 2.4992177486419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11413 , TFLOPS: 69.82335702722425, Tokens per sec: 173029.97039120342, Loss: 2.4904489517211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11414 , TFLOPS: 69.66748518004478, Tokens per sec: 172643.70278319123, Loss: 2.4923019409179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11415 , TFLOPS: 71.47859948466912, Tokens per sec: 177131.84354076107, Loss: 2.5030674934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11416 , TFLOPS: 68.7044795780188, Tokens per sec: 170257.26881754637, Loss: 2.4685068130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11417 , TFLOPS: 68.76545441418958, Tokens per sec: 170408.37117851025, Loss: 2.4747960567474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11418 , TFLOPS: 70.5648090066424, Tokens per sec: 174867.3700738798, Loss: 2.4985642433166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11419 , TFLOPS: 70.64280737183435, Tokens per sec: 175060.65861505468, Loss: 2.490788459777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11420 , TFLOPS: 68.6202584857789, Tokens per sec: 170048.55967325668, Loss: 2.5058481693267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11421 , TFLOPS: 70.02802486045066, Tokens per sec: 173537.15982796153, Loss: 2.5031075477600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11422 , TFLOPS: 70.69510532355709, Tokens per sec: 175190.2587005186, Loss: 2.4898202419281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11423 , TFLOPS: 68.701683968629, Tokens per sec: 170250.34098951664, Loss: 2.4854164123535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11424 , TFLOPS: 69.69323483295904, Tokens per sec: 172707.51325967122, Loss: 2.489777088165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11425 , TFLOPS: 71.50674563429266, Tokens per sec: 177201.59280008252, Loss: 2.490333318710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11426 , TFLOPS: 70.86880744746985, Tokens per sec: 175620.711698443, Loss: 2.485485315322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11427 , TFLOPS: 67.3709113432252, Tokens per sec: 166952.53982705946, Loss: 2.481853485107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11428 , TFLOPS: 70.96918430138642, Tokens per sec: 175869.45659987154, Loss: 2.4775991439819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11429 , TFLOPS: 70.62900406230777, Tokens per sec: 175026.45249348777, Loss: 2.500953435897827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11430 , TFLOPS: 68.7059692860533, Tokens per sec: 170260.96047816073, Loss: 2.506058931350708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11431 , TFLOPS: 68.60415056066276, Tokens per sec: 170008.64246038444, Loss: 2.511979103088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11432 , TFLOPS: 70.70642995447243, Tokens per sec: 175218.32238343792, Loss: 2.4768505096435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11433 , TFLOPS: 70.6549651378431, Tokens per sec: 175090.7868985129, Loss: 2.516185998916626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11434 , TFLOPS: 69.16654739461235, Tokens per sec: 171402.32376803717, Loss: 2.484313488006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11435 , TFLOPS: 70.49553886794635, Tokens per sec: 174695.7110394553, Loss: 2.4956390857696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11436 , TFLOPS: 69.61969609214384, Tokens per sec: 172525.2761589701, Loss: 2.5055859088897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11437 , TFLOPS: 69.58136441169637, Tokens per sec: 172430.28603223807, Loss: 2.462773561477661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11438 , TFLOPS: 68.81890752922884, Tokens per sec: 170540.83388592422, Loss: 2.5152437686920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11439 , TFLOPS: 70.65985863531286, Tokens per sec: 175102.91352430618, Loss: 2.492671489715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11440 , TFLOPS: 69.8177231131317, Tokens per sec: 173016.0089314552, Loss: 2.4907381534576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11441 , TFLOPS: 69.76401999069603, Tokens per sec: 172882.9266781723, Loss: 2.4831225872039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11442 , TFLOPS: 69.49001429394968, Tokens per sec: 172203.9103774156, Loss: 2.5156002044677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11443 , TFLOPS: 69.83152607555981, Tokens per sec: 173050.2141928739, Loss: 2.484879493713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11444 , TFLOPS: 69.84167859840822, Tokens per sec: 173075.373262885, Loss: 2.479445695877075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11445 , TFLOPS: 69.84925090654623, Tokens per sec: 173094.1382766096, Loss: 2.5128843784332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11446 , TFLOPS: 70.62332448845541, Tokens per sec: 175012.3778838249, Loss: 2.5182836055755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11447 , TFLOPS: 68.11450569089448, Tokens per sec: 168795.24853426343, Loss: 2.487802267074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11448 , TFLOPS: 70.63341317024495, Tokens per sec: 175037.37874865942, Loss: 2.4664666652679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11449 , TFLOPS: 69.61775823011736, Tokens per sec: 172520.47392339536, Loss: 2.4781181812286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11450 , TFLOPS: 68.89451696490967, Tokens per sec: 170728.20239660825, Loss: 2.5046463012695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11451 , TFLOPS: 69.63126821125636, Tokens per sec: 172553.95314490507, Loss: 2.4801182746887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11452 , TFLOPS: 70.69294809862306, Tokens per sec: 175184.91285949413, Loss: 2.504342555999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11453 , TFLOPS: 69.52239498037474, Tokens per sec: 172284.15328540397, Loss: 2.4742493629455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11454 , TFLOPS: 68.7586197157159, Tokens per sec: 170391.43404278823, Loss: 2.513808012008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11455 , TFLOPS: 71.51632048018323, Tokens per sec: 177225.3203229557, Loss: 2.502561569213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11456 , TFLOPS: 69.7297320350449, Tokens per sec: 172797.9573469387, Loss: 2.4825408458709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11457 , TFLOPS: 69.54628996204235, Tokens per sec: 172343.3676822258, Loss: 2.4844894409179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11458 , TFLOPS: 69.74799559480421, Tokens per sec: 172843.21645991947, Loss: 2.4754106998443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11459 , TFLOPS: 70.75132947797368, Tokens per sec: 175329.58834876484, Loss: 2.4898579120635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11460 , TFLOPS: 68.50472332475387, Tokens per sec: 169762.25081698797, Loss: 2.4958441257476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11461 , TFLOPS: 69.59908321464954, Tokens per sec: 172474.1951778433, Loss: 2.497509241104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11462 , TFLOPS: 71.53469765619187, Tokens per sec: 177270.86098951817, Loss: 2.4952526092529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11463 , TFLOPS: 69.2840383242599, Tokens per sec: 171693.4792343981, Loss: 2.50266170501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11464 , TFLOPS: 68.69597236877648, Tokens per sec: 170236.18701589742, Loss: 2.4932007789611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11465 , TFLOPS: 70.67493460151886, Tokens per sec: 175140.27342931938, Loss: 2.5113444328308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11466 , TFLOPS: 70.71434315413232, Tokens per sec: 175237.93216390602, Loss: 2.497443675994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11467 , TFLOPS: 68.89196095171727, Tokens per sec: 170721.86831434938, Loss: 2.4981284141540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11468 , TFLOPS: 68.75450113412107, Tokens per sec: 170381.22774389706, Loss: 2.488502025604248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11469 , TFLOPS: 70.52100262363474, Tokens per sec: 174758.81303111537, Loss: 2.491492986679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11470 , TFLOPS: 69.03262408894206, Tokens per sec: 171070.44706371616, Loss: 2.4939804077148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11471 , TFLOPS: 69.6420695928109, Tokens per sec: 172580.72015826762, Loss: 2.5078506469726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11472 , TFLOPS: 69.64011194481846, Tokens per sec: 172575.86889088963, Loss: 2.4643609523773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11473 , TFLOPS: 70.51040323558932, Tokens per sec: 174732.5465799198, Loss: 2.4672563076019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11474 , TFLOPS: 69.66687068429201, Tokens per sec: 172642.17999502356, Loss: 2.496286630630493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11475 , TFLOPS: 69.86837580469494, Tokens per sec: 173141.5318810035, Loss: 2.487391710281372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11476 , TFLOPS: 70.59464937544533, Tokens per sec: 174941.31779496145, Loss: 2.508528232574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11477 , TFLOPS: 69.24774254863857, Tokens per sec: 171603.53430409872, Loss: 2.47931170463562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11478 , TFLOPS: 69.66760853700453, Tokens per sec: 172644.0084753264, Loss: 2.4946818351745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11479 , TFLOPS: 69.72839511867623, Tokens per sec: 172794.64432090364, Loss: 2.4882960319519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11480 , TFLOPS: 69.1752761343823, Tokens per sec: 171423.95454673716, Loss: 2.4784042835235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11481 , TFLOPS: 69.595784280328, Tokens per sec: 172466.02005518693, Loss: 2.490499496459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11482 , TFLOPS: 70.38259834363093, Tokens per sec: 174415.83197877585, Loss: 2.488398790359497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11483 , TFLOPS: 69.76493495717077, Tokens per sec: 172885.19406588856, Loss: 2.4756174087524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11484 , TFLOPS: 69.11266185753057, Tokens per sec: 171268.78947116656, Loss: 2.505768299102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11485 , TFLOPS: 70.70929704338226, Tokens per sec: 175225.4273455924, Loss: 2.4997305870056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11486 , TFLOPS: 69.6606077377603, Tokens per sec: 172626.6597235975, Loss: 2.4923717975616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11487 , TFLOPS: 68.94341001685287, Tokens per sec: 170849.3647653378, Loss: 2.4909262657165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11488 , TFLOPS: 69.4037256420731, Tokens per sec: 171990.0775925857, Loss: 2.483649730682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11489 , TFLOPS: 70.56331601714133, Tokens per sec: 174863.67028142978, Loss: 2.492175579071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11490 , TFLOPS: 67.8518700997454, Tokens per sec: 168144.40860769732, Loss: 2.490159749984741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11491 , TFLOPS: 69.72166225761481, Tokens per sec: 172777.95954950756, Loss: 2.4774909019470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11492 , TFLOPS: 70.674872664314, Tokens per sec: 175140.1199421028, Loss: 2.4935810565948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11493 , TFLOPS: 71.49612542292941, Tokens per sec: 177175.27474641154, Loss: 2.48415470123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11494 , TFLOPS: 68.03779844598859, Tokens per sec: 168605.15953138607, Loss: 2.4886443614959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11495 , TFLOPS: 70.61142840532764, Tokens per sec: 174982.89807937253, Loss: 2.5071029663085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11496 , TFLOPS: 70.58119280509322, Tokens per sec: 174907.97093126536, Loss: 2.48063588142395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11497 , TFLOPS: 68.04183606733156, Tokens per sec: 168615.16520185466, Loss: 2.4873478412628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11498 , TFLOPS: 69.6464552482099, Tokens per sec: 172591.58829546618, Loss: 2.5046873092651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11499 , TFLOPS: 70.70210919328696, Tokens per sec: 175207.61506124915, Loss: 2.4900290966033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11500 , TFLOPS: 70.56777693690627, Tokens per sec: 174874.72493201817, Loss: 2.5026285648345947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11501 , TFLOPS: 68.8457363978273, Tokens per sec: 170607.31877775508, Loss: 2.4884021282196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11502 , TFLOPS: 71.50466730453805, Tokens per sec: 177196.44247000368, Loss: 2.495065212249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11503 , TFLOPS: 69.74418688149827, Tokens per sec: 172833.77804878293, Loss: 2.495051383972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11504 , TFLOPS: 69.8278615472011, Tokens per sec: 173041.13308791988, Loss: 2.4788970947265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11505 , TFLOPS: 68.63814031425507, Tokens per sec: 170092.87281406662, Loss: 2.4913768768310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11506 , TFLOPS: 70.67221277617304, Tokens per sec: 175133.52844627865, Loss: 2.4909651279449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11507 , TFLOPS: 68.89731646408045, Tokens per sec: 170735.13986394374, Loss: 2.5047693252563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11508 , TFLOPS: 69.69922541899626, Tokens per sec: 172722.358591789, Loss: 2.48427414894104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11509 , TFLOPS: 69.3648709149293, Tokens per sec: 171893.79129852183, Loss: 2.5090742111206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11510 , TFLOPS: 69.81055147739586, Tokens per sec: 172998.23682808087, Loss: 2.4993016719818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11511 , TFLOPS: 69.9652502831997, Tokens per sec: 173381.59751034307, Loss: 2.4882824420928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11512 , TFLOPS: 70.01439857893743, Tokens per sec: 173503.39240131294, Loss: 2.4791901111602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11513 , TFLOPS: 70.90925882561054, Tokens per sec: 175720.95466956278, Loss: 2.4904496669769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11514 , TFLOPS: 68.69891258413612, Tokens per sec: 170243.47319345007, Loss: 2.5020854473114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11515 , TFLOPS: 70.55932764542901, Tokens per sec: 174853.78665697124, Loss: 2.499417304992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11516 , TFLOPS: 69.55810360449891, Tokens per sec: 172372.6431896133, Loss: 2.500265121459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11517 , TFLOPS: 69.03281142926211, Tokens per sec: 171070.91131366647, Loss: 2.483713388442993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11518 , TFLOPS: 69.43807224815588, Tokens per sec: 172075.19226604982, Loss: 2.490269660949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11519 , TFLOPS: 69.45726088677179, Tokens per sec: 172122.74382634312, Loss: 2.4851131439208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11520 , TFLOPS: 70.58160450906627, Tokens per sec: 174908.99117906345, Loss: 2.5147180557250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11521 , TFLOPS: 68.80404008081204, Tokens per sec: 170503.9907109628, Loss: 2.496605157852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11522 , TFLOPS: 71.42487338931812, Tokens per sec: 176998.70435806332, Loss: 2.4734206199645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11523 , TFLOPS: 68.78167093338384, Tokens per sec: 170448.55749947068, Loss: 2.485382318496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11524 , TFLOPS: 69.87376060804328, Tokens per sec: 173154.87601688577, Loss: 2.4845306873321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11525 , TFLOPS: 69.72731742019369, Tokens per sec: 172791.9736653456, Loss: 2.5126750469207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11526 , TFLOPS: 70.64747769391475, Tokens per sec: 175072.23218764833, Loss: 2.510023832321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11527 , TFLOPS: 67.93192720142413, Tokens per sec: 168342.79892467498, Loss: 2.496478796005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11528 , TFLOPS: 69.69087852030722, Tokens per sec: 172701.67405735073, Loss: 2.489699602127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11529 , TFLOPS: 71.45449034556998, Tokens per sec: 177072.09843823247, Loss: 2.513442039489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11530 , TFLOPS: 69.19206485357432, Tokens per sec: 171465.55884232506, Loss: 2.476835250854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11531 , TFLOPS: 68.67610728132253, Tokens per sec: 170186.9591408665, Loss: 2.4794864654541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11532 , TFLOPS: 70.70337724008118, Tokens per sec: 175210.75742089213, Loss: 2.480231285095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11533 , TFLOPS: 70.6419266647402, Tokens per sec: 175058.4761258575, Loss: 2.4891586303710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11534 , TFLOPS: 67.76980872186337, Tokens per sec: 167941.0514734986, Loss: 2.5058066844940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11535 , TFLOPS: 70.62023919389665, Tokens per sec: 175004.73218403503, Loss: 2.483504295349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11536 , TFLOPS: 70.63633151678694, Tokens per sec: 175044.61073287288, Loss: 2.4948039054870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11537 , TFLOPS: 69.10744278185298, Tokens per sec: 171255.85602670885, Loss: 2.4643795490264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11538 , TFLOPS: 68.70213172765476, Tokens per sec: 170251.45058570718, Loss: 2.505845069885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11539 , TFLOPS: 70.69689271151168, Tokens per sec: 175194.68804476713, Loss: 2.512476921081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11540 , TFLOPS: 70.59451308420581, Tokens per sec: 174940.9800502562, Loss: 2.508870840072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11541 , TFLOPS: 68.70531240636102, Tokens per sec: 170259.332657922, Loss: 2.49794602394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11542 , TFLOPS: 70.56236140664795, Tokens per sec: 174861.30465146815, Loss: 2.4930293560028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11543 , TFLOPS: 70.56834635498561, Tokens per sec: 174876.13601274538, Loss: 2.5008697509765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11544 , TFLOPS: 69.33995698122453, Tokens per sec: 171832.05182630508, Loss: 2.5084593296051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11545 , TFLOPS: 69.76725529467785, Tokens per sec: 172890.94411783744, Loss: 2.4917330741882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11546 , TFLOPS: 69.06269393899453, Tokens per sec: 171144.96346461363, Loss: 2.491142988204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11547 , TFLOPS: 68.82640594157219, Tokens per sec: 170559.4157777294, Loss: 2.483281373977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11548 , TFLOPS: 69.78838633815457, Tokens per sec: 172943.30917134933, Loss: 2.4697647094726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11549 , TFLOPS: 70.45369004875465, Tokens per sec: 174592.00505547025, Loss: 2.508070230484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11550 , TFLOPS: 69.75880416033677, Tokens per sec: 172870.00127597092, Loss: 2.508110523223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11551 , TFLOPS: 69.12539340873455, Tokens per sec: 171300.33965754614, Loss: 2.4962055683135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11552 , TFLOPS: 70.67285655975653, Tokens per sec: 175135.12381291715, Loss: 2.495718240737915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11553 , TFLOPS: 69.71364908536475, Tokens per sec: 172758.10202594812, Loss: 2.508692502975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11554 , TFLOPS: 69.26230067384739, Tokens per sec: 171639.61094207605, Loss: 2.4903814792633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11555 , TFLOPS: 69.93211751079382, Tokens per sec: 173299.49085044512, Loss: 2.4933054447174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11556 , TFLOPS: 69.71466611402559, Tokens per sec: 172760.62233500462, Loss: 2.497795343399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11557 , TFLOPS: 68.90943636116079, Tokens per sec: 170765.17430402568, Loss: 2.487940788269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11558 , TFLOPS: 69.57775066965671, Tokens per sec: 172421.33078137727, Loss: 2.5003647804260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11559 , TFLOPS: 70.60578999013825, Tokens per sec: 174968.92546541052, Loss: 2.490946054458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11560 , TFLOPS: 70.45750218179403, Tokens per sec: 174601.45194108292, Loss: 2.4763498306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11561 , TFLOPS: 69.00659426817435, Tokens per sec: 171005.9423004323, Loss: 2.4850566387176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11562 , TFLOPS: 70.94867632751998, Tokens per sec: 175818.63558149067, Loss: 2.5036025047302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11563 , TFLOPS: 70.71164848605682, Tokens per sec: 175231.2544795727, Loss: 2.4985451698303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11564 , TFLOPS: 67.95983136110767, Tokens per sec: 168411.94850626774, Loss: 2.4929165840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11565 , TFLOPS: 69.58668059784146, Tokens per sec: 172443.46012713184, Loss: 2.5084497928619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11566 , TFLOPS: 70.53182612398523, Tokens per sec: 174785.63485729086, Loss: 2.496610641479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11567 , TFLOPS: 69.77293790184682, Tokens per sec: 172905.02624439297, Loss: 2.4901363849639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11568 , TFLOPS: 68.25190574454565, Tokens per sec: 169135.7410030765, Loss: 2.4890763759613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11569 , TFLOPS: 71.50196971773431, Tokens per sec: 177189.75755274028, Loss: 2.474687337875366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11570 , TFLOPS: 69.65717100586257, Tokens per sec: 172618.14312336766, Loss: 2.5147762298583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11571 , TFLOPS: 68.40225932756678, Tokens per sec: 169508.3337446173, Loss: 2.502819299697876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11572 , TFLOPS: 69.88623171082604, Tokens per sec: 173185.78078338684, Loss: 2.490971326828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11573 , TFLOPS: 70.54562568811942, Tokens per sec: 174819.83169736285, Loss: 2.498525381088257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11574 , TFLOPS: 69.86386446118611, Tokens per sec: 173130.35227482312, Loss: 2.501133918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11575 , TFLOPS: 68.75969669148229, Tokens per sec: 170394.10290737607, Loss: 2.5095129013061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11576 , TFLOPS: 70.71428338622488, Tokens per sec: 175237.7840524474, Loss: 2.4848453998565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11577 , TFLOPS: 69.7883007758846, Tokens per sec: 172943.09713861885, Loss: 2.4948883056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11578 , TFLOPS: 69.89380946675868, Tokens per sec: 173204.55929734666, Loss: 2.477604389190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11579 , TFLOPS: 69.89998499458946, Tokens per sec: 173219.86293560188, Loss: 2.4800572395324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11580 , TFLOPS: 70.85801379190153, Tokens per sec: 175593.96383092535, Loss: 2.506991386413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11581 , TFLOPS: 68.68630267495878, Tokens per sec: 170212.22444941243, Loss: 2.5035462379455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11582 , TFLOPS: 70.71908853495961, Tokens per sec: 175249.6917403429, Loss: 2.527078628540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11583 , TFLOPS: 69.6734001429455, Tokens per sec: 172658.3607128462, Loss: 2.4948856830596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11584 , TFLOPS: 68.79649759630792, Tokens per sec: 170485.2996034883, Loss: 2.4918243885040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11585 , TFLOPS: 69.63213838654518, Tokens per sec: 172556.1095351285, Loss: 2.503068208694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11586 , TFLOPS: 69.6072735268073, Tokens per sec: 172494.4916736076, Loss: 2.509526014328003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11587 , TFLOPS: 69.91219978331016, Tokens per sec: 173250.13252762178, Loss: 2.4941954612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11588 , TFLOPS: 68.87406582669487, Tokens per sec: 170677.52222323316, Loss: 2.5006768703460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11589 , TFLOPS: 71.50808813076299, Tokens per sec: 177204.9196542242, Loss: 2.4998104572296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11590 , TFLOPS: 69.64254313249049, Tokens per sec: 172581.89364175836, Loss: 2.4959847927093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11591 , TFLOPS: 69.18666716533617, Tokens per sec: 171452.1827762665, Loss: 2.511089324951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11592 , TFLOPS: 69.59142603395192, Tokens per sec: 172455.2198405665, Loss: 2.496547222137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11593 , TFLOPS: 69.53891182398445, Tokens per sec: 172325.08384334936, Loss: 2.509575605392456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11594 , TFLOPS: 69.72525888946315, Tokens per sec: 172786.87239942944, Loss: 2.4856221675872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11595 , TFLOPS: 69.71978257911275, Tokens per sec: 172773.30149911577, Loss: 2.4630634784698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11596 , TFLOPS: 71.54095869304736, Tokens per sec: 177286.3765285562, Loss: 2.525435209274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11597 , TFLOPS: 68.57109513254183, Tokens per sec: 169926.72746814473, Loss: 2.505470037460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11598 , TFLOPS: 69.0844840828086, Tokens per sec: 171198.96184136742, Loss: 2.4701666831970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11599 , TFLOPS: 70.56319681301508, Tokens per sec: 174863.3748804735, Loss: 2.5087852478027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11600 , TFLOPS: 70.59079750233255, Tokens per sec: 174931.7724290691, Loss: 2.4780046939849854 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/optimizer.pt +[rank0]:[2024-08-27 02:30:38,386] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006368078000377864, 'preprocessing_with_comm': 0.0012868490011896938, 'state_converting': 0.9769758049951633, : 0.9860109050059691}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009600 + eval ppl=10.006909370422363, eval loss=2.3032758235931396 +------------------------------------------------------------------ +iteration: 11601 , TFLOPS: 63.088045114423004, Tokens per sec: 156339.12551542462, Loss: 2.492058753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11602 , TFLOPS: 70.65032511685843, Tokens per sec: 175079.28841537234, Loss: 2.4992079734802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11603 , TFLOPS: 70.7298412933285, Tokens per sec: 175276.33826009513, Loss: 2.4947879314422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11604 , TFLOPS: 68.70149206472047, Tokens per sec: 170249.8654304913, Loss: 2.482656240463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11605 , TFLOPS: 69.62181555843695, Tokens per sec: 172530.52842992288, Loss: 2.4888153076171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11606 , TFLOPS: 71.53090894875787, Tokens per sec: 177261.47215513617, Loss: 2.480599880218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11607 , TFLOPS: 69.05344189588794, Tokens per sec: 171122.03588259884, Loss: 2.4952077865600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11608 , TFLOPS: 67.79599125817678, Tokens per sec: 168005.93468272767, Loss: 2.4739034175872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11609 , TFLOPS: 71.52642060639651, Tokens per sec: 177250.34954833298, Loss: 2.481649875640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11610 , TFLOPS: 69.6659674278241, Tokens per sec: 172639.9416259942, Loss: 2.5128705501556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11611 , TFLOPS: 68.81560771010379, Tokens per sec: 170532.65657062727, Loss: 2.4863266944885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11612 , TFLOPS: 69.54151946753285, Tokens per sec: 172331.54587131794, Loss: 2.4789159297943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11613 , TFLOPS: 70.45111058797856, Tokens per sec: 174585.61286751603, Loss: 2.498579978942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11614 , TFLOPS: 68.97794061427493, Tokens per sec: 170934.935389609, Loss: 2.4930825233459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11615 , TFLOPS: 70.01218390935799, Tokens per sec: 173497.90420612905, Loss: 2.4986276626586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11616 , TFLOPS: 70.39286198344719, Tokens per sec: 174441.26640887378, Loss: 2.49416184425354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11617 , TFLOPS: 68.73469568814537, Tokens per sec: 170332.14766701928, Loss: 2.5000593662261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11618 , TFLOPS: 69.71792184096206, Tokens per sec: 172768.69038500113, Loss: 2.472273111343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11619 , TFLOPS: 69.55199567249207, Tokens per sec: 172357.50706125612, Loss: 2.515948534011841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11620 , TFLOPS: 70.6883519297951, Tokens per sec: 175173.5230468287, Loss: 2.4923319816589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11621 , TFLOPS: 68.24849322445267, Tokens per sec: 169127.28440236597, Loss: 2.4683098793029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11622 , TFLOPS: 70.67312488033903, Tokens per sec: 175135.78874087855, Loss: 2.490192174911499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11623 , TFLOPS: 69.57279534801758, Tokens per sec: 172409.05094848174, Loss: 2.4824862480163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11624 , TFLOPS: 68.83219030575975, Tokens per sec: 170573.75006938647, Loss: 2.4800233840942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11625 , TFLOPS: 69.4039684357393, Tokens per sec: 171990.6792620365, Loss: 2.5146687030792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11626 , TFLOPS: 70.4802293586782, Tokens per sec: 174657.77238900712, Loss: 2.503124475479126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11627 , TFLOPS: 69.24356968324871, Tokens per sec: 171593.19348398357, Loss: 2.486682415008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11628 , TFLOPS: 69.6443228024109, Tokens per sec: 172586.30385986794, Loss: 2.508955240249634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11629 , TFLOPS: 70.98982515535182, Tokens per sec: 175920.60690977555, Loss: 2.5181822776794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11630 , TFLOPS: 69.52364548520718, Tokens per sec: 172287.2521741332, Loss: 2.5000357627868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11631 , TFLOPS: 69.83402456033362, Tokens per sec: 173056.4057133742, Loss: 2.4840009212493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11632 , TFLOPS: 69.86485715908688, Tokens per sec: 173132.81228957593, Loss: 2.4855098724365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11633 , TFLOPS: 70.7506946452004, Tokens per sec: 175328.01516322, Loss: 2.5125174522399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11634 , TFLOPS: 67.85957639960942, Tokens per sec: 168163.50566767936, Loss: 2.49558687210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11635 , TFLOPS: 69.8516197287955, Tokens per sec: 173100.00847909058, Loss: 2.4673147201538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11636 , TFLOPS: 71.44923838993596, Tokens per sec: 177059.0835136207, Loss: 2.4810214042663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11637 , TFLOPS: 69.66604771264285, Tokens per sec: 172640.14058061916, Loss: 2.4995574951171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11638 , TFLOPS: 68.93996822502831, Tokens per sec: 170840.83562605188, Loss: 2.4666342735290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11639 , TFLOPS: 70.60677687824484, Tokens per sec: 174971.37108285335, Loss: 2.51842999458313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11640 , TFLOPS: 70.73486340318885, Tokens per sec: 175288.78360156016, Loss: 2.4900007247924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11641 , TFLOPS: 67.10745091949119, Tokens per sec: 166299.65587448498, Loss: 2.488457679748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11642 , TFLOPS: 70.76016896545144, Tokens per sec: 175351.49357248464, Loss: 2.4876532554626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11643 , TFLOPS: 71.52790808941084, Tokens per sec: 177254.0356951023, Loss: 2.4898650646209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11644 , TFLOPS: 68.54650002505277, Tokens per sec: 169865.7780824121, Loss: 2.485403537750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11645 , TFLOPS: 68.56135465554826, Tokens per sec: 169902.58949315257, Loss: 2.4833457469940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11646 , TFLOPS: 71.50086624992328, Tokens per sec: 177187.02303794754, Loss: 2.4881272315979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11647 , TFLOPS: 69.83895552383657, Tokens per sec: 173068.625184125, Loss: 2.4722084999084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11648 , TFLOPS: 68.9795176348695, Tokens per sec: 170938.8434203655, Loss: 2.480922222137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11649 , TFLOPS: 70.49058850578757, Tokens per sec: 174683.4434966984, Loss: 2.4938790798187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11650 , TFLOPS: 70.57296418386699, Tokens per sec: 174887.57950140923, Loss: 2.487652063369751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11651 , TFLOPS: 69.32595012670504, Tokens per sec: 171797.34129782335, Loss: 2.5139029026031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11652 , TFLOPS: 69.6655787300991, Tokens per sec: 172638.97839021304, Loss: 2.4783170223236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11653 , TFLOPS: 69.64654180985245, Tokens per sec: 172591.80280475196, Loss: 2.5060901641845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11654 , TFLOPS: 69.77280975783925, Tokens per sec: 172904.70868942598, Loss: 2.510695695877075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11655 , TFLOPS: 68.58753536323482, Tokens per sec: 169967.46817084294, Loss: 2.483760356903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11656 , TFLOPS: 70.49851541178283, Tokens per sec: 174703.08724297545, Loss: 2.474334716796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11657 , TFLOPS: 69.65190879219625, Tokens per sec: 172605.1027782218, Loss: 2.483652114868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11658 , TFLOPS: 68.79961106091795, Tokens per sec: 170493.01511176839, Loss: 2.512918472290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11659 , TFLOPS: 70.59466599895008, Tokens per sec: 174941.35898983743, Loss: 2.4923577308654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11660 , TFLOPS: 70.44872642959108, Tokens per sec: 174579.70466039513, Loss: 2.493757963180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11661 , TFLOPS: 68.00764536223072, Tokens per sec: 168530.43686819766, Loss: 2.4832987785339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11662 , TFLOPS: 70.68878946176974, Tokens per sec: 175174.60729925995, Loss: 2.5083746910095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11663 , TFLOPS: 70.48211114730294, Tokens per sec: 174662.43566851536, Loss: 2.4692599773406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11664 , TFLOPS: 68.87494298425182, Tokens per sec: 170679.69591628615, Loss: 2.490152359008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11665 , TFLOPS: 69.42467179985599, Tokens per sec: 172041.98447898147, Loss: 2.496997356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11666 , TFLOPS: 70.89970921236493, Tokens per sec: 175697.28967596343, Loss: 2.5113115310668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11667 , TFLOPS: 68.86468444916288, Tokens per sec: 170654.27413626484, Loss: 2.481140613555908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11668 , TFLOPS: 69.02136634228917, Tokens per sec: 171042.54912736712, Loss: 2.4915764331817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11669 , TFLOPS: 70.64472584840621, Tokens per sec: 175065.41281133742, Loss: 2.491422653198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11670 , TFLOPS: 70.6037132443439, Tokens per sec: 174963.7790605597, Loss: 2.48214054107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11671 , TFLOPS: 67.90561761870909, Tokens per sec: 168277.6008804673, Loss: 2.499081611633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11672 , TFLOPS: 69.63499996712252, Tokens per sec: 172563.20084702826, Loss: 2.481241226196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11673 , TFLOPS: 71.50834086792707, Tokens per sec: 177205.54596475812, Loss: 2.498199939727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11674 , TFLOPS: 69.27380558802871, Tokens per sec: 171668.12138678826, Loss: 2.5006349086761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11675 , TFLOPS: 68.74711102254375, Tokens per sec: 170362.91423332048, Loss: 2.4763617515563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11676 , TFLOPS: 71.48600604197777, Tokens per sec: 177150.19780567155, Loss: 2.4769794940948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11677 , TFLOPS: 69.68613031670704, Tokens per sec: 172689.9074857717, Loss: 2.5029613971710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11678 , TFLOPS: 67.84276240143826, Tokens per sec: 168121.83872799642, Loss: 2.511680841445923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11679 , TFLOPS: 70.63876186718883, Tokens per sec: 175050.63340889398, Loss: 2.4837024211883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11680 , TFLOPS: 70.7051156713316, Tokens per sec: 175215.0654450347, Loss: 2.475532293319702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11681 , TFLOPS: 69.05750842984838, Tokens per sec: 171132.1132017185, Loss: 2.485121965408325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11682 , TFLOPS: 68.90828625828328, Tokens per sec: 170762.32422240093, Loss: 2.487502098083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11683 , TFLOPS: 71.50014766657699, Tokens per sec: 177185.24230925698, Loss: 2.5009334087371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11684 , TFLOPS: 68.91495612148863, Tokens per sec: 170778.85287816988, Loss: 2.5039758682250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11685 , TFLOPS: 69.56635173332761, Tokens per sec: 172393.08296145726, Loss: 2.502352237701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11686 , TFLOPS: 70.59336747311251, Tokens per sec: 174938.14109976753, Loss: 2.491689920425415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11687 , TFLOPS: 70.70876100706305, Tokens per sec: 175224.09898854408, Loss: 2.475051164627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11688 , TFLOPS: 68.4052734334826, Tokens per sec: 169515.80303110788, Loss: 2.5212037563323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11689 , TFLOPS: 70.69322494447242, Tokens per sec: 175185.59891400594, Loss: 2.496746063232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11690 , TFLOPS: 69.52589164974032, Tokens per sec: 172292.8184172818, Loss: 2.497844696044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11691 , TFLOPS: 69.02856876497256, Tokens per sec: 171060.39752418894, Loss: 2.463808059692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11692 , TFLOPS: 69.62835976886161, Tokens per sec: 172546.74570425952, Loss: 2.4919567108154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11693 , TFLOPS: 70.52776382871322, Tokens per sec: 174775.56804210806, Loss: 2.500922679901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11694 , TFLOPS: 68.98696181774909, Tokens per sec: 170957.29092558558, Loss: 2.474574089050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11695 , TFLOPS: 68.80841749913894, Tokens per sec: 170514.83843578925, Loss: 2.4898521900177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11696 , TFLOPS: 71.51554878323975, Tokens per sec: 177223.40797292034, Loss: 2.4989781379699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11697 , TFLOPS: 68.07684708197748, Tokens per sec: 168701.92635292886, Loss: 2.523982286453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11698 , TFLOPS: 69.82010865468874, Tokens per sec: 173021.92056622266, Loss: 2.4942820072174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11699 , TFLOPS: 69.8947821711769, Tokens per sec: 173206.96976604723, Loss: 2.4909257888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11700 , TFLOPS: 69.73442877587482, Tokens per sec: 172809.59638810394, Loss: 2.5152173042297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11701 , TFLOPS: 68.4667390065284, Tokens per sec: 169668.1214921072, Loss: 2.4762041568756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11702 , TFLOPS: 69.71414204106598, Tokens per sec: 172759.3236244793, Loss: 2.5045032501220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11703 , TFLOPS: 71.49767901203667, Tokens per sec: 177179.12471136128, Loss: 2.48956561088562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11704 , TFLOPS: 68.74910443418123, Tokens per sec: 170367.85412695055, Loss: 2.4790778160095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11705 , TFLOPS: 69.07157829523605, Tokens per sec: 171166.97987807274, Loss: 2.501671314239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11706 , TFLOPS: 70.73705949799037, Tokens per sec: 175294.2257663981, Loss: 2.486543893814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11707 , TFLOPS: 70.73985878760577, Tokens per sec: 175301.1627144323, Loss: 2.4718503952026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11708 , TFLOPS: 67.7691836304436, Tokens per sec: 167939.50242810094, Loss: 2.4924817085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11709 , TFLOPS: 70.56108615917533, Tokens per sec: 174858.1444477509, Loss: 2.500023365020752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11710 , TFLOPS: 71.46648168612543, Tokens per sec: 177101.81430108412, Loss: 2.4679646492004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11711 , TFLOPS: 68.33876289698259, Tokens per sec: 169350.9825949204, Loss: 2.490689754486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11712 , TFLOPS: 68.79010270071771, Tokens per sec: 170469.45234775986, Loss: 2.4944891929626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11713 , TFLOPS: 71.51731552946681, Tokens per sec: 177227.78616469415, Loss: 2.4905452728271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11714 , TFLOPS: 69.71808179668182, Tokens per sec: 172769.08677289527, Loss: 2.488369941711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11715 , TFLOPS: 68.21122281572825, Tokens per sec: 169034.92422387237, Loss: 2.497894287109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11716 , TFLOPS: 70.90871398854503, Tokens per sec: 175719.6045032957, Loss: 2.4900269508361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11717 , TFLOPS: 69.8015850124188, Tokens per sec: 172976.0169401301, Loss: 2.500075340270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11718 , TFLOPS: 69.79300222687365, Tokens per sec: 172954.74785207736, Loss: 2.4963057041168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11719 , TFLOPS: 68.66958744762589, Tokens per sec: 170170.80227474723, Loss: 2.5105440616607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11720 , TFLOPS: 70.6557203823925, Tokens per sec: 175092.6584777033, Loss: 2.4935996532440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11721 , TFLOPS: 69.85261012472026, Tokens per sec: 173102.46278929323, Loss: 2.4856271743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11722 , TFLOPS: 68.70867349009188, Tokens per sec: 170267.66179366098, Loss: 2.471101999282837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11723 , TFLOPS: 70.43203755592768, Tokens per sec: 174538.347793026, Loss: 2.489267349243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11724 , TFLOPS: 69.79894121959926, Tokens per sec: 172969.46533028543, Loss: 2.4973936080932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11725 , TFLOPS: 68.79416091062838, Tokens per sec: 170479.50903896723, Loss: 2.475417137145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11726 , TFLOPS: 70.7146371309066, Tokens per sec: 175238.6606707369, Loss: 2.490220069885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11727 , TFLOPS: 69.75578344199155, Tokens per sec: 172862.51560315187, Loss: 2.48197603225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11728 , TFLOPS: 68.98617571272305, Tokens per sec: 170955.34287073443, Loss: 2.48876953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11729 , TFLOPS: 69.82292876164925, Tokens per sec: 173028.90910193123, Loss: 2.4773447513580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11730 , TFLOPS: 70.46594274637154, Tokens per sec: 174622.36858991094, Loss: 2.4881370067596436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11731 , TFLOPS: 68.9488713967185, Tokens per sec: 170862.8986662655, Loss: 2.4958057403564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11732 , TFLOPS: 69.85531964620002, Tokens per sec: 173109.1772819985, Loss: 2.4894416332244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11733 , TFLOPS: 70.96161982508309, Tokens per sec: 175850.71099429607, Loss: 2.4881114959716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11734 , TFLOPS: 68.78120591887894, Tokens per sec: 170447.40514230198, Loss: 2.4839179515838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11735 , TFLOPS: 68.92382184160915, Tokens per sec: 170800.82310926824, Loss: 2.4864726066589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11736 , TFLOPS: 70.6531394014863, Tokens per sec: 175086.26252269925, Loss: 2.499152660369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11737 , TFLOPS: 69.63363403083125, Tokens per sec: 172559.81590642818, Loss: 2.4879138469696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11738 , TFLOPS: 69.26925548042072, Tokens per sec: 171656.84571889995, Loss: 2.4923088550567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11739 , TFLOPS: 69.75927822269067, Tokens per sec: 172871.17605470616, Loss: 2.5024914741516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11740 , TFLOPS: 70.65103245239295, Tokens per sec: 175081.04127074633, Loss: 2.511443614959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11741 , TFLOPS: 68.70019474945538, Tokens per sec: 170246.65054035335, Loss: 2.4802350997924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11742 , TFLOPS: 68.76763699972932, Tokens per sec: 170413.77986590937, Loss: 2.474625587463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11743 , TFLOPS: 71.49781408169704, Tokens per sec: 177179.45942885886, Loss: 2.490953207015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11744 , TFLOPS: 69.87589416074204, Tokens per sec: 173160.1631955035, Loss: 2.4992752075195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11745 , TFLOPS: 68.04095456418416, Tokens per sec: 168612.98073994982, Loss: 2.487410068511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11746 , TFLOPS: 70.73094359963092, Tokens per sec: 175279.06989654156, Loss: 2.5174312591552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11747 , TFLOPS: 70.58184267556761, Tokens per sec: 174909.58138189002, Loss: 2.5024514198303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11748 , TFLOPS: 69.0524175625516, Tokens per sec: 171119.4974717517, Loss: 2.5049524307250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11749 , TFLOPS: 68.59327963579103, Tokens per sec: 169981.70311102105, Loss: 2.523362159729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11750 , TFLOPS: 71.48265313090963, Tokens per sec: 177141.88892269105, Loss: 2.4946019649505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11751 , TFLOPS: 69.85604958768369, Tokens per sec: 173110.98615740472, Loss: 2.488722085952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11752 , TFLOPS: 68.0548253282594, Tokens per sec: 168647.35402131907, Loss: 2.491917133331299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11753 , TFLOPS: 70.6594884638444, Tokens per sec: 175101.99619862958, Loss: 2.4894983768463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11754 , TFLOPS: 70.57600091460297, Tokens per sec: 174895.10485469652, Loss: 2.5022213459014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11755 , TFLOPS: 68.28347609581785, Tokens per sec: 169213.97581129026, Loss: 2.4923794269561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11756 , TFLOPS: 69.6667243386148, Tokens per sec: 172641.81733431417, Loss: 2.4976747035980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11757 , TFLOPS: 70.71768958034136, Tokens per sec: 175246.22497669127, Loss: 2.5071463584899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11758 , TFLOPS: 68.97874606959398, Tokens per sec: 170936.93139661782, Loss: 2.5030808448791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11759 , TFLOPS: 69.71407529543072, Tokens per sec: 172759.15822144249, Loss: 2.508204460144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11760 , TFLOPS: 70.5908373904863, Tokens per sec: 174931.87127630817, Loss: 2.5040719509124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11761 , TFLOPS: 70.05304324321631, Tokens per sec: 173599.15813645718, Loss: 2.4900100231170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11762 , TFLOPS: 68.14678267920951, Tokens per sec: 168875.23446691188, Loss: 2.5126664638519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11763 , TFLOPS: 71.46525447565405, Tokens per sec: 177098.7731383476, Loss: 2.4953176975250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11764 , TFLOPS: 68.72758907337374, Tokens per sec: 170314.53669275693, Loss: 2.512448310852051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11765 , TFLOPS: 69.90380654395548, Tokens per sec: 173229.3331559093, Loss: 2.48498272895813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11766 , TFLOPS: 70.02713547235872, Tokens per sec: 173534.9558262955, Loss: 2.517815351486206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11767 , TFLOPS: 69.58328794769179, Tokens per sec: 172435.0527663297, Loss: 2.5269553661346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11768 , TFLOPS: 69.06411507480534, Tokens per sec: 171148.48519570386, Loss: 2.4950997829437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11769 , TFLOPS: 69.57912616471629, Tokens per sec: 172424.73940965816, Loss: 2.500807285308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11770 , TFLOPS: 71.48398899936456, Tokens per sec: 177145.1993518804, Loss: 2.5057849884033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11771 , TFLOPS: 68.68228735789067, Tokens per sec: 170202.274051394, Loss: 2.5153560638427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11772 , TFLOPS: 68.83548947811099, Tokens per sec: 170581.92578190667, Loss: 2.478994369506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11773 , TFLOPS: 70.72904030855963, Tokens per sec: 175274.35333160212, Loss: 2.4710004329681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11774 , TFLOPS: 69.64817087158666, Tokens per sec: 172595.83979918528, Loss: 2.4776711463928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11775 , TFLOPS: 67.96797797793555, Tokens per sec: 168432.13672019113, Loss: 2.4969542026519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11776 , TFLOPS: 70.66223622539232, Tokens per sec: 175108.80545443625, Loss: 2.494478225708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11777 , TFLOPS: 71.53997029388262, Tokens per sec: 177283.92716654483, Loss: 2.4932451248168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11778 , TFLOPS: 68.29140878909813, Tokens per sec: 169233.6338990971, Loss: 2.4803473949432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11779 , TFLOPS: 69.71151277071797, Tokens per sec: 172752.80800291893, Loss: 2.4930810928344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11780 , TFLOPS: 71.4795638477357, Tokens per sec: 177134.2333386734, Loss: 2.4944441318511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11781 , TFLOPS: 69.784035320648, Tokens per sec: 172932.5268706639, Loss: 2.506260871887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11782 , TFLOPS: 68.52603335462086, Tokens per sec: 169815.059418492, Loss: 2.4696450233459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11783 , TFLOPS: 70.92103271789915, Tokens per sec: 175750.13167165528, Loss: 2.51802134513855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11784 , TFLOPS: 69.82398388211539, Tokens per sec: 173031.52380667743, Loss: 2.4693763256073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11785 , TFLOPS: 69.86842536416164, Tokens per sec: 173141.65469482142, Loss: 2.4885172843933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11786 , TFLOPS: 68.40322039766866, Tokens per sec: 169510.71538220037, Loss: 2.5069806575775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11787 , TFLOPS: 70.57708130765363, Tokens per sec: 174897.7821876912, Loss: 2.49399471282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11788 , TFLOPS: 69.83747958583206, Tokens per sec: 173064.9676471618, Loss: 2.4812047481536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11789 , TFLOPS: 68.08381596780235, Tokens per sec: 168719.1960196891, Loss: 2.4634170532226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11790 , TFLOPS: 71.52222071611855, Tokens per sec: 177239.9417575681, Loss: 2.4945473670959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11791 , TFLOPS: 69.76438501285908, Tokens per sec: 172883.83124330288, Loss: 2.4899938106536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11792 , TFLOPS: 68.93177016336391, Tokens per sec: 170820.51992615644, Loss: 2.4788408279418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11793 , TFLOPS: 69.63210187308815, Tokens per sec: 172556.01905075967, Loss: 2.4876110553741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11794 , TFLOPS: 70.53885445113947, Tokens per sec: 174803.0518262132, Loss: 2.4949893951416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11795 , TFLOPS: 67.93306026726864, Tokens per sec: 168345.60678665517, Loss: 2.4664666652679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11796 , TFLOPS: 70.71388409256922, Tokens per sec: 175236.79455878283, Loss: 2.4799513816833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11797 , TFLOPS: 68.92838307198838, Tokens per sec: 170812.126340609, Loss: 2.495751142501831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11798 , TFLOPS: 68.97988801961418, Tokens per sec: 170939.76127456408, Loss: 2.4939889907836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11799 , TFLOPS: 70.04414763017483, Tokens per sec: 173577.1138274077, Loss: 2.481656551361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11800 , TFLOPS: 70.94192483130246, Tokens per sec: 175801.90463012495, Loss: 2.5030715465545654 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/optimizer.pt +[rank0]:[2024-08-27 02:56:04,827] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006505146011477336, 'preprocessing_with_comm': 0.0015232909936457872, 'state_converting': 0.9764622299990151, : 0.985880817999714}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800, took 4.58s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0009800 + eval ppl=10.091530799865723, eval loss=2.3116965293884277 +------------------------------------------------------------------ +iteration: 11801 , TFLOPS: 64.32866777936648, Tokens per sec: 159413.52514502316, Loss: 2.4841954708099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11802 , TFLOPS: 69.54471544916811, Tokens per sec: 172339.46586587655, Loss: 2.509521007537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11803 , TFLOPS: 70.90321867718995, Tokens per sec: 175705.98651639928, Loss: 2.486128091812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11804 , TFLOPS: 68.89680986155437, Tokens per sec: 170733.88444707726, Loss: 2.5243029594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11805 , TFLOPS: 67.91701818575568, Tokens per sec: 168305.8527414849, Loss: 2.4897241592407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11806 , TFLOPS: 70.56339641813764, Tokens per sec: 174863.8695239555, Loss: 2.492332935333252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11807 , TFLOPS: 70.59231827309327, Tokens per sec: 174935.54106654093, Loss: 2.4995439052581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11808 , TFLOPS: 68.75407639762741, Tokens per sec: 170380.1752000772, Loss: 2.4801158905029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11809 , TFLOPS: 68.7281536271263, Tokens per sec: 170315.93571914654, Loss: 2.510286331176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11810 , TFLOPS: 71.45769734409387, Tokens per sec: 177080.04573385473, Loss: 2.4930784702301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11811 , TFLOPS: 69.79901099868465, Tokens per sec: 172969.63825054583, Loss: 2.489403247833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11812 , TFLOPS: 67.92940019891122, Tokens per sec: 168336.53673407566, Loss: 2.473970890045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11813 , TFLOPS: 70.60116173561593, Tokens per sec: 174957.45614085987, Loss: 2.507279634475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11814 , TFLOPS: 70.68380981799284, Tokens per sec: 175162.26719342775, Loss: 2.5073933601379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11815 , TFLOPS: 68.07378976930038, Tokens per sec: 168694.34999532424, Loss: 2.5209362506866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11816 , TFLOPS: 70.7549676188927, Tokens per sec: 175338.6040627365, Loss: 2.4803664684295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11817 , TFLOPS: 70.57653779047475, Tokens per sec: 174896.43529224856, Loss: 2.488614082336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11818 , TFLOPS: 69.70004772485, Tokens per sec: 172724.396356279, Loss: 2.460447311401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11819 , TFLOPS: 67.62340750480547, Tokens per sec: 167578.2531302037, Loss: 2.491358995437622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11820 , TFLOPS: 71.50749260015333, Tokens per sec: 177203.44386377034, Loss: 2.513792037963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11821 , TFLOPS: 70.14071328396957, Tokens per sec: 173816.41415509602, Loss: 2.473158359527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11822 , TFLOPS: 68.2507547676896, Tokens per sec: 169132.8887556366, Loss: 2.4892921447753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11823 , TFLOPS: 70.54134808216483, Tokens per sec: 174809.23131859023, Loss: 2.4886865615844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11824 , TFLOPS: 70.60440341417484, Tokens per sec: 174965.48937742927, Loss: 2.489743232727051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11825 , TFLOPS: 68.82137850408017, Tokens per sec: 170546.9572338062, Loss: 2.4926419258117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11826 , TFLOPS: 69.77807672815491, Tokens per sec: 172917.7608220729, Loss: 2.500500202178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11827 , TFLOPS: 69.5444952564372, Tokens per sec: 172338.92020403311, Loss: 2.5012152194976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11828 , TFLOPS: 69.91237235639544, Tokens per sec: 173250.56018273762, Loss: 2.490792751312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11829 , TFLOPS: 68.96834426038733, Tokens per sec: 170911.15456754985, Loss: 2.479346990585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11830 , TFLOPS: 70.3859934879957, Tokens per sec: 174424.24552051778, Loss: 2.4882521629333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11831 , TFLOPS: 69.59549680653022, Tokens per sec: 172465.30766344818, Loss: 2.496912717819214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11832 , TFLOPS: 68.65054998659191, Tokens per sec: 170123.62534915557, Loss: 2.50384521484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11833 , TFLOPS: 70.60788449541383, Tokens per sec: 174974.11588021222, Loss: 2.484581708908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11834 , TFLOPS: 69.69403461184201, Tokens per sec: 172709.49519984616, Loss: 2.4796531200408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11835 , TFLOPS: 68.20609385416594, Tokens per sec: 169022.21409211922, Loss: 2.502579689025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11836 , TFLOPS: 70.72708191286104, Tokens per sec: 175269.5002113259, Loss: 2.487344264984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11837 , TFLOPS: 70.4435819391286, Tokens per sec: 174566.9560463167, Loss: 2.4974822998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11838 , TFLOPS: 69.0032276274827, Tokens per sec: 170997.5993939323, Loss: 2.4997012615203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11839 , TFLOPS: 68.66999991826879, Tokens per sec: 170171.82442243773, Loss: 2.4850990772247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11840 , TFLOPS: 71.50648691102975, Tokens per sec: 177200.95165533636, Loss: 2.482842445373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11841 , TFLOPS: 68.95108439113467, Tokens per sec: 170868.38271021022, Loss: 2.499098777770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11842 , TFLOPS: 68.97054967299535, Tokens per sec: 170916.61982293878, Loss: 2.4965627193450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11843 , TFLOPS: 70.97244238734429, Tokens per sec: 175877.53049578294, Loss: 2.502429962158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11844 , TFLOPS: 70.57550615664783, Tokens per sec: 174893.8787899816, Loss: 2.496572971343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11845 , TFLOPS: 69.49942709220541, Tokens per sec: 172227.23632839858, Loss: 2.477614402770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11846 , TFLOPS: 69.06867086240325, Tokens per sec: 171159.77493923288, Loss: 2.484626054763794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11847 , TFLOPS: 70.69104004641756, Tokens per sec: 175180.18449593958, Loss: 2.488255023956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11848 , TFLOPS: 70.73179844336046, Tokens per sec: 175281.18829347217, Loss: 2.477773666381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11849 , TFLOPS: 68.13047219484059, Tokens per sec: 168834.81528989752, Loss: 2.5257999897003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11850 , TFLOPS: 71.51491990741474, Tokens per sec: 177221.84954934957, Loss: 2.488729238510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11851 , TFLOPS: 70.61821376220546, Tokens per sec: 174999.71294118455, Loss: 2.4827001094818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11852 , TFLOPS: 67.91003588524329, Tokens per sec: 168288.5498316513, Loss: 2.498753309249878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11853 , TFLOPS: 70.53958899201746, Tokens per sec: 174804.87209942524, Loss: 2.494460344314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11854 , TFLOPS: 70.75594123520564, Tokens per sec: 175341.0167912126, Loss: 2.4684524536132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11855 , TFLOPS: 67.99496107944458, Tokens per sec: 168499.00381816452, Loss: 2.4960386753082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11856 , TFLOPS: 69.68635433316854, Tokens per sec: 172690.46262324072, Loss: 2.5048091411590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11857 , TFLOPS: 71.49901085549774, Tokens per sec: 177182.42516617253, Loss: 2.488281488418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11858 , TFLOPS: 68.74611741287775, Tokens per sec: 170360.45195911298, Loss: 2.504997491836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11859 , TFLOPS: 69.76282125564391, Tokens per sec: 172879.9560806616, Loss: 2.5021469593048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11860 , TFLOPS: 69.92175533821866, Tokens per sec: 173273.81224531634, Loss: 2.514488697052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11861 , TFLOPS: 69.9025382099432, Tokens per sec: 173226.19008450833, Loss: 2.4900195598602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11862 , TFLOPS: 68.85368056266651, Tokens per sec: 170627.00529334863, Loss: 2.4750077724456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11863 , TFLOPS: 70.7524614154018, Tokens per sec: 175332.39341440488, Loss: 2.4930312633514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11864 , TFLOPS: 69.53913407286944, Tokens per sec: 172325.6346005691, Loss: 2.477536916732788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11865 , TFLOPS: 69.70617812812341, Tokens per sec: 172739.5881709102, Loss: 2.5032644271850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11866 , TFLOPS: 68.89203112553012, Tokens per sec: 170722.04221278793, Loss: 2.4920670986175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11867 , TFLOPS: 69.80308964750529, Tokens per sec: 172979.74559162365, Loss: 2.502823829650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11868 , TFLOPS: 70.61062439322072, Tokens per sec: 174980.90564880232, Loss: 2.485398292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11869 , TFLOPS: 68.06467929679012, Tokens per sec: 168671.77324084242, Loss: 2.4804434776306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11870 , TFLOPS: 71.50178107732154, Tokens per sec: 177189.290081017, Loss: 2.4979255199432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11871 , TFLOPS: 70.46540710153151, Tokens per sec: 174621.0412029912, Loss: 2.48087739944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11872 , TFLOPS: 67.99790250704734, Tokens per sec: 168506.29299978918, Loss: 2.507305145263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11873 , TFLOPS: 70.59435473885827, Tokens per sec: 174940.58765304182, Loss: 2.508760452270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11874 , TFLOPS: 69.66821169636665, Tokens per sec: 172645.50317066925, Loss: 2.4900851249694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11875 , TFLOPS: 68.97606636894837, Tokens per sec: 170930.29080322463, Loss: 2.483187675476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11876 , TFLOPS: 69.89603937202227, Tokens per sec: 173210.08524823416, Loss: 2.4907095432281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11877 , TFLOPS: 71.56057031272486, Tokens per sec: 177334.97628251536, Loss: 2.478318691253662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11878 , TFLOPS: 68.83770574383425, Tokens per sec: 170587.41793251073, Loss: 2.526824712753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11879 , TFLOPS: 69.70150364529201, Tokens per sec: 172728.0042875172, Loss: 2.498424530029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11880 , TFLOPS: 70.63503990164138, Tokens per sec: 175041.40996825902, Loss: 2.487179756164551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11881 , TFLOPS: 69.8857543532351, Tokens per sec: 173184.5978386912, Loss: 2.5045642852783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11882 , TFLOPS: 67.81240357046237, Tokens per sec: 168046.6062005368, Loss: 2.4976601600646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11883 , TFLOPS: 70.70136136910386, Tokens per sec: 175205.76187054382, Loss: 2.527085542678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11884 , TFLOPS: 70.6049346208504, Tokens per sec: 174966.80576608854, Loss: 2.4871394634246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11885 , TFLOPS: 69.84207047988953, Tokens per sec: 173076.344388365, Loss: 2.4877119064331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11886 , TFLOPS: 68.78434882133826, Tokens per sec: 170455.19360081543, Loss: 2.48425555229187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11887 , TFLOPS: 71.50167774296276, Tokens per sec: 177189.03400709273, Loss: 2.487903356552124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11888 , TFLOPS: 70.17247561879223, Tokens per sec: 173895.12471969659, Loss: 2.5016324520111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11889 , TFLOPS: 67.67301240545648, Tokens per sec: 167701.17953844782, Loss: 2.4643945693969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11890 , TFLOPS: 70.87853326775107, Tokens per sec: 175644.81335248743, Loss: 2.4870524406433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11891 , TFLOPS: 70.62504132045825, Tokens per sec: 175016.63238265258, Loss: 2.4925739765167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11892 , TFLOPS: 69.09510308062387, Tokens per sec: 171225.2768877328, Loss: 2.503772497177124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11893 , TFLOPS: 68.42806523745504, Tokens per sec: 169572.28363207605, Loss: 2.5229954719543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11894 , TFLOPS: 70.68308279243564, Tokens per sec: 175160.46554400853, Loss: 2.494457721710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11895 , TFLOPS: 69.60964111160601, Tokens per sec: 172500.3588095498, Loss: 2.5184452533721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11896 , TFLOPS: 69.13668258140707, Tokens per sec: 171328.3154710337, Loss: 2.484368324279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11897 , TFLOPS: 69.69772866228044, Tokens per sec: 172718.64946376582, Loss: 2.498518943786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11898 , TFLOPS: 69.74329358093057, Tokens per sec: 172831.56435157097, Loss: 2.486370086669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11899 , TFLOPS: 69.2137547232213, Tokens per sec: 171519.30872864666, Loss: 2.4933900833129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11900 , TFLOPS: 70.67908004030429, Tokens per sec: 175150.5462832875, Loss: 2.4982147216796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11901 , TFLOPS: 69.6028957784761, Tokens per sec: 172483.64313099408, Loss: 2.507449150085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11902 , TFLOPS: 68.25303257176152, Tokens per sec: 169138.53340504767, Loss: 2.5094680786132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11903 , TFLOPS: 70.63563725455492, Tokens per sec: 175042.89027458263, Loss: 2.489685297012329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11904 , TFLOPS: 70.57266518452465, Tokens per sec: 174886.83854810146, Loss: 2.493009090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11905 , TFLOPS: 69.29233181359254, Tokens per sec: 171714.03141456767, Loss: 2.487455368041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11906 , TFLOPS: 68.88685170806377, Tokens per sec: 170709.20704574636, Loss: 2.4763848781585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11907 , TFLOPS: 71.53264723698808, Tokens per sec: 177265.77982485885, Loss: 2.5002152919769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11908 , TFLOPS: 68.7918923782441, Tokens per sec: 170473.88736581954, Loss: 2.4833221435546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11909 , TFLOPS: 68.94780105722218, Tokens per sec: 170860.24624708598, Loss: 2.4897446632385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11910 , TFLOPS: 70.64145837266625, Tokens per sec: 175057.31564651162, Loss: 2.463972568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11911 , TFLOPS: 69.72224742015985, Tokens per sec: 172779.4096467551, Loss: 2.490631103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11912 , TFLOPS: 69.72175133822344, Tokens per sec: 172778.1803010688, Loss: 2.4811089038848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11913 , TFLOPS: 69.05379824101934, Tokens per sec: 171122.91894508907, Loss: 2.486961841583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11914 , TFLOPS: 70.62394258588759, Tokens per sec: 175013.9095973509, Loss: 2.453935384750366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11915 , TFLOPS: 69.51255916893962, Tokens per sec: 172259.77906116366, Loss: 2.4944701194763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11916 , TFLOPS: 68.35407177237117, Tokens per sec: 169388.91967454518, Loss: 2.5019912719726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11917 , TFLOPS: 71.47231455533026, Tokens per sec: 177116.26879351723, Loss: 2.4896178245544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11918 , TFLOPS: 70.68782938626451, Tokens per sec: 175172.22812639695, Loss: 2.466510772705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11919 , TFLOPS: 67.32500251227351, Tokens per sec: 166838.77268668593, Loss: 2.502476930618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11920 , TFLOPS: 70.73772962272504, Tokens per sec: 175295.88640931097, Loss: 2.49434757232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11921 , TFLOPS: 70.20688854301576, Tokens per sec: 173980.40373681905, Loss: 2.5114660263061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11922 , TFLOPS: 68.11737570265213, Tokens per sec: 168802.3607395568, Loss: 2.5209553241729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11923 , TFLOPS: 69.65015371259425, Tokens per sec: 172600.75349763007, Loss: 2.5056142807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11924 , TFLOPS: 71.50955195091117, Tokens per sec: 177208.5471618045, Loss: 2.480823516845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11925 , TFLOPS: 69.06138437890205, Tokens per sec: 171141.7182304445, Loss: 2.507833957672119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11926 , TFLOPS: 68.75507699644336, Tokens per sec: 170382.65479416816, Loss: 2.4969875812530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11927 , TFLOPS: 71.02461356185034, Tokens per sec: 176006.81641333824, Loss: 2.500054359436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11928 , TFLOPS: 69.72312186322317, Tokens per sec: 172781.5766129939, Loss: 2.5073776245117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11929 , TFLOPS: 68.80602010045277, Tokens per sec: 170508.89741774966, Loss: 2.4941208362579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11930 , TFLOPS: 69.56533374412543, Tokens per sec: 172390.56027207346, Loss: 2.488605260848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11931 , TFLOPS: 70.70986268456883, Tokens per sec: 175226.8290667632, Loss: 2.503816843032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11932 , TFLOPS: 69.8921126482604, Tokens per sec: 173200.35439418792, Loss: 2.495176076889038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11933 , TFLOPS: 68.88760542550706, Tokens per sec: 170711.07484059967, Loss: 2.49054217338562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11934 , TFLOPS: 70.52401430131202, Tokens per sec: 174766.27630016336, Loss: 2.4863507747650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11935 , TFLOPS: 69.89870868743091, Tokens per sec: 173216.70010586613, Loss: 2.4831738471984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11936 , TFLOPS: 68.01728256765209, Tokens per sec: 168554.31892485765, Loss: 2.488483428955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11937 , TFLOPS: 71.50005826103494, Tokens per sec: 177185.0207524749, Loss: 2.497602939605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11938 , TFLOPS: 69.6373499837362, Tokens per sec: 172569.02444707265, Loss: 2.486746311187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11939 , TFLOPS: 68.09013428970884, Tokens per sec: 168734.853517394, Loss: 2.4955105781555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11940 , TFLOPS: 70.7365005002153, Tokens per sec: 175292.8405083328, Loss: 2.5004000663757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11941 , TFLOPS: 69.6214242586403, Tokens per sec: 172529.5587459216, Loss: 2.4881327152252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11942 , TFLOPS: 68.94937841497112, Tokens per sec: 170864.15511334816, Loss: 2.480673313140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11943 , TFLOPS: 69.9125534052687, Tokens per sec: 173251.00884178974, Loss: 2.4589619636535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11944 , TFLOPS: 71.5069681907099, Tokens per sec: 177202.14431940106, Loss: 2.493932008743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11945 , TFLOPS: 68.09673838818568, Tokens per sec: 168751.2192009211, Loss: 2.4945662021636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11946 , TFLOPS: 69.57560849467465, Tokens per sec: 172416.02223579166, Loss: 2.5078060626983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11947 , TFLOPS: 70.00672030214324, Tokens per sec: 173484.36478558488, Loss: 2.507338762283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11948 , TFLOPS: 69.63379155758518, Tokens per sec: 172560.20627507742, Loss: 2.4825756549835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11949 , TFLOPS: 68.93007916967161, Tokens per sec: 170816.32945750863, Loss: 2.487565040588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11950 , TFLOPS: 70.65750430817742, Tokens per sec: 175097.079242314, Loss: 2.486250400543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11951 , TFLOPS: 70.73527526683195, Tokens per sec: 175289.80424503822, Loss: 2.5172512531280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11952 , TFLOPS: 68.91208193297726, Tokens per sec: 170771.73032242068, Loss: 2.4915614128112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11953 , TFLOPS: 68.81542167190065, Tokens per sec: 170532.1955474661, Loss: 2.48831844329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11954 , TFLOPS: 71.54407051373093, Tokens per sec: 177294.08796300547, Loss: 2.475965738296509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11955 , TFLOPS: 69.65804061877697, Tokens per sec: 172620.29811996527, Loss: 2.4862735271453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11956 , TFLOPS: 67.22358276286249, Tokens per sec: 166587.44337533772, Loss: 2.4858438968658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11957 , TFLOPS: 71.43098904211759, Tokens per sec: 177013.85961939517, Loss: 2.4898502826690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11958 , TFLOPS: 70.7016440968536, Tokens per sec: 175206.4625010527, Loss: 2.4941530227661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11959 , TFLOPS: 68.85740092648936, Tokens per sec: 170636.22476473358, Loss: 2.4888672828674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11960 , TFLOPS: 68.91113512321068, Tokens per sec: 170769.38402351947, Loss: 2.5206713676452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11961 , TFLOPS: 70.63486828471747, Tokens per sec: 175040.98468261643, Loss: 2.4894940853118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11962 , TFLOPS: 69.80071471065048, Tokens per sec: 172973.86023647655, Loss: 2.487454891204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11963 , TFLOPS: 68.31563976646534, Tokens per sec: 169293.68093027556, Loss: 2.4954466819763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11964 , TFLOPS: 71.51110060839487, Tokens per sec: 177212.38490564784, Loss: 2.48422908782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11965 , TFLOPS: 69.83046209691837, Tokens per sec: 173047.57753659348, Loss: 2.4740962982177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11966 , TFLOPS: 68.89112271355398, Tokens per sec: 170719.7910678412, Loss: 2.477421760559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11967 , TFLOPS: 69.20459864126349, Tokens per sec: 171496.61894892954, Loss: 2.4843785762786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11968 , TFLOPS: 70.64699350028205, Tokens per sec: 175071.0323024882, Loss: 2.4877264499664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11969 , TFLOPS: 68.32788634160461, Tokens per sec: 169324.02929254228, Loss: 2.504788398742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11970 , TFLOPS: 70.5580911083576, Tokens per sec: 174850.7223818925, Loss: 2.5180556774139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11971 , TFLOPS: 70.48804237009941, Tokens per sec: 174677.13389198313, Loss: 2.475454807281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11972 , TFLOPS: 69.19804697994098, Tokens per sec: 171480.3832104473, Loss: 2.489871025085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11973 , TFLOPS: 68.38751582735102, Tokens per sec: 169471.79772695206, Loss: 2.480250358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11974 , TFLOPS: 71.46734373175764, Tokens per sec: 177103.95054512288, Loss: 2.4957869052886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11975 , TFLOPS: 68.79942462652613, Tokens per sec: 170492.55310680802, Loss: 2.5020041465759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11976 , TFLOPS: 68.81449686298755, Tokens per sec: 170529.90376910334, Loss: 2.4956302642822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11977 , TFLOPS: 70.98918395112095, Tokens per sec: 175919.01793505703, Loss: 2.4870080947875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11978 , TFLOPS: 69.69788629570436, Tokens per sec: 172719.04009675502, Loss: 2.511613130569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11979 , TFLOPS: 69.84013819312572, Tokens per sec: 173071.5559689053, Loss: 2.503441572189331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11980 , TFLOPS: 69.24663367388948, Tokens per sec: 171600.7863903178, Loss: 2.51102614402771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11981 , TFLOPS: 70.690422652361, Tokens per sec: 175178.65452545587, Loss: 2.482982635498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11982 , TFLOPS: 70.52162464268439, Tokens per sec: 174760.35446284144, Loss: 2.5029971599578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11983 , TFLOPS: 68.20428824868227, Tokens per sec: 169017.73960282683, Loss: 2.497218370437622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11984 , TFLOPS: 71.49410622575867, Tokens per sec: 177170.2709533895, Loss: 2.5019657611846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11985 , TFLOPS: 69.53620809677032, Tokens per sec: 172318.3837094726, Loss: 2.4877254962921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11986 , TFLOPS: 68.29141732484682, Tokens per sec: 169233.65505162266, Loss: 2.5139541625976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11987 , TFLOPS: 70.65687915448196, Tokens per sec: 175095.53004259028, Loss: 2.4856250286102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11988 , TFLOPS: 70.70763599176802, Tokens per sec: 175221.31107671236, Loss: 2.4681060314178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11989 , TFLOPS: 68.79043691977569, Tokens per sec: 170470.28057940278, Loss: 2.4925308227539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11990 , TFLOPS: 69.79965244376439, Tokens per sec: 172971.22782211445, Loss: 2.494157314300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11991 , TFLOPS: 71.51515054215908, Tokens per sec: 177222.42108765253, Loss: 2.48606014251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11992 , TFLOPS: 68.93167432145712, Tokens per sec: 170820.28241935358, Loss: 2.481173038482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11993 , TFLOPS: 68.69523372200501, Tokens per sec: 170234.35656782906, Loss: 2.510904312133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11994 , TFLOPS: 70.8133465311124, Tokens per sec: 175483.27343818496, Loss: 2.4958066940307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11995 , TFLOPS: 69.80978231617887, Tokens per sec: 172996.3307618551, Loss: 2.4863176345825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11996 , TFLOPS: 68.70660393219075, Tokens per sec: 170262.53320120135, Loss: 2.5023701190948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11997 , TFLOPS: 69.66455149146317, Tokens per sec: 172636.43277971353, Loss: 2.4947407245635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11998 , TFLOPS: 70.5942398537639, Tokens per sec: 174940.30295512232, Loss: 2.50297474861145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11999 , TFLOPS: 69.75566335685006, Tokens per sec: 172862.21801894283, Loss: 2.513352632522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12000 , TFLOPS: 68.72526560395708, Tokens per sec: 170308.77887959118, Loss: 2.485687732696533 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/model.pt +[rank0]:[2024-08-27 03:21:31,001] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006369211987475865, 'preprocessing_with_comm': 0.0013768320059170946, 'state_converting': 0.9748536909901304, : 0.9839656319963979}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010000 + eval ppl=10.765033721923828, eval loss=2.376303195953369 +------------------------------------------------------------------ +iteration: 12001 , TFLOPS: 65.48729869257262, Tokens per sec: 162284.74018167952, Loss: 2.4772300720214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12002 , TFLOPS: 69.75415306433467, Tokens per sec: 172858.47534771718, Loss: 2.5039844512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12003 , TFLOPS: 67.61973041034068, Tokens per sec: 167569.14088505457, Loss: 2.503650665283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12004 , TFLOPS: 71.46857402395295, Tokens per sec: 177106.99934471076, Loss: 2.5054407119750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12005 , TFLOPS: 70.66746983557245, Tokens per sec: 175121.77491699232, Loss: 2.4878132343292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12006 , TFLOPS: 68.23379314312614, Tokens per sec: 169090.85598149113, Loss: 2.4914798736572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12007 , TFLOPS: 69.45851131839747, Tokens per sec: 172125.84253365797, Loss: 2.4753055572509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12008 , TFLOPS: 70.59179384475618, Tokens per sec: 174934.24147535107, Loss: 2.4935832023620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12009 , TFLOPS: 68.05626199392114, Tokens per sec: 168650.91423709065, Loss: 2.5081002712249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12010 , TFLOPS: 70.60112418005225, Tokens per sec: 174957.36307403602, Loss: 2.4882972240448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12011 , TFLOPS: 70.33765336491251, Tokens per sec: 174304.45337041348, Loss: 2.466503858566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12012 , TFLOPS: 68.87115199979539, Tokens per sec: 170670.3014391917, Loss: 2.462242364883423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12013 , TFLOPS: 68.70137050072327, Tokens per sec: 170249.56418151487, Loss: 2.501770496368408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12014 , TFLOPS: 71.5303192327211, Tokens per sec: 177260.01077383445, Loss: 2.476433038711548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12015 , TFLOPS: 69.57075741306174, Tokens per sec: 172404.00072116414, Loss: 2.4961111545562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12016 , TFLOPS: 68.19368847485596, Tokens per sec: 168991.47219562373, Loss: 2.4809682369232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12017 , TFLOPS: 71.47423030871823, Tokens per sec: 177121.01624145033, Loss: 2.5050554275512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12018 , TFLOPS: 69.55133792090467, Tokens per sec: 172355.87708036523, Loss: 2.4952659606933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12019 , TFLOPS: 68.87206984761691, Tokens per sec: 170672.57596720272, Loss: 2.498375177383423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12020 , TFLOPS: 69.81867143787274, Tokens per sec: 173018.35898463006, Loss: 2.488344192504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12021 , TFLOPS: 70.71354371977847, Tokens per sec: 175235.9510775122, Loss: 2.4892733097076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12022 , TFLOPS: 68.92967940823131, Tokens per sec: 170815.33880462218, Loss: 2.5002710819244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12023 , TFLOPS: 69.14921801365476, Tokens per sec: 171359.37965303534, Loss: 2.4796249866485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12024 , TFLOPS: 71.52712972600456, Tokens per sec: 177252.10682483768, Loss: 2.469839572906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12025 , TFLOPS: 70.6185392487454, Tokens per sec: 175000.51953268584, Loss: 2.4907891750335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12026 , TFLOPS: 68.1162477588429, Tokens per sec: 168799.56557054428, Loss: 2.498664379119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12027 , TFLOPS: 70.58542415464399, Tokens per sec: 174918.45668157373, Loss: 2.4952027797698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12028 , TFLOPS: 70.6849420496141, Tokens per sec: 175165.07298811083, Loss: 2.483630657196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12029 , TFLOPS: 67.97084767950693, Tokens per sec: 168439.24815680875, Loss: 2.4930524826049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12030 , TFLOPS: 69.60995295599281, Tokens per sec: 172501.1315942928, Loss: 2.5117363929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12031 , TFLOPS: 71.47837136295082, Tokens per sec: 177131.27823001344, Loss: 2.478982448577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12032 , TFLOPS: 69.52995908687548, Tokens per sec: 172302.8979745679, Loss: 2.499039649963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12033 , TFLOPS: 67.70384981632263, Tokens per sec: 167777.59803959535, Loss: 2.486175298690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12034 , TFLOPS: 71.52005176052778, Tokens per sec: 177234.56684668397, Loss: 2.4846243858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12035 , TFLOPS: 69.8523866694258, Tokens per sec: 173101.90904245793, Loss: 2.4625556468963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12036 , TFLOPS: 68.64144388871885, Tokens per sec: 170101.05943550667, Loss: 2.4936347007751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12037 , TFLOPS: 69.96183396973728, Tokens per sec: 173373.1315092452, Loss: 2.5109190940856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12038 , TFLOPS: 70.54711348264436, Tokens per sec: 174823.51861608977, Loss: 2.5006494522094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12039 , TFLOPS: 69.7257331387398, Tokens per sec: 172788.04764137985, Loss: 2.4901506900787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12040 , TFLOPS: 68.22811317549396, Tokens per sec: 169076.7803959985, Loss: 2.5199880599975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12041 , TFLOPS: 71.53236607782193, Tokens per sec: 177265.08308147293, Loss: 2.486588478088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12042 , TFLOPS: 70.18009110429328, Tokens per sec: 173913.99673168318, Loss: 2.4975428581237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12043 , TFLOPS: 68.2072230357081, Tokens per sec: 169025.01232837027, Loss: 2.5116195678710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12044 , TFLOPS: 70.50145152790907, Tokens per sec: 174710.36326217616, Loss: 2.513411521911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12045 , TFLOPS: 70.568481952533, Tokens per sec: 174876.47203840507, Loss: 2.47538423538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12046 , TFLOPS: 68.08382553737471, Tokens per sec: 168719.21973414355, Loss: 2.5068955421447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12047 , TFLOPS: 70.6849239867984, Tokens per sec: 175165.0282264637, Loss: 2.467012643814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12048 , TFLOPS: 69.72494920658494, Tokens per sec: 172786.10497114283, Loss: 2.4852449893951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12049 , TFLOPS: 68.92516121168752, Tokens per sec: 170804.14221586622, Loss: 2.491896152496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12050 , TFLOPS: 69.99459410517538, Tokens per sec: 173454.31473368764, Loss: 2.4833297729492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12051 , TFLOPS: 71.5055756910615, Tokens per sec: 177198.69355187585, Loss: 2.4909512996673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12052 , TFLOPS: 69.45707636562805, Tokens per sec: 172122.28656262223, Loss: 2.491532325744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12053 , TFLOPS: 69.66221025285125, Tokens per sec: 172630.6309325245, Loss: 2.4722843170166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12054 , TFLOPS: 70.86377087225024, Tokens per sec: 175608.23051022427, Loss: 2.4843804836273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12055 , TFLOPS: 69.71571162170378, Tokens per sec: 172763.21321820334, Loss: 2.481718063354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12056 , TFLOPS: 68.32523313694652, Tokens per sec: 169317.45435912494, Loss: 2.489534854888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12057 , TFLOPS: 70.70017462665832, Tokens per sec: 175202.82099203332, Loss: 2.472982406616211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12058 , TFLOPS: 71.47932879091795, Tokens per sec: 177133.6508419854, Loss: 2.4776456356048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12059 , TFLOPS: 68.85748872585899, Tokens per sec: 170636.4423412434, Loss: 2.4999759197235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12060 , TFLOPS: 68.7944193120032, Tokens per sec: 170480.1493860393, Loss: 2.4991581439971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12061 , TFLOPS: 71.51265659657881, Tokens per sec: 177216.24081577387, Loss: 2.4702024459838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12062 , TFLOPS: 69.84480734208567, Tokens per sec: 173083.12663437726, Loss: 2.5006015300750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12063 , TFLOPS: 67.9083566717854, Tokens per sec: 168284.38855572202, Loss: 2.4875895977020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12064 , TFLOPS: 71.52379767278174, Tokens per sec: 177243.8496298956, Loss: 2.4584293365478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12065 , TFLOPS: 70.58705662865303, Tokens per sec: 174922.50213200002, Loss: 2.4786250591278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12066 , TFLOPS: 67.94660234954844, Tokens per sec: 168379.16555832804, Loss: 2.5216028690338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12067 , TFLOPS: 69.61398419360042, Tokens per sec: 172511.1214451619, Loss: 2.5132951736450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12068 , TFLOPS: 70.69765142988514, Tokens per sec: 175196.56823247627, Loss: 2.469566583633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12069 , TFLOPS: 69.84986962832012, Tokens per sec: 173095.67153732313, Loss: 2.473694324493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12070 , TFLOPS: 68.2179770348335, Tokens per sec: 169051.66192285382, Loss: 2.4762191772460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12071 , TFLOPS: 71.44744991764495, Tokens per sec: 177054.6514822671, Loss: 2.479353427886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12072 , TFLOPS: 70.55330038514626, Tokens per sec: 174838.85044203306, Loss: 2.501893997192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12073 , TFLOPS: 68.70264622308893, Tokens per sec: 170252.72556206942, Loss: 2.503713607788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12074 , TFLOPS: 69.66325187775227, Tokens per sec: 172633.21219377365, Loss: 2.492730140686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12075 , TFLOPS: 70.68995314133663, Tokens per sec: 175177.49102541656, Loss: 2.4648618698120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12076 , TFLOPS: 68.17657384576188, Tokens per sec: 168949.0602593966, Loss: 2.484121561050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12077 , TFLOPS: 69.67691308262343, Tokens per sec: 172667.06616435034, Loss: 2.472322702407837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12078 , TFLOPS: 71.49425224964676, Tokens per sec: 177170.63281667008, Loss: 2.4680581092834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12079 , TFLOPS: 68.81302712129045, Tokens per sec: 170526.2615872725, Loss: 2.4780144691467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12080 , TFLOPS: 68.08375733304892, Tokens per sec: 168719.05071631094, Loss: 2.4909186363220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12081 , TFLOPS: 70.4699551697337, Tokens per sec: 174632.3118170073, Loss: 2.4946303367614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12082 , TFLOPS: 69.85170649701463, Tokens per sec: 173100.2235002959, Loss: 2.4793171882629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12083 , TFLOPS: 68.13232041629152, Tokens per sec: 168839.39538625136, Loss: 2.4992165565490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12084 , TFLOPS: 71.53816473910521, Tokens per sec: 177279.45280290817, Loss: 2.495762825012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12085 , TFLOPS: 69.5733087953383, Tokens per sec: 172410.3233275034, Loss: 2.482503890991211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12086 , TFLOPS: 68.90947763653305, Tokens per sec: 170765.27658894504, Loss: 2.5044047832489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12087 , TFLOPS: 69.93458619214606, Tokens per sec: 173305.60851478367, Loss: 2.5052313804626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12088 , TFLOPS: 71.51071037583024, Tokens per sec: 177211.417866365, Loss: 2.4809489250183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12089 , TFLOPS: 67.61374503038884, Tokens per sec: 167554.3084541899, Loss: 2.4961206912994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12090 , TFLOPS: 68.95021538638268, Tokens per sec: 170866.229220706, Loss: 2.478433609008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12091 , TFLOPS: 71.51523987917498, Tokens per sec: 177222.64247461924, Loss: 2.498997688293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12092 , TFLOPS: 69.57172703412441, Tokens per sec: 172406.40354896995, Loss: 2.506871223449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12093 , TFLOPS: 68.5555481381423, Tokens per sec: 169888.2003033799, Loss: 2.473360300064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12094 , TFLOPS: 70.7231068921951, Tokens per sec: 175259.64967223105, Loss: 2.4914040565490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12095 , TFLOPS: 70.68807227174268, Tokens per sec: 175172.8300233679, Loss: 2.487375020980835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12096 , TFLOPS: 67.80235044609759, Tokens per sec: 168021.69345091784, Loss: 2.5062365531921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12097 , TFLOPS: 70.62825272422732, Tokens per sec: 175024.5905949577, Loss: 2.4934566020965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12098 , TFLOPS: 71.5487986338428, Tokens per sec: 177305.8047654881, Loss: 2.479870557785034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12099 , TFLOPS: 69.0499411995945, Tokens per sec: 171113.36077155013, Loss: 2.5009841918945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12100 , TFLOPS: 67.76780634302776, Tokens per sec: 167936.0893581637, Loss: 2.476201057434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12101 , TFLOPS: 71.5524226670691, Tokens per sec: 177314.7855190431, Loss: 2.4891679286956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12102 , TFLOPS: 69.67366467796123, Tokens per sec: 172659.01625975608, Loss: 2.4901089668273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12103 , TFLOPS: 68.63888157787244, Tokens per sec: 170094.70974696742, Loss: 2.474632978439331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12104 , TFLOPS: 70.4909287063973, Tokens per sec: 174684.2865512855, Loss: 2.5017104148864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12105 , TFLOPS: 70.67166010459461, Tokens per sec: 175132.1588652255, Loss: 2.506990671157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12106 , TFLOPS: 69.82809649604243, Tokens per sec: 173041.71531703047, Loss: 2.521739959716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12107 , TFLOPS: 68.20035453948546, Tokens per sec: 169007.9914381033, Loss: 2.491023302078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12108 , TFLOPS: 71.54290895828248, Tokens per sec: 177291.2095006483, Loss: 2.5148355960845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12109 , TFLOPS: 69.702424910392, Tokens per sec: 172730.28728392068, Loss: 2.4903414249420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12110 , TFLOPS: 68.20752272244088, Tokens per sec: 169025.75498510737, Loss: 2.5023422241210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12111 , TFLOPS: 70.4107388903672, Tokens per sec: 174485.56735352543, Loss: 2.4915430545806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12112 , TFLOPS: 70.70712992466792, Tokens per sec: 175220.0569866904, Loss: 2.495861053466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12113 , TFLOPS: 68.70685243868269, Tokens per sec: 170263.14902766363, Loss: 2.5151760578155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12114 , TFLOPS: 69.73736928986199, Tokens per sec: 172816.8833056885, Loss: 2.4974937438964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12115 , TFLOPS: 70.58322574011103, Tokens per sec: 174913.00876818574, Loss: 2.497500419616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12116 , TFLOPS: 67.91909255309743, Tokens per sec: 168310.99325226818, Loss: 2.505584955215454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12117 , TFLOPS: 70.65280312069194, Tokens per sec: 175085.42918184644, Loss: 2.491121768951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12118 , TFLOPS: 70.40444102218133, Tokens per sec: 174469.9605423934, Loss: 2.464336395263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12119 , TFLOPS: 68.9860788294585, Tokens per sec: 170955.10278333232, Loss: 2.473189115524292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12120 , TFLOPS: 69.64318408692708, Tokens per sec: 172583.4819974592, Loss: 2.506730794906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12121 , TFLOPS: 70.96834473390142, Tokens per sec: 175867.37605915772, Loss: 2.4795680046081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12122 , TFLOPS: 69.61964398283712, Tokens per sec: 172525.14702636772, Loss: 2.481407880783081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12123 , TFLOPS: 68.34461242498591, Tokens per sec: 169365.47836969135, Loss: 2.502981185913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12124 , TFLOPS: 70.62927602612675, Tokens per sec: 175027.12644979046, Loss: 2.49478816986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12125 , TFLOPS: 71.55858119022609, Tokens per sec: 177330.04701784346, Loss: 2.504608392715454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12126 , TFLOPS: 68.21414844382177, Tokens per sec: 169042.17425257267, Loss: 2.482490062713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12127 , TFLOPS: 68.93345172620664, Tokens per sec: 170824.68702412018, Loss: 2.488800048828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12128 , TFLOPS: 71.50708501310336, Tokens per sec: 177202.43381816114, Loss: 2.4798526763916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12129 , TFLOPS: 68.66658680372984, Tokens per sec: 170163.36634862464, Loss: 2.5161561965942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12130 , TFLOPS: 67.92525696368736, Tokens per sec: 168326.26934077594, Loss: 2.5079262256622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12131 , TFLOPS: 71.51450424676848, Tokens per sec: 177220.81949647935, Loss: 2.4800467491149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12132 , TFLOPS: 70.64545193436135, Tokens per sec: 175067.2121323192, Loss: 2.50439715385437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12133 , TFLOPS: 67.87520513965339, Tokens per sec: 168202.2354071561, Loss: 2.4870798587799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12134 , TFLOPS: 70.7482471041902, Tokens per sec: 175321.94988697785, Loss: 2.4826788902282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12135 , TFLOPS: 70.52832541382928, Tokens per sec: 174776.9597118894, Loss: 2.4941914081573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12136 , TFLOPS: 68.9046362429122, Tokens per sec: 170753.27908222907, Loss: 2.509996175765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12137 , TFLOPS: 68.88218483546476, Tokens per sec: 170697.64202134754, Loss: 2.4906060695648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12138 , TFLOPS: 71.5378249298996, Tokens per sec: 177278.61071826358, Loss: 2.514397621154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12139 , TFLOPS: 69.83008940177452, Tokens per sec: 173046.65395697125, Loss: 2.5000836849212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12140 , TFLOPS: 69.16427106993493, Tokens per sec: 171396.68278472862, Loss: 2.4954335689544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12141 , TFLOPS: 70.50749642119563, Tokens per sec: 174725.34317364023, Loss: 2.474499464035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12142 , TFLOPS: 70.60477396906029, Tokens per sec: 174966.4076532554, Loss: 2.492558002471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12143 , TFLOPS: 67.99464052707998, Tokens per sec: 168498.20945409333, Loss: 2.4878740310668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12144 , TFLOPS: 69.53119105278596, Tokens per sec: 172305.95092180633, Loss: 2.4879093170166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12145 , TFLOPS: 71.53718151114128, Tokens per sec: 177277.01625570204, Loss: 2.5021862983703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12146 , TFLOPS: 68.9888327897267, Tokens per sec: 170961.92740025066, Loss: 2.4727487564086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12147 , TFLOPS: 69.01643989351457, Tokens per sec: 171030.34084461056, Loss: 2.508692502975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12148 , TFLOPS: 70.58415423738609, Tokens per sec: 174915.30968671577, Loss: 2.498969316482544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12149 , TFLOPS: 69.60035825606576, Tokens per sec: 172477.35487093014, Loss: 2.5058722496032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12150 , TFLOPS: 68.80770953749891, Tokens per sec: 170513.0840288569, Loss: 2.4983437061309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12151 , TFLOPS: 69.90101666858843, Tokens per sec: 173222.41953741957, Loss: 2.5105741024017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12152 , TFLOPS: 70.68903258994071, Tokens per sec: 175175.2097976504, Loss: 2.4752094745635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12153 , TFLOPS: 69.81665778752205, Tokens per sec: 173013.36893723917, Loss: 2.4873695373535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12154 , TFLOPS: 70.02455271586459, Tokens per sec: 173528.5554711889, Loss: 2.5152764320373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12155 , TFLOPS: 70.49469885834502, Tokens per sec: 174693.62940312846, Loss: 2.4861559867858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12156 , TFLOPS: 69.66929622301188, Tokens per sec: 172648.19074716658, Loss: 2.486537456512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12157 , TFLOPS: 69.04497576130636, Tokens per sec: 171101.05586848393, Loss: 2.484575033187866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12158 , TFLOPS: 71.50590087155345, Tokens per sec: 177199.4993849565, Loss: 2.5106494426727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12159 , TFLOPS: 69.65086750647521, Tokens per sec: 172602.522357498, Loss: 2.5028610229492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12160 , TFLOPS: 68.48417268214449, Tokens per sec: 169711.32406076178, Loss: 2.4903481006622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12161 , TFLOPS: 70.34670374672721, Tokens per sec: 174326.88121353215, Loss: 2.490288019180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12162 , TFLOPS: 70.54623937117918, Tokens per sec: 174821.35247158774, Loss: 2.4864182472229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12163 , TFLOPS: 67.79131534242997, Tokens per sec: 167994.34724841246, Loss: 2.4981958866119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12164 , TFLOPS: 70.66073649829475, Tokens per sec: 175105.0889654799, Loss: 2.4916293621063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12165 , TFLOPS: 71.5010086319455, Tokens per sec: 177187.37587628327, Loss: 2.485743999481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12166 , TFLOPS: 68.76531968188368, Tokens per sec: 170408.0372970142, Loss: 2.4991750717163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12167 , TFLOPS: 68.96877207050758, Tokens per sec: 170912.21472815506, Loss: 2.5029382705688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12168 , TFLOPS: 71.52419975922983, Tokens per sec: 177244.84604440743, Loss: 2.494107961654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12169 , TFLOPS: 70.54792325919115, Tokens per sec: 174825.52533157697, Loss: 2.5119242668151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12170 , TFLOPS: 67.1741684980688, Tokens per sec: 166464.98938374833, Loss: 2.510991334915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12171 , TFLOPS: 70.5675721995899, Tokens per sec: 174874.21757039454, Loss: 2.5093977451324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12172 , TFLOPS: 70.71673363136648, Tokens per sec: 175243.85602982508, Loss: 2.4812512397766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12173 , TFLOPS: 70.59403174366757, Tokens per sec: 174939.7872353784, Loss: 2.472553253173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12174 , TFLOPS: 69.05845662667605, Tokens per sec: 171134.46293790994, Loss: 2.491563320159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12175 , TFLOPS: 71.50815756387185, Tokens per sec: 177205.09171711662, Loss: 2.4970297813415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12176 , TFLOPS: 69.8441142611506, Tokens per sec: 173081.40910347097, Loss: 2.497525930404663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12177 , TFLOPS: 68.48319355030168, Tokens per sec: 169708.897664195, Loss: 2.5014941692352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12178 , TFLOPS: 70.47093211101733, Tokens per sec: 174634.7327851271, Loss: 2.4957821369171143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12179 , TFLOPS: 70.67909687344327, Tokens per sec: 175150.58799766022, Loss: 2.48628830909729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12180 , TFLOPS: 68.04348641902794, Tokens per sec: 168619.25495515956, Loss: 2.485590696334839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12181 , TFLOPS: 69.74750106674341, Tokens per sec: 172841.99096490804, Loss: 2.4827136993408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12182 , TFLOPS: 70.65424101544932, Tokens per sec: 175088.9924434524, Loss: 2.5068395137786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12183 , TFLOPS: 68.114897998304, Tokens per sec: 168796.2207152405, Loss: 2.483459234237671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12184 , TFLOPS: 70.74131595645315, Tokens per sec: 175304.7737393565, Loss: 2.5048294067382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12185 , TFLOPS: 70.44738826330517, Tokens per sec: 174576.3885369275, Loss: 2.4956488609313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12186 , TFLOPS: 68.98920259137941, Tokens per sec: 170962.8438094831, Loss: 2.5125620365142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12187 , TFLOPS: 69.74789590756798, Tokens per sec: 172842.9694239666, Loss: 2.4984488487243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12188 , TFLOPS: 69.89146623647298, Tokens per sec: 173198.75251457051, Loss: 2.4849765300750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12189 , TFLOPS: 69.93336798149288, Tokens per sec: 173302.58965458808, Loss: 2.5006377696990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12190 , TFLOPS: 68.02318501039967, Tokens per sec: 168568.9458281939, Loss: 2.4811277389526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12191 , TFLOPS: 70.6155390550122, Tokens per sec: 174993.08472211636, Loss: 2.476673126220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12192 , TFLOPS: 70.37678121794926, Tokens per sec: 174401.41650052735, Loss: 2.5062708854675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12193 , TFLOPS: 69.94326676318904, Tokens per sec: 173327.11992607205, Loss: 2.484224319458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12194 , TFLOPS: 68.91848067095184, Tokens per sec: 170787.5871000009, Loss: 2.494377613067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12195 , TFLOPS: 71.5540931365083, Tokens per sec: 177318.9251263308, Loss: 2.484283447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12196 , TFLOPS: 68.47899974945985, Tokens per sec: 169698.50496372365, Loss: 2.502589225769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12197 , TFLOPS: 67.97881036808305, Tokens per sec: 168458.9805762629, Loss: 2.4849178791046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12198 , TFLOPS: 71.50291080141488, Tokens per sec: 177192.08966176969, Loss: 2.48752498626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12199 , TFLOPS: 70.6262894761669, Tokens per sec: 175019.7254499943, Loss: 2.491987943649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12200 , TFLOPS: 67.79543582020355, Tokens per sec: 168004.5582462435, Loss: 2.5130937099456787 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/optimizer.pt +[rank0]:[2024-08-27 03:46:55,909] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00642655800038483, 'preprocessing_with_comm': 0.0012374829966574907, 'state_converting': 0.9681332750042202, : 0.9772026290011127}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010200 + eval ppl=10.045722007751465, eval loss=2.3071467876434326 +------------------------------------------------------------------ +iteration: 12201 , TFLOPS: 65.51591427020746, Tokens per sec: 162355.65273532478, Loss: 2.5058465003967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12202 , TFLOPS: 71.44804939476923, Tokens per sec: 177056.13705261893, Loss: 2.5042734146118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12203 , TFLOPS: 68.47478716436932, Tokens per sec: 169688.06571381475, Loss: 2.4811606407165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12204 , TFLOPS: 68.83884545421192, Tokens per sec: 170590.24226037605, Loss: 2.4681849479675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12205 , TFLOPS: 71.5034399991196, Tokens per sec: 177193.40107197795, Loss: 2.4996023178100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12206 , TFLOPS: 68.92780515541224, Tokens per sec: 170810.69419967057, Loss: 2.4862637519836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12207 , TFLOPS: 68.59795057549348, Tokens per sec: 169993.27821415022, Loss: 2.518799066543579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12208 , TFLOPS: 70.87489446808688, Tokens per sec: 175635.79600607266, Loss: 2.497331380844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12209 , TFLOPS: 69.7967777657148, Tokens per sec: 172964.1040532356, Loss: 2.5011844635009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12210 , TFLOPS: 69.85374107260341, Tokens per sec: 173105.26540273186, Loss: 2.493438720703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12211 , TFLOPS: 68.84374898903378, Tokens per sec: 170602.3937598336, Loss: 2.5029077529907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12212 , TFLOPS: 70.53664928499808, Tokens per sec: 174797.58718159574, Loss: 2.49515700340271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12213 , TFLOPS: 70.49211805513508, Tokens per sec: 174687.2338884753, Loss: 2.485905647277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12214 , TFLOPS: 69.12063942942055, Tokens per sec: 171288.558773112, Loss: 2.5034189224243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12215 , TFLOPS: 70.43629579530075, Tokens per sec: 174548.90017927627, Loss: 2.488384485244751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12216 , TFLOPS: 69.87545028285821, Tokens per sec: 173159.0632172102, Loss: 2.49650239944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12217 , TFLOPS: 68.85595089733903, Tokens per sec: 170632.63143276513, Loss: 2.5000534057617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12218 , TFLOPS: 69.53047472381692, Tokens per sec: 172304.1757797109, Loss: 2.470223903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12219 , TFLOPS: 70.45658631488327, Tokens per sec: 174599.182321987, Loss: 2.4947285652160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12220 , TFLOPS: 68.61463831365532, Tokens per sec: 170034.63226762135, Loss: 2.4992523193359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12221 , TFLOPS: 70.64068019946917, Tokens per sec: 175055.3872476064, Loss: 2.5077157020568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12222 , TFLOPS: 70.47219534038899, Tokens per sec: 174637.8632066663, Loss: 2.508085250854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12223 , TFLOPS: 69.35229664418998, Tokens per sec: 171862.63087045954, Loss: 2.5012667179107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12224 , TFLOPS: 68.9364076469317, Tokens per sec: 170832.0121212984, Loss: 2.5038821697235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12225 , TFLOPS: 71.53152833486875, Tokens per sec: 177263.00706214993, Loss: 2.5081048011779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12226 , TFLOPS: 69.43982219312052, Tokens per sec: 172079.52882244458, Loss: 2.4833362102508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12227 , TFLOPS: 68.93713975728929, Tokens per sec: 170833.82637141316, Loss: 2.4956631660461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12228 , TFLOPS: 70.60858569866649, Tokens per sec: 174975.85353911621, Loss: 2.4875590801239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12229 , TFLOPS: 70.73018071749622, Tokens per sec: 175277.17939057338, Loss: 2.4791946411132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12230 , TFLOPS: 69.07624469504964, Tokens per sec: 171178.5437308572, Loss: 2.4795119762420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12231 , TFLOPS: 69.24740968590586, Tokens per sec: 171602.7094335792, Loss: 2.5026845932006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12232 , TFLOPS: 70.6326507319828, Tokens per sec: 175035.48934265634, Loss: 2.495793342590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12233 , TFLOPS: 69.45330631834258, Tokens per sec: 172112.94397014385, Loss: 2.489044427871704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12234 , TFLOPS: 68.91191439753398, Tokens per sec: 170771.31515113672, Loss: 2.494187116622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12235 , TFLOPS: 71.52389554407624, Tokens per sec: 177244.09216574478, Loss: 2.5000617504119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12236 , TFLOPS: 69.8877401634992, Tokens per sec: 173189.5188952807, Loss: 2.504617929458618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12237 , TFLOPS: 67.98781293421797, Tokens per sec: 168481.2899268598, Loss: 2.475050687789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12238 , TFLOPS: 70.71179432485457, Tokens per sec: 175231.615884179, Loss: 2.499349594116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12239 , TFLOPS: 70.60160900667853, Tokens per sec: 174958.564527824, Loss: 2.518455743789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12240 , TFLOPS: 68.51328636912719, Tokens per sec: 169783.47098424227, Loss: 2.4731926918029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12241 , TFLOPS: 70.62520993136471, Tokens per sec: 175017.0502190528, Loss: 2.479375123977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12242 , TFLOPS: 71.56189796169474, Tokens per sec: 177338.26634291527, Loss: 2.480658769607544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12243 , TFLOPS: 69.03750190390228, Tokens per sec: 171082.5348265233, Loss: 2.4877512454986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12244 , TFLOPS: 68.73128715044834, Tokens per sec: 170323.70093512465, Loss: 2.507394790649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12245 , TFLOPS: 70.93210504904867, Tokens per sec: 175777.57012796175, Loss: 2.482383966445923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12246 , TFLOPS: 70.64207672144508, Tokens per sec: 175058.84798290228, Loss: 2.5047318935394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12247 , TFLOPS: 68.16189099268807, Tokens per sec: 168912.67453202987, Loss: 2.48553466796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12248 , TFLOPS: 69.630363679865, Tokens per sec: 172551.71161647493, Loss: 2.486412763595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12249 , TFLOPS: 70.6928886501115, Tokens per sec: 175184.7655395337, Loss: 2.513092041015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12250 , TFLOPS: 69.74398165819834, Tokens per sec: 172833.26948283848, Loss: 2.49122953414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12251 , TFLOPS: 68.89965196750254, Tokens per sec: 170740.92749870062, Loss: 2.4904003143310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12252 , TFLOPS: 70.40919255153, Tokens per sec: 174481.73535554358, Loss: 2.479809045791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12253 , TFLOPS: 69.85793254679649, Tokens per sec: 173115.65233751133, Loss: 2.482079267501831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12254 , TFLOPS: 68.9829858292736, Tokens per sec: 170947.437988152, Loss: 2.5042550563812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12255 , TFLOPS: 70.49777341832619, Tokens per sec: 174701.24850145256, Loss: 2.4971959590911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12256 , TFLOPS: 69.7032430151088, Tokens per sec: 172732.31463753057, Loss: 2.4873344898223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12257 , TFLOPS: 68.76247625131569, Tokens per sec: 170400.990962837, Loss: 2.4652655124664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12258 , TFLOPS: 70.72244038895734, Tokens per sec: 175257.99800378646, Loss: 2.4847447872161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12259 , TFLOPS: 69.53610031963721, Tokens per sec: 172318.11662586412, Loss: 2.4949967861175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12260 , TFLOPS: 69.74520335873474, Tokens per sec: 172836.2969913491, Loss: 2.4929420948028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12261 , TFLOPS: 69.87284708220264, Tokens per sec: 173152.6121992195, Loss: 2.482837677001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12262 , TFLOPS: 71.49237154778709, Tokens per sec: 177165.97223028538, Loss: 2.4774322509765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12263 , TFLOPS: 68.89775988186989, Tokens per sec: 170736.2387020725, Loss: 2.498157024383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12264 , TFLOPS: 69.14503063592322, Tokens per sec: 171349.00286973885, Loss: 2.5120224952697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12265 , TFLOPS: 70.73942845860812, Tokens per sec: 175300.09631177146, Loss: 2.4892075061798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12266 , TFLOPS: 70.65256461665585, Tokens per sec: 175084.8381425716, Loss: 2.4945242404937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12267 , TFLOPS: 66.84208452671288, Tokens per sec: 165642.04872065896, Loss: 2.4875540733337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12268 , TFLOPS: 70.64924237651529, Tokens per sec: 175076.60526552837, Loss: 2.490222692489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12269 , TFLOPS: 70.71919800788126, Tokens per sec: 175249.96302630217, Loss: 2.518644332885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12270 , TFLOPS: 68.86061585476277, Tokens per sec: 170644.19171114868, Loss: 2.484177589416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12271 , TFLOPS: 68.78944421142332, Tokens per sec: 170467.82053874977, Loss: 2.497058391571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12272 , TFLOPS: 71.46162017791588, Tokens per sec: 177089.7669482015, Loss: 2.464430332183838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12273 , TFLOPS: 69.578065584181, Tokens per sec: 172422.11117425916, Loss: 2.502758264541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12274 , TFLOPS: 68.12610600645169, Tokens per sec: 168823.99539409447, Loss: 2.506941556930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12275 , TFLOPS: 71.50879157489011, Tokens per sec: 177206.66286626196, Loss: 2.4964869022369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12276 , TFLOPS: 69.80767409749744, Tokens per sec: 172991.10636372305, Loss: 2.495537281036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12277 , TFLOPS: 69.74519144207392, Tokens per sec: 172836.26746055085, Loss: 2.4933362007141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12278 , TFLOPS: 69.82362663420064, Tokens per sec: 173030.63850699057, Loss: 2.5024545192718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12279 , TFLOPS: 70.70981366181233, Tokens per sec: 175226.7075829723, Loss: 2.4976017475128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12280 , TFLOPS: 70.46255439233418, Tokens per sec: 174613.9718753484, Loss: 2.469344139099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12281 , TFLOPS: 68.33782097021839, Tokens per sec: 169348.64839663694, Loss: 2.512575387954712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12282 , TFLOPS: 71.53570944560794, Tokens per sec: 177273.36831514898, Loss: 2.50291109085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12283 , TFLOPS: 69.68331329759036, Tokens per sec: 172682.92660208026, Loss: 2.494986057281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12284 , TFLOPS: 68.83407461218509, Tokens per sec: 170578.41958828195, Loss: 2.4826290607452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12285 , TFLOPS: 69.57184456680672, Tokens per sec: 172406.69480790404, Loss: 2.5108482837677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12286 , TFLOPS: 70.73174228391865, Tokens per sec: 175281.0491241889, Loss: 2.487597942352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12287 , TFLOPS: 67.7162438001578, Tokens per sec: 167808.31169684895, Loss: 2.473712205886841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12288 , TFLOPS: 70.67041715563525, Tokens per sec: 175129.07870078206, Loss: 2.5004560947418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12289 , TFLOPS: 70.44920117849432, Tokens per sec: 174580.88114047528, Loss: 2.4757206439971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12290 , TFLOPS: 69.13232943968366, Tokens per sec: 171317.52790630495, Loss: 2.489758014678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12291 , TFLOPS: 68.69653043989356, Tokens per sec: 170237.56997760216, Loss: 2.474682092666626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12292 , TFLOPS: 70.52588017481675, Tokens per sec: 174770.900140251, Loss: 2.4894909858703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12293 , TFLOPS: 69.69705619607129, Tokens per sec: 172716.98301842128, Loss: 2.5146641731262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12294 , TFLOPS: 69.71579730891914, Tokens per sec: 172763.42556056226, Loss: 2.518334150314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12295 , TFLOPS: 71.02256028595463, Tokens per sec: 176001.72816948165, Loss: 2.5018768310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12296 , TFLOPS: 69.55674026290897, Tokens per sec: 172369.264678969, Loss: 2.496518135070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12297 , TFLOPS: 70.12267737340188, Tokens per sec: 173771.71918188973, Loss: 2.504091501235962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12298 , TFLOPS: 68.97033844708321, Tokens per sec: 170916.0963818601, Loss: 2.488067626953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12299 , TFLOPS: 70.70112841898131, Tokens per sec: 175205.18459447857, Loss: 2.4910058975219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12300 , TFLOPS: 69.05629407497683, Tokens per sec: 171129.10389657452, Loss: 2.5216944217681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12301 , TFLOPS: 68.90852297007095, Tokens per sec: 170762.91082028675, Loss: 2.5100455284118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12302 , TFLOPS: 71.47463316959775, Tokens per sec: 177122.01457508854, Loss: 2.4776604175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12303 , TFLOPS: 69.72700116226471, Tokens per sec: 172791.1899433589, Loss: 2.4669854640960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12304 , TFLOPS: 67.40549176595754, Tokens per sec: 167038.23392393818, Loss: 2.471705913543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12305 , TFLOPS: 70.55932814394026, Tokens per sec: 174853.78789233704, Loss: 2.4897987842559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12306 , TFLOPS: 70.71792007961668, Tokens per sec: 175246.7961792871, Loss: 2.52221417427063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12307 , TFLOPS: 67.92689553820516, Tokens per sec: 168330.3299089349, Loss: 2.476120948791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12308 , TFLOPS: 69.89291599253899, Tokens per sec: 173202.34516980592, Loss: 2.478792905807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12309 , TFLOPS: 71.50579404869228, Tokens per sec: 177199.23466613897, Loss: 2.5041768550872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12310 , TFLOPS: 68.87218079950232, Tokens per sec: 170672.8509181971, Loss: 2.510213613510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12311 , TFLOPS: 68.46941140346382, Tokens per sec: 169674.74398610202, Loss: 2.493509292602539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12312 , TFLOPS: 70.862138624939, Tokens per sec: 175604.18562158014, Loss: 2.4741642475128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12313 , TFLOPS: 70.71640313007337, Tokens per sec: 175243.03701121302, Loss: 2.495054006576538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12314 , TFLOPS: 68.02034915599813, Tokens per sec: 168561.91826859175, Loss: 2.495840072631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12315 , TFLOPS: 70.68107039020504, Tokens per sec: 175155.4785895967, Loss: 2.496746063232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12316 , TFLOPS: 70.54686897963023, Tokens per sec: 174822.91271068653, Loss: 2.4862852096557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12317 , TFLOPS: 69.69300418530698, Tokens per sec: 172706.94168938143, Loss: 2.5033276081085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12318 , TFLOPS: 67.36802874344995, Tokens per sec: 166945.3964272718, Loss: 2.481881856918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12319 , TFLOPS: 70.75516626787453, Tokens per sec: 175339.0963367964, Loss: 2.4590978622436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12320 , TFLOPS: 70.49630352107053, Tokens per sec: 174697.6059341305, Loss: 2.482722759246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12321 , TFLOPS: 68.82718032115466, Tokens per sec: 170561.3347756397, Loss: 2.496051073074341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12322 , TFLOPS: 70.27219109351695, Tokens per sec: 174142.23065063078, Loss: 2.486456871032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12323 , TFLOPS: 69.69878027754442, Tokens per sec: 172721.2554822349, Loss: 2.5081629753112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12324 , TFLOPS: 68.87321247038275, Tokens per sec: 170675.40751228674, Loss: 2.496584177017212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12325 , TFLOPS: 70.66721404663231, Tokens per sec: 175121.14104382123, Loss: 2.4737422466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12326 , TFLOPS: 69.7828847655404, Tokens per sec: 172929.67566836317, Loss: 2.485827922821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12327 , TFLOPS: 69.76953141500493, Tokens per sec: 172896.58459474327, Loss: 2.4817559719085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12328 , TFLOPS: 69.06185503643975, Tokens per sec: 171142.88457167012, Loss: 2.4910013675689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12329 , TFLOPS: 70.4928077814509, Tokens per sec: 174688.94310626553, Loss: 2.493638038635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12330 , TFLOPS: 69.77041444623458, Tokens per sec: 172898.77284340427, Loss: 2.4913411140441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12331 , TFLOPS: 68.99748471827327, Tokens per sec: 170983.36783227767, Loss: 2.4901318550109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12332 , TFLOPS: 70.5392537726616, Tokens per sec: 174804.04138893395, Loss: 2.5026118755340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12333 , TFLOPS: 69.68551770243522, Tokens per sec: 172688.3893601211, Loss: 2.486862897872925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12334 , TFLOPS: 67.90541074019059, Tokens per sec: 168277.0882127089, Loss: 2.476876735687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12335 , TFLOPS: 70.748191949333, Tokens per sec: 175321.81320716592, Loss: 2.496950387954712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12336 , TFLOPS: 70.59543722823878, Tokens per sec: 174943.27018097276, Loss: 2.48360276222229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12337 , TFLOPS: 68.67002786780147, Tokens per sec: 170171.8936844586, Loss: 2.4925925731658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12338 , TFLOPS: 68.7696513789426, Tokens per sec: 170418.77171950208, Loss: 2.508458137512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12339 , TFLOPS: 71.4516730455618, Tokens per sec: 177065.116858461, Loss: 2.484797954559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12340 , TFLOPS: 70.00670143673148, Tokens per sec: 173484.31803501633, Loss: 2.473311185836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12341 , TFLOPS: 67.87579928628074, Tokens per sec: 168203.70776794938, Loss: 2.484647274017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12342 , TFLOPS: 71.55014273867796, Tokens per sec: 177309.13560533503, Loss: 2.495496988296509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12343 , TFLOPS: 70.5868490973303, Tokens per sec: 174921.98784652085, Loss: 2.482128143310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12344 , TFLOPS: 68.3761611219979, Tokens per sec: 169443.6595162629, Loss: 2.4949021339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12345 , TFLOPS: 69.78851977359999, Tokens per sec: 172943.6398390822, Loss: 2.4939849376678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12346 , TFLOPS: 70.05754237861382, Tokens per sec: 173610.30748959287, Loss: 2.4926395416259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12347 , TFLOPS: 69.69613932114065, Tokens per sec: 172714.71090134102, Loss: 2.487804412841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12348 , TFLOPS: 67.77441264489919, Tokens per sec: 167952.46050194534, Loss: 2.4882400035858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12349 , TFLOPS: 71.51609232048372, Tokens per sec: 177224.7549180865, Loss: 2.499333620071411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12350 , TFLOPS: 69.87622962541809, Tokens per sec: 173160.99451392525, Loss: 2.4959254264831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12351 , TFLOPS: 67.03175420968236, Tokens per sec: 166112.07108889776, Loss: 2.4899938106536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12352 , TFLOPS: 70.65143099897244, Tokens per sec: 175082.02891307382, Loss: 2.503154993057251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12353 , TFLOPS: 70.70425807320932, Tokens per sec: 175212.94022241543, Loss: 2.499629020690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12354 , TFLOPS: 68.79221846278674, Tokens per sec: 170474.69543923732, Loss: 2.4825942516326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12355 , TFLOPS: 69.69335627204262, Tokens per sec: 172707.81419909795, Loss: 2.4864678382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12356 , TFLOPS: 71.53919208346785, Tokens per sec: 177281.9986754101, Loss: 2.5055289268493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12357 , TFLOPS: 69.16014361884167, Tokens per sec: 171386.4545062433, Loss: 2.4960837364196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12358 , TFLOPS: 68.86997293852141, Tokens per sec: 170667.37959547044, Loss: 2.5006816387176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12359 , TFLOPS: 70.52502015510156, Tokens per sec: 174768.76891665778, Loss: 2.496232748031616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12360 , TFLOPS: 69.72744287175355, Tokens per sec: 172792.28454813058, Loss: 2.4944372177124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12361 , TFLOPS: 69.71464501880912, Tokens per sec: 172760.5700587344, Loss: 2.4921655654907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12362 , TFLOPS: 70.96786204898534, Tokens per sec: 175866.17991276353, Loss: 2.501962661743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12363 , TFLOPS: 69.5990500113988, Tokens per sec: 172474.11289653037, Loss: 2.503567695617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12364 , TFLOPS: 69.78144195284347, Tokens per sec: 172926.10021956122, Loss: 2.502293586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12365 , TFLOPS: 69.10457583182072, Tokens per sec: 171248.75140870828, Loss: 2.4698774814605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12366 , TFLOPS: 69.69299892842903, Tokens per sec: 172706.9286622588, Loss: 2.5022997856140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12367 , TFLOPS: 69.81707601444592, Tokens per sec: 173014.405349628, Loss: 2.5076205730438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12368 , TFLOPS: 68.58857610861516, Tokens per sec: 169970.04725254295, Loss: 2.494105100631714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12369 , TFLOPS: 71.49247433377147, Tokens per sec: 177166.22694527748, Loss: 2.486506938934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12370 , TFLOPS: 68.66105944728385, Tokens per sec: 170149.6689504629, Loss: 2.4922473430633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12371 , TFLOPS: 68.82494453067473, Tokens per sec: 170555.79424053777, Loss: 2.4910435676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12372 , TFLOPS: 70.62079351071066, Tokens per sec: 175006.1058421632, Loss: 2.507617473602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12373 , TFLOPS: 70.67391454215023, Tokens per sec: 175137.7456098351, Loss: 2.506394386291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12374 , TFLOPS: 67.91915258595725, Tokens per sec: 168311.14202030798, Loss: 2.489591360092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12375 , TFLOPS: 70.6151140893526, Tokens per sec: 174992.031610398, Loss: 2.4824812412261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12376 , TFLOPS: 71.50220125416064, Tokens per sec: 177190.33132551057, Loss: 2.506412982940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12377 , TFLOPS: 68.88095614375824, Tokens per sec: 170694.59718794728, Loss: 2.4855549335479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12378 , TFLOPS: 68.71286520300275, Tokens per sec: 170278.04932000942, Loss: 2.497809410095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12379 , TFLOPS: 70.93261644206821, Tokens per sec: 175778.83741619874, Loss: 2.515352725982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12380 , TFLOPS: 70.71183658823878, Tokens per sec: 175231.7206175008, Loss: 2.4795479774475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12381 , TFLOPS: 68.75416587132455, Tokens per sec: 170380.39692575502, Loss: 2.485476016998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12382 , TFLOPS: 70.68606524804379, Tokens per sec: 175167.85639755012, Loss: 2.4885849952697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12383 , TFLOPS: 70.63055714106594, Tokens per sec: 175030.30119373626, Loss: 2.4752135276794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12384 , TFLOPS: 69.82753373974516, Tokens per sec: 173040.3207449331, Loss: 2.494777202606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12385 , TFLOPS: 68.04839247381476, Tokens per sec: 168631.41269936782, Loss: 2.5011744499206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12386 , TFLOPS: 70.62169749605803, Tokens per sec: 175008.34601743618, Loss: 2.494281053543091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12387 , TFLOPS: 69.73029846931071, Tokens per sec: 172799.36103344726, Loss: 2.4873528480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12388 , TFLOPS: 68.68201112317341, Tokens per sec: 170201.58951133484, Loss: 2.502150535583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12389 , TFLOPS: 71.50699107040303, Tokens per sec: 177202.20101780107, Loss: 2.4862465858459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12390 , TFLOPS: 69.79072130901599, Tokens per sec: 172949.0954863622, Loss: 2.49053692817688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12391 , TFLOPS: 68.58094817993317, Tokens per sec: 169951.14440498326, Loss: 2.4822871685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12392 , TFLOPS: 69.55321957439456, Tokens per sec: 172360.54002499444, Loss: 2.504523515701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12393 , TFLOPS: 70.67882365763327, Tokens per sec: 175149.91093878582, Loss: 2.4814512729644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12394 , TFLOPS: 69.77358879878255, Tokens per sec: 172906.63923870196, Loss: 2.5076282024383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12395 , TFLOPS: 69.94153795699853, Tokens per sec: 173322.83575388824, Loss: 2.4967477321624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12396 , TFLOPS: 70.47607262272982, Tokens per sec: 174647.4715394248, Loss: 2.518789768218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12397 , TFLOPS: 69.89588050893913, Tokens per sec: 173209.69156801392, Loss: 2.481295585632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12398 , TFLOPS: 68.99255395495543, Tokens per sec: 170971.14885760762, Loss: 2.4877920150756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12399 , TFLOPS: 70.7006446981429, Tokens per sec: 175203.98588095471, Loss: 2.4858579635620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12400 , TFLOPS: 69.56805185811807, Tokens per sec: 172397.29605797218, Loss: 2.5045435428619385 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/model.pt +[rank0]:[2024-08-27 04:12:20,768] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006469492000178434, 'preprocessing_with_comm': 0.0012834110093535855, 'state_converting': 0.977483572991332, : 0.9865967629884835}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010400 + eval ppl=10.599713325500488, eval loss=2.3608269691467285 +------------------------------------------------------------------ +iteration: 12401 , TFLOPS: 64.09325186283472, Tokens per sec: 158830.13857065342, Loss: 2.4958934783935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12402 , TFLOPS: 70.60836254086293, Tokens per sec: 174975.3005294961, Loss: 2.484037399291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12403 , TFLOPS: 69.16430089601003, Tokens per sec: 171396.75669702835, Loss: 2.4779458045959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12404 , TFLOPS: 69.89811429299715, Tokens per sec: 173215.22713098134, Loss: 2.513451337814331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12405 , TFLOPS: 68.79003270760762, Tokens per sec: 170469.27889712268, Loss: 2.4761712551116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12406 , TFLOPS: 71.48354682829539, Tokens per sec: 177144.10360326184, Loss: 2.4870963096618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12407 , TFLOPS: 68.49330795874148, Tokens per sec: 169733.96228250294, Loss: 2.502678155899048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12408 , TFLOPS: 68.01166671090616, Tokens per sec: 168540.40221320355, Loss: 2.5010182857513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12409 , TFLOPS: 71.49791669027697, Tokens per sec: 177179.7137042232, Loss: 2.474074125289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12410 , TFLOPS: 70.71677586271839, Tokens per sec: 175243.96068376728, Loss: 2.4874913692474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12411 , TFLOPS: 67.5718327098784, Tokens per sec: 167450.4451069417, Loss: 2.5013108253479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12412 , TFLOPS: 69.52690654028217, Tokens per sec: 172295.33342784393, Loss: 2.479912281036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12413 , TFLOPS: 70.67587775945817, Tokens per sec: 175142.61067859043, Loss: 2.4901931285858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12414 , TFLOPS: 67.66866434759119, Tokens per sec: 167690.4045720795, Loss: 2.481985092163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12415 , TFLOPS: 69.57278091186359, Tokens per sec: 172409.01517410186, Loss: 2.497741460800171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12416 , TFLOPS: 71.51386105429644, Tokens per sec: 177219.22559468012, Loss: 2.5082626342773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12417 , TFLOPS: 69.81775907578863, Tokens per sec: 173016.0980508807, Loss: 2.5159144401550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12418 , TFLOPS: 67.71825062524593, Tokens per sec: 167813.2848304874, Loss: 2.5095155239105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12419 , TFLOPS: 70.66305716349537, Tokens per sec: 175110.8398294894, Loss: 2.493645191192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12420 , TFLOPS: 70.5470686428615, Tokens per sec: 174823.40749816835, Loss: 2.5064730644226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12421 , TFLOPS: 68.02299044588395, Tokens per sec: 168568.4636758909, Loss: 2.484992504119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12422 , TFLOPS: 69.54541370031146, Tokens per sec: 172341.19620912845, Loss: 2.4987828731536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12423 , TFLOPS: 70.67691035774529, Tokens per sec: 175145.16957089602, Loss: 2.4800405502319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12424 , TFLOPS: 70.70232055642707, Tokens per sec: 175208.13884239394, Loss: 2.4904987812042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12425 , TFLOPS: 68.27662623057466, Tokens per sec: 169197.0010906427, Loss: 2.511343002319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12426 , TFLOPS: 70.41742034703262, Tokens per sec: 174502.12473916556, Loss: 2.4873995780944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12427 , TFLOPS: 70.53231611648157, Tokens per sec: 174786.8491126739, Loss: 2.4974522590637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12428 , TFLOPS: 67.44863904921597, Tokens per sec: 167145.15764491822, Loss: 2.4701833724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12429 , TFLOPS: 70.37998535218026, Tokens per sec: 174409.3566981163, Loss: 2.465259313583374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12430 , TFLOPS: 70.67682675435181, Tokens per sec: 175144.9623924773, Loss: 2.4965360164642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12431 , TFLOPS: 69.03505529036168, Tokens per sec: 171076.47184865293, Loss: 2.4999794960021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12432 , TFLOPS: 69.60723800039806, Tokens per sec: 172494.40363525195, Loss: 2.4891088008880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12433 , TFLOPS: 69.52880843930757, Tokens per sec: 172300.04654314034, Loss: 2.5045485496520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12434 , TFLOPS: 69.77345941103196, Tokens per sec: 172906.31860160275, Loss: 2.4531495571136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12435 , TFLOPS: 69.0004259625674, Tokens per sec: 170990.65655964336, Loss: 2.5103371143341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12436 , TFLOPS: 71.53995162713291, Tokens per sec: 177283.88090828276, Loss: 2.5099172592163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12437 , TFLOPS: 68.86082699120722, Tokens per sec: 170644.7149305166, Loss: 2.4901554584503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12438 , TFLOPS: 68.8436595238015, Tokens per sec: 170602.1720551326, Loss: 2.4890027046203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12439 , TFLOPS: 70.69784596234823, Tokens per sec: 175197.05030534934, Loss: 2.501861095428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12440 , TFLOPS: 70.60403060340191, Tokens per sec: 174964.56551126542, Loss: 2.502246856689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12441 , TFLOPS: 67.95033472055533, Tokens per sec: 168388.41478483772, Loss: 2.479942560195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12442 , TFLOPS: 69.86888247673213, Tokens per sec: 173142.7874701262, Loss: 2.4941656589508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12443 , TFLOPS: 71.53916636052365, Tokens per sec: 177281.93493112075, Loss: 2.495878219604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12444 , TFLOPS: 69.06175858927597, Tokens per sec: 171142.64556497373, Loss: 2.485637903213501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12445 , TFLOPS: 68.04111177051632, Tokens per sec: 168613.37031455862, Loss: 2.4770333766937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12446 , TFLOPS: 71.56756179248033, Tokens per sec: 177352.30193952448, Loss: 2.472533702850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12447 , TFLOPS: 70.60139786863465, Tokens per sec: 174958.04130449248, Loss: 2.5051403045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12448 , TFLOPS: 67.85195009260595, Tokens per sec: 168144.60683881765, Loss: 2.4800643920898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12449 , TFLOPS: 70.81626464777958, Tokens per sec: 175490.50485274332, Loss: 2.497711181640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12450 , TFLOPS: 70.7117124427149, Tokens per sec: 175231.41297121687, Loss: 2.4973390102386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12451 , TFLOPS: 69.70753359475214, Tokens per sec: 172742.94716653312, Loss: 2.4917001724243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12452 , TFLOPS: 69.08837669230184, Tokens per sec: 171208.60815649794, Loss: 2.476914644241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12453 , TFLOPS: 70.73090039678095, Tokens per sec: 175278.96283512015, Loss: 2.5024874210357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12454 , TFLOPS: 69.75486833880262, Tokens per sec: 172860.24787664277, Loss: 2.461003541946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12455 , TFLOPS: 67.9206951658233, Tokens per sec: 168314.96470314526, Loss: 2.5093748569488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12456 , TFLOPS: 71.52255933354625, Tokens per sec: 177240.78088885572, Loss: 2.4860198497772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12457 , TFLOPS: 69.72138888311869, Tokens per sec: 172777.2820973915, Loss: 2.4963464736938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12458 , TFLOPS: 69.76529148502071, Tokens per sec: 172886.07758117552, Loss: 2.491548776626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12459 , TFLOPS: 69.00916403894382, Tokens per sec: 171012.3104754827, Loss: 2.476224899291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12460 , TFLOPS: 70.68176740481985, Tokens per sec: 175157.2058685947, Loss: 2.4941318035125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12461 , TFLOPS: 69.06261918290124, Tokens per sec: 171144.77821077936, Loss: 2.4832708835601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12462 , TFLOPS: 69.50639234578728, Tokens per sec: 172244.4969940605, Loss: 2.4977846145629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12463 , TFLOPS: 70.49461696190323, Tokens per sec: 174693.4264547241, Loss: 2.479666233062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12464 , TFLOPS: 69.7671794285469, Tokens per sec: 172890.75611320758, Loss: 2.4793834686279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12465 , TFLOPS: 68.89775988200037, Tokens per sec: 170736.2387023958, Loss: 2.4850072860717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12466 , TFLOPS: 69.92615768150688, Tokens per sec: 173284.7217369427, Loss: 2.4936270713806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12467 , TFLOPS: 70.67061133041963, Tokens per sec: 175129.5598872879, Loss: 2.496082067489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12468 , TFLOPS: 68.00499478190758, Tokens per sec: 168523.86843817143, Loss: 2.5012104511260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12469 , TFLOPS: 70.6170752561378, Tokens per sec: 174996.89159773206, Loss: 2.4998526573181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12470 , TFLOPS: 69.4764259805184, Tokens per sec: 172170.23703985658, Loss: 2.4754273891448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12471 , TFLOPS: 69.72025391691315, Tokens per sec: 172774.4695261073, Loss: 2.4886538982391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12472 , TFLOPS: 68.81226960603773, Tokens per sec: 170524.38438102906, Loss: 2.490389585494995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12473 , TFLOPS: 70.71783730494332, Tokens per sec: 175246.59105452814, Loss: 2.4910426139831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12474 , TFLOPS: 69.61487424865997, Tokens per sec: 172513.32709964717, Loss: 2.480372667312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12475 , TFLOPS: 68.25306987053388, Tokens per sec: 169138.62583551434, Loss: 2.478508710861206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12476 , TFLOPS: 71.53804002407541, Tokens per sec: 177279.14374532585, Loss: 2.4849424362182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12477 , TFLOPS: 69.5748551352502, Tokens per sec: 172414.15532814848, Loss: 2.4622414112091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12478 , TFLOPS: 69.30244927184205, Tokens per sec: 171739.10359063998, Loss: 2.4977059364318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12479 , TFLOPS: 69.7706410276833, Tokens per sec: 172899.3343371942, Loss: 2.4926161766052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12480 , TFLOPS: 70.74201850775954, Tokens per sec: 175306.51473888615, Loss: 2.466984272003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12481 , TFLOPS: 68.78358520843578, Tokens per sec: 170453.3012839242, Loss: 2.4734904766082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12482 , TFLOPS: 69.10466816833863, Tokens per sec: 171248.9802287714, Loss: 2.4851598739624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12483 , TFLOPS: 71.47387909374305, Tokens per sec: 177120.14589205227, Loss: 2.477060317993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12484 , TFLOPS: 69.53677515418057, Tokens per sec: 172319.78894020175, Loss: 2.501343011856079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12485 , TFLOPS: 67.86481375362797, Tokens per sec: 168176.4844078795, Loss: 2.4867289066314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12486 , TFLOPS: 70.71563560285776, Tokens per sec: 175241.13499422264, Loss: 2.5032927989959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12487 , TFLOPS: 70.72926761261543, Tokens per sec: 175274.91661609197, Loss: 2.474431037902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12488 , TFLOPS: 67.14666215500155, Tokens per sec: 166396.82563555386, Loss: 2.5071988105773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12489 , TFLOPS: 70.59322786428345, Tokens per sec: 174937.7951337099, Loss: 2.50362491607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12490 , TFLOPS: 70.54058408689994, Tokens per sec: 174807.3380541628, Loss: 2.486858367919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12491 , TFLOPS: 70.65794040229287, Tokens per sec: 175098.15993157183, Loss: 2.4836177825927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12492 , TFLOPS: 67.21541342624035, Tokens per sec: 166567.19885926135, Loss: 2.4877731800079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12493 , TFLOPS: 71.50422519619984, Tokens per sec: 177195.34687683935, Loss: 2.49373197555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12494 , TFLOPS: 70.99134328242843, Tokens per sec: 175924.36899590693, Loss: 2.509523868560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12495 , TFLOPS: 67.51238752677733, Tokens per sec: 167303.13339478982, Loss: 2.487145185470581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12496 , TFLOPS: 70.54183928788807, Tokens per sec: 174810.44858048315, Loss: 2.5012621879577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12497 , TFLOPS: 70.54468959405737, Tokens per sec: 174817.51195315784, Loss: 2.5032811164855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12498 , TFLOPS: 68.87940287511584, Tokens per sec: 170690.7480171447, Loss: 2.518601179122925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12499 , TFLOPS: 69.75688853655349, Tokens per sec: 172865.25414921256, Loss: 2.4830422401428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12500 , TFLOPS: 69.75854016399843, Tokens per sec: 172869.3470639629, Loss: 2.4893674850463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12501 , TFLOPS: 69.81719596117665, Tokens per sec: 173014.70259084002, Loss: 2.491722583770752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12502 , TFLOPS: 68.81821006841456, Tokens per sec: 170539.1055011948, Loss: 2.5071628093719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12503 , TFLOPS: 70.40665395888621, Tokens per sec: 174475.4444433232, Loss: 2.4821481704711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12504 , TFLOPS: 69.74019112231693, Tokens per sec: 172823.87611736235, Loss: 2.504988193511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12505 , TFLOPS: 68.56759346719811, Tokens per sec: 169918.0499556827, Loss: 2.48937726020813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12506 , TFLOPS: 70.67623802619622, Tokens per sec: 175143.503459254, Loss: 2.4926133155822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12507 , TFLOPS: 70.5807283759494, Tokens per sec: 174906.82002468585, Loss: 2.4879794120788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12508 , TFLOPS: 69.13317641042384, Tokens per sec: 171319.62679310064, Loss: 2.523810386657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12509 , TFLOPS: 69.93227873720797, Tokens per sec: 173299.89038725995, Loss: 2.5007247924804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12510 , TFLOPS: 71.53113586907256, Tokens per sec: 177262.03448867326, Loss: 2.494309425354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12511 , TFLOPS: 69.5201452258084, Tokens per sec: 172278.57814575767, Loss: 2.482051134109497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12512 , TFLOPS: 67.74455475187926, Tokens per sec: 167878.46935392354, Loss: 2.4993526935577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12513 , TFLOPS: 71.4617833348374, Tokens per sec: 177090.17126902626, Loss: 2.484063148498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12514 , TFLOPS: 70.68882475109342, Tokens per sec: 175174.69475009147, Loss: 2.4724764823913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12515 , TFLOPS: 67.23030355360957, Tokens per sec: 166604.0982351652, Loss: 2.482940912246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12516 , TFLOPS: 71.07030392926225, Tokens per sec: 176120.04217699444, Loss: 2.513566732406616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12517 , TFLOPS: 70.59998463384775, Tokens per sec: 174954.5391530099, Loss: 2.4913384914398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12518 , TFLOPS: 68.66607071093435, Tokens per sec: 170162.08741382507, Loss: 2.4752871990203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12519 , TFLOPS: 69.02612737398412, Tokens per sec: 171054.34748837817, Loss: 2.4990251064300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12520 , TFLOPS: 70.70120162821397, Tokens per sec: 175205.36601502175, Loss: 2.5103909969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12521 , TFLOPS: 69.86940003633298, Tokens per sec: 173144.07003983101, Loss: 2.4836926460266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12522 , TFLOPS: 68.009726046745, Tokens per sec: 168535.5930336304, Loss: 2.490483045578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12523 , TFLOPS: 71.50294597153015, Tokens per sec: 177192.17681718964, Loss: 2.496537923812866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12524 , TFLOPS: 69.71701624130814, Tokens per sec: 172766.44620929868, Loss: 2.4923484325408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12525 , TFLOPS: 68.82703371493554, Tokens per sec: 170560.97146927868, Loss: 2.4781999588012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12526 , TFLOPS: 69.84475835982666, Tokens per sec: 173083.0052509436, Loss: 2.4735913276672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12527 , TFLOPS: 70.71130606658843, Tokens per sec: 175230.40592640944, Loss: 2.471710443496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12528 , TFLOPS: 69.01440367192437, Tokens per sec: 171025.2948632017, Loss: 2.47432279586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12529 , TFLOPS: 68.83126176193433, Tokens per sec: 170571.44903549954, Loss: 2.4715070724487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12530 , TFLOPS: 71.54654764233211, Tokens per sec: 177300.2265605574, Loss: 2.5007851123809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12531 , TFLOPS: 69.64784535371842, Tokens per sec: 172595.03313004907, Loss: 2.495215892791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12532 , TFLOPS: 68.81439505007695, Tokens per sec: 170529.65146549526, Loss: 2.5091652870178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12533 , TFLOPS: 69.97738781854262, Tokens per sec: 173411.67565998275, Loss: 2.4764864444732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12534 , TFLOPS: 70.6469510221783, Tokens per sec: 175070.92703706777, Loss: 2.4915640354156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12535 , TFLOPS: 68.03547443135687, Tokens per sec: 168599.40036711734, Loss: 2.501023769378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12536 , TFLOPS: 70.66847638038374, Tokens per sec: 175124.2692459148, Loss: 2.497645139694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12537 , TFLOPS: 69.59250262774438, Tokens per sec: 172457.88775858085, Loss: 2.4882545471191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12538 , TFLOPS: 69.68752639102739, Tokens per sec: 172693.36711172777, Loss: 2.476977825164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12539 , TFLOPS: 68.89726024827304, Tokens per sec: 170735.00055498033, Loss: 2.514446258544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12540 , TFLOPS: 69.69513264963568, Tokens per sec: 172712.2162584577, Loss: 2.501293897628784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12541 , TFLOPS: 70.6795862061757, Tokens per sec: 175151.80061807562, Loss: 2.485889434814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12542 , TFLOPS: 68.32595951861296, Tokens per sec: 169319.25441291192, Loss: 2.4990787506103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12543 , TFLOPS: 71.53571636621723, Tokens per sec: 177273.3854651812, Loss: 2.480633020401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12544 , TFLOPS: 69.57655963463175, Tokens per sec: 172418.37926538204, Loss: 2.4899864196777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12545 , TFLOPS: 69.27633425727517, Tokens per sec: 171674.38770773693, Loss: 2.4976234436035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12546 , TFLOPS: 69.52339483278577, Tokens per sec: 172286.63102982135, Loss: 2.4945385456085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12547 , TFLOPS: 70.65733562811363, Tokens per sec: 175096.6612345343, Loss: 2.4860188961029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12548 , TFLOPS: 67.77754725493058, Tokens per sec: 167960.22841090895, Loss: 2.4805550575256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12549 , TFLOPS: 69.73188294901527, Tokens per sec: 172803.2875487, Loss: 2.4966416358947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12550 , TFLOPS: 71.49924097257136, Tokens per sec: 177182.9954216304, Loss: 2.4953224658966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12551 , TFLOPS: 68.599781502574, Tokens per sec: 169997.815453149, Loss: 2.483809232711792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12552 , TFLOPS: 68.90745860915065, Tokens per sec: 170760.27321667722, Loss: 2.492353916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12553 , TFLOPS: 70.62129708991564, Tokens per sec: 175007.35376690698, Loss: 2.4877476692199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12554 , TFLOPS: 70.68720090212878, Tokens per sec: 175170.67067347537, Loss: 2.4846413135528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12555 , TFLOPS: 68.72983876974574, Tokens per sec: 170320.1116881913, Loss: 2.493769645690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12556 , TFLOPS: 69.93083034059364, Tokens per sec: 173296.3011008956, Loss: 2.481316566467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12557 , TFLOPS: 70.6019401774555, Tokens per sec: 174959.3852054908, Loss: 2.4906978607177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12558 , TFLOPS: 70.5534838942622, Tokens per sec: 174839.30519783753, Loss: 2.4813473224639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12559 , TFLOPS: 67.3200645561907, Tokens per sec: 166826.53588755196, Loss: 2.492443084716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12560 , TFLOPS: 71.45274561259242, Tokens per sec: 177067.7747977159, Loss: 2.497072696685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12561 , TFLOPS: 70.98007794425632, Tokens per sec: 175896.452246937, Loss: 2.4989876747131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12562 , TFLOPS: 67.32086337163913, Tokens per sec: 166828.51544022997, Loss: 2.481717586517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12563 , TFLOPS: 70.99025667053458, Tokens per sec: 175921.67625193376, Loss: 2.500037908554077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12564 , TFLOPS: 70.63217406327762, Tokens per sec: 175034.30810509546, Loss: 2.4732911586761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12565 , TFLOPS: 69.1059284962189, Tokens per sec: 171252.10346009524, Loss: 2.474539279937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12566 , TFLOPS: 68.58999796139787, Tokens per sec: 169973.57076036852, Loss: 2.4870545864105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12567 , TFLOPS: 70.62113906716169, Tokens per sec: 175006.96216911502, Loss: 2.488125801086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12568 , TFLOPS: 69.76334737544954, Tokens per sec: 172881.25986349682, Loss: 2.4618403911590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12569 , TFLOPS: 69.19496467875116, Tokens per sec: 171472.74492855542, Loss: 2.4821012020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12570 , TFLOPS: 70.50724249312682, Tokens per sec: 174724.71391191325, Loss: 2.5047035217285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12571 , TFLOPS: 69.62674605538541, Tokens per sec: 172542.74674450027, Loss: 2.491872549057007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12572 , TFLOPS: 68.94391875941197, Tokens per sec: 170850.62548544188, Loss: 2.4961512088775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12573 , TFLOPS: 70.62067826634913, Tokens per sec: 175005.82025394012, Loss: 2.5001273155212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12574 , TFLOPS: 69.6447223850238, Tokens per sec: 172587.29406960035, Loss: 2.5059773921966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12575 , TFLOPS: 68.81570734392679, Tokens per sec: 170532.90347421626, Loss: 2.485872745513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12576 , TFLOPS: 69.72017171679717, Tokens per sec: 172774.2658251648, Loss: 2.4842917919158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12577 , TFLOPS: 70.51436631807921, Tokens per sec: 174742.36753489936, Loss: 2.4901058673858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12578 , TFLOPS: 70.11423638361923, Tokens per sec: 173750.8014793565, Loss: 2.4507999420166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12579 , TFLOPS: 68.09514685670345, Tokens per sec: 168747.27521058646, Loss: 2.511331081390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12580 , TFLOPS: 71.49943776864396, Tokens per sec: 177183.48310397717, Loss: 2.4903838634490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12581 , TFLOPS: 69.69425711950795, Tokens per sec: 172710.04659835357, Loss: 2.4840128421783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12582 , TFLOPS: 68.01276051782884, Tokens per sec: 168543.11278725072, Loss: 2.501587390899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12583 , TFLOPS: 70.49884790671761, Tokens per sec: 174703.91120205115, Loss: 2.515200614929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12584 , TFLOPS: 70.72010262466797, Tokens per sec: 175252.2047663514, Loss: 2.4935407638549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12585 , TFLOPS: 68.79846327106344, Tokens per sec: 170490.17076206944, Loss: 2.5034656524658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12586 , TFLOPS: 69.09898290839428, Tokens per sec: 171234.89152834617, Loss: 2.474083662033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12587 , TFLOPS: 70.56356085998853, Tokens per sec: 174864.2770289768, Loss: 2.4927244186401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12588 , TFLOPS: 70.74114230103636, Tokens per sec: 175304.34340210396, Loss: 2.4847936630249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12589 , TFLOPS: 67.93731621559301, Tokens per sec: 168356.15349543438, Loss: 2.4766438007354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12590 , TFLOPS: 71.53884436706056, Tokens per sec: 177281.1369958487, Loss: 2.4649672508239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12591 , TFLOPS: 70.71678711263652, Tokens per sec: 175243.98856230365, Loss: 2.4732630252838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12592 , TFLOPS: 68.17850253336863, Tokens per sec: 168953.83975974857, Loss: 2.483109951019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12593 , TFLOPS: 69.89411481970184, Tokens per sec: 173205.31599557717, Loss: 2.475200653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12594 , TFLOPS: 70.61069334683228, Tokens per sec: 174981.07652344764, Loss: 2.509965419769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12595 , TFLOPS: 68.95738119539037, Tokens per sec: 170883.98688480607, Loss: 2.4859304428100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12596 , TFLOPS: 68.69416707152233, Tokens per sec: 170231.7132904317, Loss: 2.4842700958251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12597 , TFLOPS: 71.48072356919519, Tokens per sec: 177137.10725620386, Loss: 2.5020861625671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12598 , TFLOPS: 69.92288547853653, Tokens per sec: 173276.61285751924, Loss: 2.472052574157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12599 , TFLOPS: 67.88967151834687, Tokens per sec: 168238.08468716178, Loss: 2.4937071800231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12600 , TFLOPS: 70.93497698320185, Tokens per sec: 175784.687097163, Loss: 2.4853079319000244 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/optimizer.pt +[rank0]:[2024-08-27 04:37:48,056] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006388378998963162, 'preprocessing_with_comm': 0.0012240570067660883, 'state_converting': 0.9836629390047165, : 0.9926374950009631}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600, took 4.32s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010600 + eval ppl=10.079267501831055, eval loss=2.3104805946350098 +------------------------------------------------------------------ +iteration: 12601 , TFLOPS: 65.35495820865953, Tokens per sec: 161956.78588403508, Loss: 2.4978039264678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12602 , TFLOPS: 67.9897070417554, Tokens per sec: 168485.9837339907, Loss: 2.4718291759490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12603 , TFLOPS: 69.25035695095335, Tokens per sec: 171610.01308103494, Loss: 2.514763116836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12604 , TFLOPS: 70.51735803363559, Tokens per sec: 174749.78133561273, Loss: 2.4954757690429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12605 , TFLOPS: 70.56346899424793, Tokens per sec: 174864.04937555175, Loss: 2.4886698722839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12606 , TFLOPS: 68.20592102613384, Tokens per sec: 169021.78580521708, Loss: 2.48950457572937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12607 , TFLOPS: 69.56576227428364, Tokens per sec: 172391.62221701202, Loss: 2.4875388145446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12608 , TFLOPS: 71.45889025804534, Tokens per sec: 177083.0019060368, Loss: 2.486135482788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12609 , TFLOPS: 68.25357020045975, Tokens per sec: 169139.86570818655, Loss: 2.500598192214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12610 , TFLOPS: 69.82422531533416, Tokens per sec: 173032.1221047896, Loss: 2.4891421794891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12611 , TFLOPS: 70.68557130507176, Tokens per sec: 175166.63235245313, Loss: 2.505579948425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12612 , TFLOPS: 68.93398300260633, Tokens per sec: 170826.00358556354, Loss: 2.492492914199829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12613 , TFLOPS: 69.87705961771796, Tokens per sec: 173163.0513262755, Loss: 2.5022499561309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12614 , TFLOPS: 69.50548017006811, Tokens per sec: 172242.23652214394, Loss: 2.5076072216033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12615 , TFLOPS: 69.77057844948573, Tokens per sec: 172899.17926152705, Loss: 2.482637405395508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12616 , TFLOPS: 69.17864211206134, Tokens per sec: 171432.29581021884, Loss: 2.49503493309021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12617 , TFLOPS: 71.51908254645713, Tokens per sec: 177232.16502744914, Loss: 2.47921085357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12618 , TFLOPS: 68.5532420735455, Tokens per sec: 169882.48562127506, Loss: 2.4911677837371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12619 , TFLOPS: 68.90453094385462, Tokens per sec: 170753.0181395647, Loss: 2.4855666160583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12620 , TFLOPS: 70.57853323647058, Tokens per sec: 174901.38022724277, Loss: 2.4911012649536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12621 , TFLOPS: 68.86140955325881, Tokens per sec: 170646.15858345496, Loss: 2.4640963077545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12622 , TFLOPS: 67.9527016675994, Tokens per sec: 168394.28034035364, Loss: 2.494744062423706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12623 , TFLOPS: 70.70974068966672, Tokens per sec: 175226.52674995695, Loss: 2.4846158027648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12624 , TFLOPS: 70.6447507623338, Tokens per sec: 175065.47455079446, Loss: 2.4739837646484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12625 , TFLOPS: 70.44127217122205, Tokens per sec: 174561.23218700234, Loss: 2.506985664367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12626 , TFLOPS: 68.19136059661524, Tokens per sec: 168985.70345690628, Loss: 2.5007238388061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12627 , TFLOPS: 70.5826661598956, Tokens per sec: 174911.62206676902, Loss: 2.508404493331909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12628 , TFLOPS: 71.48322993891797, Tokens per sec: 177143.31831647636, Loss: 2.4773831367492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12629 , TFLOPS: 66.9589690359463, Tokens per sec: 165931.701410431, Loss: 2.472700595855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12630 , TFLOPS: 70.61028464649473, Tokens per sec: 174980.06371898917, Loss: 2.4789583683013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12631 , TFLOPS: 70.718352385754, Tokens per sec: 175247.86748151787, Loss: 2.4861927032470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12632 , TFLOPS: 69.7855417932184, Tokens per sec: 172936.260075645, Loss: 2.504584550857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12633 , TFLOPS: 68.60425913098668, Tokens per sec: 170008.9115096071, Loss: 2.485577344894409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12634 , TFLOPS: 70.60321048077387, Tokens per sec: 174962.53315704918, Loss: 2.514049768447876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12635 , TFLOPS: 70.653434953214, Tokens per sec: 175086.9949324382, Loss: 2.500441789627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12636 , TFLOPS: 67.27559665806167, Tokens per sec: 166716.33953744487, Loss: 2.488027572631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12637 , TFLOPS: 71.51922574489652, Tokens per sec: 177232.51988895657, Loss: 2.4822208881378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12638 , TFLOPS: 70.7328202402089, Tokens per sec: 175283.72041862283, Loss: 2.498460531234741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12639 , TFLOPS: 68.35571781889942, Tokens per sec: 169392.99875916916, Loss: 2.491844415664673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12640 , TFLOPS: 70.40764600905919, Tokens per sec: 174477.90285293522, Loss: 2.491154670715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12641 , TFLOPS: 70.58381516874252, Tokens per sec: 174914.46943726548, Loss: 2.47171950340271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12642 , TFLOPS: 67.82786579042676, Tokens per sec: 168084.92328491271, Loss: 2.482846975326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12643 , TFLOPS: 69.57569396596509, Tokens per sec: 172416.23404306482, Loss: 2.5137836933135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12644 , TFLOPS: 70.69731644354584, Tokens per sec: 175195.7380994257, Loss: 2.480356216430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12645 , TFLOPS: 70.63972863112413, Tokens per sec: 175053.02915642346, Loss: 2.5002007484436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12646 , TFLOPS: 68.92558230379973, Tokens per sec: 170805.1857285074, Loss: 2.4836044311523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12647 , TFLOPS: 69.94220142809135, Tokens per sec: 173324.4799083436, Loss: 2.474184036254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12648 , TFLOPS: 70.84974780806441, Tokens per sec: 175573.47981240373, Loss: 2.510716438293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12649 , TFLOPS: 67.91829173666895, Tokens per sec: 168309.0087409412, Loss: 2.4971518516540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12650 , TFLOPS: 70.51712833258945, Tokens per sec: 174749.21211111674, Loss: 2.498016834259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12651 , TFLOPS: 69.52259411779718, Tokens per sec: 172284.6467698736, Loss: 2.4915874004364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12652 , TFLOPS: 69.46745799675762, Tokens per sec: 172148.01338819237, Loss: 2.4831600189208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12653 , TFLOPS: 68.90333316344761, Tokens per sec: 170750.04990776972, Loss: 2.4952428340911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12654 , TFLOPS: 70.60836455604263, Tokens per sec: 174975.30552333343, Loss: 2.500863790512085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12655 , TFLOPS: 69.59200641456243, Tokens per sec: 172456.65808765357, Loss: 2.4995741844177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12656 , TFLOPS: 68.0440364402818, Tokens per sec: 168620.61796841645, Loss: 2.4799563884735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12657 , TFLOPS: 71.48768124965113, Tokens per sec: 177154.34915482547, Loss: 2.4818904399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12658 , TFLOPS: 69.73770360930749, Tokens per sec: 172817.71178610277, Loss: 2.4793701171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12659 , TFLOPS: 67.99369985434288, Tokens per sec: 168495.8783634271, Loss: 2.5024867057800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12660 , TFLOPS: 70.67198428222528, Tokens per sec: 175132.96221310543, Loss: 2.493422269821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12661 , TFLOPS: 70.61341674559746, Tokens per sec: 174987.82540559492, Loss: 2.502443552017212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12662 , TFLOPS: 68.72676963135257, Tokens per sec: 170312.50602515947, Loss: 2.4892499446868896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12663 , TFLOPS: 69.9031472439256, Tokens per sec: 173227.69933780734, Loss: 2.494018793106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12664 , TFLOPS: 71.54732069489248, Tokens per sec: 177302.14226996078, Loss: 2.4911863803863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12665 , TFLOPS: 70.11938596347801, Tokens per sec: 173763.5627055196, Loss: 2.4892706871032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12666 , TFLOPS: 67.91574196445775, Tokens per sec: 168302.69012452124, Loss: 2.488126754760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12667 , TFLOPS: 70.6423146458032, Tokens per sec: 175059.43758567123, Loss: 2.491016387939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12668 , TFLOPS: 70.60559032391524, Tokens per sec: 174968.43067051485, Loss: 2.491774320602417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12669 , TFLOPS: 66.92345966798082, Tokens per sec: 165843.70528493277, Loss: 2.489769697189331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12670 , TFLOPS: 70.55581488662982, Tokens per sec: 174845.08165370452, Loss: 2.4780337810516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12671 , TFLOPS: 70.71390678862154, Tokens per sec: 175236.85080210067, Loss: 2.4844157695770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12672 , TFLOPS: 70.71205168342514, Tokens per sec: 175232.25364706726, Loss: 2.478156089782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12673 , TFLOPS: 67.45592487187159, Tokens per sec: 167163.21271605848, Loss: 2.4863667488098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12674 , TFLOPS: 70.58675465837094, Tokens per sec: 174921.7538163762, Loss: 2.4966254234313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12675 , TFLOPS: 71.52401721427587, Tokens per sec: 177244.39367790252, Loss: 2.490199565887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12676 , TFLOPS: 67.56489253039598, Tokens per sec: 167433.24657766044, Loss: 2.49862003326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12677 , TFLOPS: 70.48096816802538, Tokens per sec: 174659.60323995605, Loss: 2.4935827255249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12678 , TFLOPS: 70.6183136638422, Tokens per sec: 174999.96050844545, Loss: 2.475780963897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12679 , TFLOPS: 69.12679748620934, Tokens per sec: 171303.8191162006, Loss: 2.5086419582366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12680 , TFLOPS: 68.93472218804503, Tokens per sec: 170827.83536850865, Loss: 2.5042247772216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12681 , TFLOPS: 70.59413732810384, Tokens per sec: 174940.04888524258, Loss: 2.4800758361816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12682 , TFLOPS: 69.66041833097897, Tokens per sec: 172626.1903527285, Loss: 2.4766809940338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12683 , TFLOPS: 69.1984519766162, Tokens per sec: 171481.38683682252, Loss: 2.4948089122772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12684 , TFLOPS: 70.49238112861462, Tokens per sec: 174687.88581353755, Loss: 2.4999279975891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12685 , TFLOPS: 69.87157569416968, Tokens per sec: 173149.46155961885, Loss: 2.4745240211486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12686 , TFLOPS: 68.87867312856396, Tokens per sec: 170688.93962480096, Loss: 2.4971303939819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12687 , TFLOPS: 70.73296614751133, Tokens per sec: 175284.0819929913, Loss: 2.511629581451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12688 , TFLOPS: 70.42434140375033, Tokens per sec: 174519.27588012943, Loss: 2.487884759902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12689 , TFLOPS: 67.90563425402262, Tokens per sec: 168277.64210460676, Loss: 2.4940547943115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12690 , TFLOPS: 70.6162245395261, Tokens per sec: 174994.7834282541, Loss: 2.489604949951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12691 , TFLOPS: 70.5475945690645, Tokens per sec: 174824.71080123493, Loss: 2.4728846549987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12692 , TFLOPS: 69.65332976786851, Tokens per sec: 172608.62411247107, Loss: 2.494065046310425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12693 , TFLOPS: 68.04126510375737, Tokens per sec: 168613.750291221, Loss: 2.5036306381225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12694 , TFLOPS: 70.73073311422002, Tokens per sec: 175278.5482905065, Loss: 2.4863502979278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12695 , TFLOPS: 70.48306330638475, Tokens per sec: 174664.79522360905, Loss: 2.4765312671661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12696 , TFLOPS: 68.03195022030604, Tokens per sec: 168590.66698390947, Loss: 2.490165948867798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12697 , TFLOPS: 70.8328570263516, Tokens per sec: 175531.62259464618, Loss: 2.4748294353485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12698 , TFLOPS: 70.5247530801558, Tokens per sec: 174768.10707552143, Loss: 2.479250192642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12699 , TFLOPS: 68.91240745352611, Tokens per sec: 170772.53699819982, Loss: 2.4965994358062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12700 , TFLOPS: 69.06431007288035, Tokens per sec: 171148.96842241453, Loss: 2.475994825363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12701 , TFLOPS: 70.69874508596266, Tokens per sec: 175199.27843271254, Loss: 2.514252185821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12702 , TFLOPS: 69.36430322266429, Tokens per sec: 171892.38449455268, Loss: 2.5071945190429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12703 , TFLOPS: 67.81483847795681, Tokens per sec: 168052.64016953527, Loss: 2.504096031188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12704 , TFLOPS: 71.51836915203421, Tokens per sec: 177230.39715748228, Loss: 2.5068271160125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12705 , TFLOPS: 69.7435225588179, Tokens per sec: 172832.1317839998, Loss: 2.49811053276062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12706 , TFLOPS: 68.7128873372861, Tokens per sec: 170278.10417120188, Loss: 2.4856693744659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12707 , TFLOPS: 69.73364991118879, Tokens per sec: 172807.66627561292, Loss: 2.497032642364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12708 , TFLOPS: 70.67992145868376, Tokens per sec: 175152.63141072172, Loss: 2.4826173782348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12709 , TFLOPS: 69.0971594967098, Tokens per sec: 171230.3729133267, Loss: 2.462524890899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12710 , TFLOPS: 68.88459776204162, Tokens per sec: 170703.62151921084, Loss: 2.4780707359313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12711 , TFLOPS: 70.56278150538753, Tokens per sec: 174862.34570242252, Loss: 2.499976634979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12712 , TFLOPS: 70.66627587377016, Tokens per sec: 175118.81614812053, Loss: 2.488502025604248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12713 , TFLOPS: 68.61766630111106, Tokens per sec: 170042.13595409703, Loss: 2.509929895401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12714 , TFLOPS: 69.9649527590341, Tokens per sec: 173380.86021268574, Loss: 2.5021908283233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12715 , TFLOPS: 71.0018988517528, Tokens per sec: 175950.52685948464, Loss: 2.469212532043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12716 , TFLOPS: 68.145693477725, Tokens per sec: 168872.53530564785, Loss: 2.496727705001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12717 , TFLOPS: 69.63453355479825, Tokens per sec: 172562.04502590914, Loss: 2.490154981613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12718 , TFLOPS: 70.62793458195007, Tokens per sec: 175023.80220334837, Loss: 2.464265823364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12719 , TFLOPS: 69.83093530084956, Tokens per sec: 173048.7501880627, Loss: 2.4835000038146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12720 , TFLOPS: 68.69219252151669, Tokens per sec: 170226.8201380031, Loss: 2.5093517303466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12721 , TFLOPS: 70.43957428038854, Tokens per sec: 174557.02462647876, Loss: 2.494819164276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12722 , TFLOPS: 70.57304569789711, Tokens per sec: 174887.7815021554, Loss: 2.4950685501098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12723 , TFLOPS: 68.33024252003953, Tokens per sec: 169329.8681622585, Loss: 2.4889330863952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12724 , TFLOPS: 71.53353621182038, Tokens per sec: 177267.98280242196, Loss: 2.4854061603546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12725 , TFLOPS: 69.33386308267258, Tokens per sec: 171816.95047439498, Loss: 2.5038440227508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12726 , TFLOPS: 67.65602078421516, Tokens per sec: 167659.0724292302, Loss: 2.5032546520233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12727 , TFLOPS: 70.53895074501614, Tokens per sec: 174803.29045304726, Loss: 2.4995975494384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12728 , TFLOPS: 70.73772510501799, Tokens per sec: 175295.87521393521, Loss: 2.478952646255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12729 , TFLOPS: 68.77516402637195, Tokens per sec: 170432.43266710025, Loss: 2.465437412261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12730 , TFLOPS: 69.16631843976812, Tokens per sec: 171401.75639271166, Loss: 2.4831199645996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12731 , TFLOPS: 71.54821854322985, Tokens per sec: 177304.36723704697, Loss: 2.484156608581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12732 , TFLOPS: 68.99743733977579, Tokens per sec: 170983.25042314173, Loss: 2.502009391784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12733 , TFLOPS: 68.08582717098167, Tokens per sec: 168724.18000271978, Loss: 2.4697680473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12734 , TFLOPS: 70.59815180601245, Tokens per sec: 174949.9972037314, Loss: 2.4946095943450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12735 , TFLOPS: 70.66450335114936, Tokens per sec: 175114.4236418067, Loss: 2.4735403060913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12736 , TFLOPS: 67.73448323111593, Tokens per sec: 167853.51101600242, Loss: 2.5065019130706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12737 , TFLOPS: 70.70886392241563, Tokens per sec: 175224.35402412486, Loss: 2.5002710819244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12738 , TFLOPS: 70.56461133093741, Tokens per sec: 174866.88021170706, Loss: 2.5077240467071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12739 , TFLOPS: 70.60265725898955, Tokens per sec: 174961.16221252523, Loss: 2.488518714904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12740 , TFLOPS: 67.73961851367082, Tokens per sec: 167866.2368118715, Loss: 2.4781103134155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12741 , TFLOPS: 70.71276874619026, Tokens per sec: 175234.03060759025, Loss: 2.4907984733581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12742 , TFLOPS: 71.49222369743204, Tokens per sec: 177165.60584081852, Loss: 2.492027521133423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12743 , TFLOPS: 67.35320344132533, Tokens per sec: 166908.65769546514, Loss: 2.496386766433716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12744 , TFLOPS: 71.51661353091326, Tokens per sec: 177226.04653494692, Loss: 2.498528003692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12745 , TFLOPS: 70.5910309587836, Tokens per sec: 174932.35095987222, Loss: 2.4785516262054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12746 , TFLOPS: 68.9774207022713, Tokens per sec: 170933.64699039096, Loss: 2.4954962730407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12747 , TFLOPS: 68.92375075953218, Tokens per sec: 170800.64696005115, Loss: 2.48813796043396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12748 , TFLOPS: 70.77103389290953, Tokens per sec: 175378.41805959636, Loss: 2.4896152019500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12749 , TFLOPS: 69.79952712793857, Tokens per sec: 172970.9172756934, Loss: 2.5132830142974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12750 , TFLOPS: 68.95605531078418, Tokens per sec: 170880.70119669373, Loss: 2.4811055660247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12751 , TFLOPS: 70.45052304539855, Tokens per sec: 174584.15687227962, Loss: 2.478841543197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12752 , TFLOPS: 69.75281190301223, Tokens per sec: 172855.15180221907, Loss: 2.480497360229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12753 , TFLOPS: 68.8680628105395, Tokens per sec: 170662.6460879133, Loss: 2.4910640716552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12754 , TFLOPS: 69.57129587060992, Tokens per sec: 172405.33507828446, Loss: 2.493469715118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12755 , TFLOPS: 70.73803632643168, Tokens per sec: 175296.64645488223, Loss: 2.483144998550415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12756 , TFLOPS: 68.04824265940164, Tokens per sec: 168631.04144274868, Loss: 2.4803354740142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12757 , TFLOPS: 70.7641790727222, Tokens per sec: 175361.43106005137, Loss: 2.494413375854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12758 , TFLOPS: 69.61992418589834, Tokens per sec: 172525.84140042032, Loss: 2.5003232955932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12759 , TFLOPS: 70.5848488336594, Tokens per sec: 174917.03097279707, Loss: 2.500056743621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12760 , TFLOPS: 68.06277315216225, Tokens per sec: 168667.0496044754, Loss: 2.4927945137023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12761 , TFLOPS: 71.51892311782069, Tokens per sec: 177231.76994572554, Loss: 2.4751672744750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12762 , TFLOPS: 69.97787590007471, Tokens per sec: 173412.88517978587, Loss: 2.480642318725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12763 , TFLOPS: 68.05197535019117, Tokens per sec: 168640.29146171498, Loss: 2.4912118911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12764 , TFLOPS: 71.01085222032124, Tokens per sec: 175972.71429309223, Loss: 2.5054409503936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12765 , TFLOPS: 70.60774385913348, Tokens per sec: 174973.76736801714, Loss: 2.4736576080322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12766 , TFLOPS: 69.5026673081448, Tokens per sec: 172235.26594043564, Loss: 2.4858920574188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12767 , TFLOPS: 69.11291321094578, Tokens per sec: 171269.4123526183, Loss: 2.49168062210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12768 , TFLOPS: 70.76167984874131, Tokens per sec: 175355.2377077145, Loss: 2.5013699531555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12769 , TFLOPS: 68.84159242468603, Tokens per sec: 170597.0495558147, Loss: 2.491957187652588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12770 , TFLOPS: 68.71448686438909, Tokens per sec: 170282.0679755655, Loss: 2.4892797470092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12771 , TFLOPS: 71.50292403130771, Tokens per sec: 177192.1224469015, Loss: 2.4883246421813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12772 , TFLOPS: 69.6975205744776, Tokens per sec: 172718.1337992676, Loss: 2.4805567264556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12773 , TFLOPS: 68.78358267057766, Tokens per sec: 170453.2949948322, Loss: 2.495713472366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12774 , TFLOPS: 70.59280266827153, Tokens per sec: 174936.74145115318, Loss: 2.501889944076538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12775 , TFLOPS: 70.72571554947802, Tokens per sec: 175266.11421234545, Loss: 2.4947543144226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12776 , TFLOPS: 67.96353328042953, Tokens per sec: 168421.12227014737, Loss: 2.4988510608673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12777 , TFLOPS: 69.64075323907858, Tokens per sec: 172577.45808871058, Loss: 2.484903573989868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12778 , TFLOPS: 70.59381085218511, Tokens per sec: 174939.23984195167, Loss: 2.491412878036499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12779 , TFLOPS: 70.55422237861023, Tokens per sec: 174841.13524340276, Loss: 2.5072953701019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12780 , TFLOPS: 68.88968435125473, Tokens per sec: 170716.226647615, Loss: 2.492053270339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12781 , TFLOPS: 70.50913472964596, Tokens per sec: 174729.40308245472, Loss: 2.5197386741638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12782 , TFLOPS: 70.64564761232948, Tokens per sec: 175067.69704387995, Loss: 2.4792890548706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12783 , TFLOPS: 68.07032053170737, Tokens per sec: 168685.75284240072, Loss: 2.472733497619629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12784 , TFLOPS: 69.6580264955803, Tokens per sec: 172620.26312112817, Loss: 2.482527017593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12785 , TFLOPS: 70.56466794929382, Tokens per sec: 174867.02051823126, Loss: 2.501885414123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12786 , TFLOPS: 69.61638787135352, Tokens per sec: 172517.07802342108, Loss: 2.48568058013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12787 , TFLOPS: 68.97274920624935, Tokens per sec: 170922.07050864078, Loss: 2.4851160049438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12788 , TFLOPS: 69.73895478003331, Tokens per sec: 172820.8123249892, Loss: 2.4862594604492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12789 , TFLOPS: 70.66589839131582, Tokens per sec: 175117.88070501556, Loss: 2.470945119857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12790 , TFLOPS: 68.37954131856176, Tokens per sec: 169452.0360157076, Loss: 2.473745346069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12791 , TFLOPS: 70.40759790995585, Tokens per sec: 174477.7836580586, Loss: 2.496382713317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12792 , TFLOPS: 70.57046854284873, Tokens per sec: 174881.39502804846, Loss: 2.504521131515503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12793 , TFLOPS: 68.25782807065328, Tokens per sec: 169150.41717956925, Loss: 2.4874424934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12794 , TFLOPS: 70.61112565200644, Tokens per sec: 174982.14782329154, Loss: 2.4795031547546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12795 , TFLOPS: 69.57534277435713, Tokens per sec: 172415.3637515733, Loss: 2.4707465171813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12796 , TFLOPS: 69.10327137545227, Tokens per sec: 171245.51882212952, Loss: 2.481621265411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12797 , TFLOPS: 69.99548009399969, Tokens per sec: 173456.51031159403, Loss: 2.489382266998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12798 , TFLOPS: 71.49012253199749, Tokens per sec: 177160.3989214092, Loss: 2.488672971725464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12799 , TFLOPS: 68.96875689579129, Tokens per sec: 170912.1771235364, Loss: 2.470991373062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12800 , TFLOPS: 68.40951804188728, Tokens per sec: 169526.32163831688, Loss: 2.4932451248168945 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/model.pt +[rank0]:[2024-08-27 05:03:13,986] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063953479984775186, 'preprocessing_with_comm': 0.0012555160064948723, 'state_converting': 0.9853367609903216, : 0.9943686949991388}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0010800 + eval ppl=10.504597663879395, eval loss=2.3518130779266357 +------------------------------------------------------------------ +iteration: 12801 , TFLOPS: 65.04373240985839, Tokens per sec: 161185.5340702539, Loss: 2.481438398361206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12802 , TFLOPS: 69.24810847385, Tokens per sec: 171604.44110708262, Loss: 2.4911773204803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12803 , TFLOPS: 69.55205147251715, Tokens per sec: 172357.64533986518, Loss: 2.503967523574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12804 , TFLOPS: 70.66161767446621, Tokens per sec: 175107.2726171023, Loss: 2.4874727725982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12805 , TFLOPS: 70.57323723074971, Tokens per sec: 174888.25614166324, Loss: 2.498579502105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12806 , TFLOPS: 68.64402407505538, Tokens per sec: 170107.4534214796, Loss: 2.4938409328460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12807 , TFLOPS: 68.9940222408155, Tokens per sec: 170974.78743171395, Loss: 2.5125064849853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12808 , TFLOPS: 71.51250744531639, Tokens per sec: 177215.8712025152, Loss: 2.4941587448120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12809 , TFLOPS: 70.07012488669135, Tokens per sec: 173641.4883306867, Loss: 2.5100202560424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12810 , TFLOPS: 68.63681925637877, Tokens per sec: 170089.59908712262, Loss: 2.4810664653778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12811 , TFLOPS: 70.70571884654764, Tokens per sec: 175216.56017966528, Loss: 2.479661464691162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12812 , TFLOPS: 70.67215590973949, Tokens per sec: 175133.38752499197, Loss: 2.485527992248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12813 , TFLOPS: 67.75103620744135, Tokens per sec: 167894.53111479763, Loss: 2.485409736633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12814 , TFLOPS: 70.2117144234035, Tokens per sec: 173992.36280003446, Loss: 2.4631524085998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12815 , TFLOPS: 70.53022998195826, Tokens per sec: 174781.67944151847, Loss: 2.4899325370788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12816 , TFLOPS: 68.12931607815112, Tokens per sec: 168831.95030538426, Loss: 2.488853693008423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12817 , TFLOPS: 69.61806834378105, Tokens per sec: 172521.2424192158, Loss: 2.486546039581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12818 , TFLOPS: 71.53721844523835, Tokens per sec: 177277.1077824632, Loss: 2.5243887901306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12819 , TFLOPS: 69.78605243403082, Tokens per sec: 172937.5254998299, Loss: 2.4968574047088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12820 , TFLOPS: 68.77108330403006, Tokens per sec: 170422.32018760833, Loss: 2.470393657684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12821 , TFLOPS: 69.92536098896454, Tokens per sec: 173282.74744505988, Loss: 2.509587526321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12822 , TFLOPS: 70.58842562983544, Tokens per sec: 174925.89466773797, Loss: 2.499481201171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12823 , TFLOPS: 68.13687005956838, Tokens per sec: 168850.669903476, Loss: 2.505512237548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12824 , TFLOPS: 70.65424751203362, Tokens per sec: 175089.00854270396, Loss: 2.4782681465148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12825 , TFLOPS: 69.04778006360023, Tokens per sec: 171108.00523848753, Loss: 2.4937210083007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12826 , TFLOPS: 70.60245792121883, Tokens per sec: 174960.6682315705, Loss: 2.492213487625122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12827 , TFLOPS: 68.24020063883434, Tokens per sec: 169106.73446170002, Loss: 2.499861240386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12828 , TFLOPS: 71.49297441463946, Tokens per sec: 177167.46620075684, Loss: 2.4850914478302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12829 , TFLOPS: 70.39310438105646, Tokens per sec: 174441.86709685187, Loss: 2.48992657661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12830 , TFLOPS: 69.07274361491501, Tokens per sec: 171169.8676686078, Loss: 2.476144790649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12831 , TFLOPS: 70.62983659065986, Tokens per sec: 175028.5155904548, Loss: 2.4979615211486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12832 , TFLOPS: 69.41134373633945, Tokens per sec: 172008.95606938007, Loss: 2.501751661300659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12833 , TFLOPS: 67.71019319316747, Tokens per sec: 167793.31762619753, Loss: 2.4856231212615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12834 , TFLOPS: 70.66062662154002, Tokens per sec: 175104.81667877774, Loss: 2.4809839725494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12835 , TFLOPS: 70.6269027450406, Tokens per sec: 175021.24519782048, Loss: 2.494358539581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12836 , TFLOPS: 67.68056646655214, Tokens per sec: 167719.89933399137, Loss: 2.503009080886841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12837 , TFLOPS: 70.86428585212082, Tokens per sec: 175609.50668707324, Loss: 2.496373176574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12838 , TFLOPS: 70.89659340917663, Tokens per sec: 175689.56837242874, Loss: 2.5017471313476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12839 , TFLOPS: 68.3825508699608, Tokens per sec: 169459.49401560385, Loss: 2.4858672618865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12840 , TFLOPS: 68.88835010624996, Tokens per sec: 170712.92024151396, Loss: 2.5117108821868896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12841 , TFLOPS: 70.66268501585219, Tokens per sec: 175109.91760663415, Loss: 2.4992055892944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12842 , TFLOPS: 70.62779271614879, Tokens per sec: 175023.450644265, Loss: 2.487424373626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12843 , TFLOPS: 66.89615467796055, Tokens per sec: 165776.0405117696, Loss: 2.4971084594726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12844 , TFLOPS: 70.58835705866716, Tokens per sec: 174925.72474082923, Loss: 2.509974479675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12845 , TFLOPS: 71.515161496762, Tokens per sec: 177222.44823436535, Loss: 2.512590169906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12846 , TFLOPS: 70.05719422281821, Tokens per sec: 173609.44472117882, Loss: 2.4696362018585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12847 , TFLOPS: 68.00485196308253, Tokens per sec: 168523.51451739014, Loss: 2.4721059799194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12848 , TFLOPS: 71.51784856326326, Tokens per sec: 177229.10708116027, Loss: 2.4850993156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12849 , TFLOPS: 70.58062413006492, Tokens per sec: 174906.56169190045, Loss: 2.5077342987060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12850 , TFLOPS: 68.80092016257021, Tokens per sec: 170496.25920987214, Loss: 2.4714818000793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12851 , TFLOPS: 69.3804102643566, Tokens per sec: 171932.29951820333, Loss: 2.507807731628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12852 , TFLOPS: 70.69909718349155, Tokens per sec: 175200.15096917588, Loss: 2.4818520545959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12853 , TFLOPS: 68.61933787444639, Tokens per sec: 170046.27829696098, Loss: 2.4976460933685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12854 , TFLOPS: 69.18354336937422, Tokens per sec: 171444.44166575823, Loss: 2.4937264919281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12855 , TFLOPS: 70.59512067118858, Tokens per sec: 174942.48571773074, Loss: 2.4834537506103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12856 , TFLOPS: 69.75647895302552, Tokens per sec: 172864.2391561109, Loss: 2.508741855621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12857 , TFLOPS: 69.9397610999903, Tokens per sec: 173318.43250648488, Loss: 2.4882965087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12858 , TFLOPS: 69.16015735102859, Tokens per sec: 171386.48853611518, Loss: 2.4783596992492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12859 , TFLOPS: 69.87392915490275, Tokens per sec: 173155.29369457052, Loss: 2.49061918258667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12860 , TFLOPS: 68.80201136592169, Tokens per sec: 170498.9633319831, Loss: 2.477069139480591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12861 , TFLOPS: 70.74364140726223, Tokens per sec: 175310.53646263096, Loss: 2.4870829582214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12862 , TFLOPS: 69.57873051667657, Tokens per sec: 172423.75895023154, Loss: 2.5034048557281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12863 , TFLOPS: 67.98196069310532, Tokens per sec: 168466.78742870508, Loss: 2.504611015319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12864 , TFLOPS: 70.61335779806812, Tokens per sec: 174987.6793271236, Loss: 2.497103214263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12865 , TFLOPS: 71.4588493152042, Tokens per sec: 177082.90044516625, Loss: 2.477482557296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12866 , TFLOPS: 68.8192190472047, Tokens per sec: 170541.60586178495, Loss: 2.491436719894409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12867 , TFLOPS: 68.8900969872293, Tokens per sec: 170717.24920501557, Loss: 2.4954378604888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12868 , TFLOPS: 70.95407852780436, Tokens per sec: 175832.02282889685, Loss: 2.479703664779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12869 , TFLOPS: 69.68358721708186, Tokens per sec: 172683.60540475496, Loss: 2.504517078399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12870 , TFLOPS: 68.71920264638544, Tokens per sec: 170293.75420283878, Loss: 2.4882683753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12871 , TFLOPS: 70.5954080659641, Tokens per sec: 174943.19791364358, Loss: 2.4749999046325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12872 , TFLOPS: 70.63474550253429, Tokens per sec: 175040.68041484093, Loss: 2.4861366748809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12873 , TFLOPS: 68.8435087385027, Tokens per sec: 170601.7983925518, Loss: 2.497684955596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12874 , TFLOPS: 69.28083944847248, Tokens per sec: 171685.55206781163, Loss: 2.4987316131591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12875 , TFLOPS: 70.7666909022994, Tokens per sec: 175367.65565044453, Loss: 2.486121654510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12876 , TFLOPS: 70.27370859573796, Tokens per sec: 174145.9911883011, Loss: 2.502579689025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12877 , TFLOPS: 69.13146040989085, Tokens per sec: 171315.3743547468, Loss: 2.493165969848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12878 , TFLOPS: 70.58389293456499, Tokens per sec: 174914.66214954018, Loss: 2.4759292602539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12879 , TFLOPS: 70.56598348714134, Tokens per sec: 174870.28056593586, Loss: 2.5027658939361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12880 , TFLOPS: 68.19431540489914, Tokens per sec: 168993.0257973328, Loss: 2.4836225509643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12881 , TFLOPS: 70.62036113430369, Tokens per sec: 175005.0343657965, Loss: 2.4801299571990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12882 , TFLOPS: 70.69601726332439, Tokens per sec: 175192.51858772043, Loss: 2.4940357208251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12883 , TFLOPS: 68.74794945534362, Tokens per sec: 170364.99196215955, Loss: 2.4855732917785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12884 , TFLOPS: 68.93874552303502, Tokens per sec: 170837.80563582075, Loss: 2.486948013305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12885 , TFLOPS: 71.54288651971, Tokens per sec: 177291.1538953939, Loss: 2.481574535369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12886 , TFLOPS: 69.77235808546094, Tokens per sec: 172903.58939551673, Loss: 2.4817874431610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12887 , TFLOPS: 68.96490363492654, Tokens per sec: 170902.62831864186, Loss: 2.477989912033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12888 , TFLOPS: 69.58321694400152, Tokens per sec: 172434.8768113635, Loss: 2.5176517963409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12889 , TFLOPS: 70.65951776403033, Tokens per sec: 175102.06880771797, Loss: 2.5049476623535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12890 , TFLOPS: 67.95652880887022, Tokens per sec: 168403.76441801715, Loss: 2.4938642978668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12891 , TFLOPS: 69.51758634941245, Tokens per sec: 172272.236968165, Loss: 2.511521339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12892 , TFLOPS: 70.58340427660639, Tokens per sec: 174913.45120128864, Loss: 2.4922449588775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12893 , TFLOPS: 70.70691155420627, Tokens per sec: 175219.51584063095, Loss: 2.4923999309539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12894 , TFLOPS: 68.4405847804127, Tokens per sec: 169603.30843866666, Loss: 2.4885149002075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12895 , TFLOPS: 70.50887847992513, Tokens per sec: 174728.76806741825, Loss: 2.4715609550476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12896 , TFLOPS: 70.95617253201999, Tokens per sec: 175837.2120020168, Loss: 2.4821720123291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12897 , TFLOPS: 68.13548838796018, Tokens per sec: 168847.24596902716, Loss: 2.492540121078491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12898 , TFLOPS: 70.66694137549855, Tokens per sec: 175120.46533471456, Loss: 2.5031685829162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12899 , TFLOPS: 69.56521245829073, Tokens per sec: 172390.25971241435, Loss: 2.5106191635131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12900 , TFLOPS: 68.14135386768635, Tokens per sec: 168861.78127392454, Loss: 2.483450412750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12901 , TFLOPS: 70.7225885018408, Tokens per sec: 175258.36504382768, Loss: 2.49594783782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12902 , TFLOPS: 70.66774747238331, Tokens per sec: 175122.46293159388, Loss: 2.5186855792999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12903 , TFLOPS: 67.82666934025885, Tokens per sec: 168081.95834959677, Loss: 2.467388391494751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12904 , TFLOPS: 70.52454609356323, Tokens per sec: 174767.59413994357, Loss: 2.474050998687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12905 , TFLOPS: 70.97401602058544, Tokens per sec: 175881.4301323045, Loss: 2.5155184268951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12906 , TFLOPS: 68.77301741159822, Tokens per sec: 170427.11311922182, Loss: 2.4841771125793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12907 , TFLOPS: 68.89954222717809, Tokens per sec: 170740.65555008774, Loss: 2.490658760070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12908 , TFLOPS: 70.71710187139091, Tokens per sec: 175244.76856917058, Loss: 2.488715648651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12909 , TFLOPS: 70.74262505405682, Tokens per sec: 175308.01782742736, Loss: 2.5065643787384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12910 , TFLOPS: 68.4588802715687, Tokens per sec: 169648.64668116593, Loss: 2.4895894527435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12911 , TFLOPS: 70.64868318212379, Tokens per sec: 175075.21952022586, Loss: 2.50260853767395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12912 , TFLOPS: 71.54574372575232, Tokens per sec: 177298.23436671396, Loss: 2.4764671325683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12913 , TFLOPS: 68.94559355285803, Tokens per sec: 170854.7758080949, Loss: 2.4798221588134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12914 , TFLOPS: 68.18378453607426, Tokens per sec: 168966.92914431373, Loss: 2.4784669876098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12915 , TFLOPS: 71.5270473264453, Tokens per sec: 177251.90262965276, Loss: 2.4874160289764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12916 , TFLOPS: 70.69727541535896, Tokens per sec: 175195.63642705898, Loss: 2.4941935539245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12917 , TFLOPS: 66.89887161523855, Tokens per sec: 165782.7733816398, Loss: 2.5112721920013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12918 , TFLOPS: 70.63591749651034, Tokens per sec: 175043.58474501938, Loss: 2.497534990310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12919 , TFLOPS: 70.5305444539812, Tokens per sec: 174782.45873783325, Loss: 2.4983434677124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12920 , TFLOPS: 68.88945818481, Tokens per sec: 170715.66618225066, Loss: 2.497086524963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12921 , TFLOPS: 68.7210638858838, Tokens per sec: 170298.36655934813, Loss: 2.509791612625122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12922 , TFLOPS: 70.3488914528853, Tokens per sec: 174332.3025903876, Loss: 2.4722254276275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12923 , TFLOPS: 69.71502287194099, Tokens per sec: 172761.50642041911, Loss: 2.502530097961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12924 , TFLOPS: 70.02907838000618, Tokens per sec: 173539.77056547487, Loss: 2.488081455230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12925 , TFLOPS: 68.95603485174324, Tokens per sec: 170880.65049693652, Loss: 2.4870214462280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12926 , TFLOPS: 69.66550340562037, Tokens per sec: 172638.79172785708, Loss: 2.4809908866882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12927 , TFLOPS: 68.79334347374079, Tokens per sec: 170477.48334031168, Loss: 2.468994617462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12928 , TFLOPS: 69.75793821529031, Tokens per sec: 172867.85536875407, Loss: 2.4963016510009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12929 , TFLOPS: 70.7490897541861, Tokens per sec: 175324.03806649326, Loss: 2.4942188262939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12930 , TFLOPS: 68.64673964003435, Tokens per sec: 170114.18289064162, Loss: 2.4867818355560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12931 , TFLOPS: 70.71264731739289, Tokens per sec: 175233.72969365388, Loss: 2.4945597648620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12932 , TFLOPS: 70.40215444430102, Tokens per sec: 174464.29415051866, Loss: 2.5086634159088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12933 , TFLOPS: 69.69996513186467, Tokens per sec: 172724.191681763, Loss: 2.46028208732605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12934 , TFLOPS: 67.89238684038229, Tokens per sec: 168244.81355428294, Loss: 2.495438814163208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12935 , TFLOPS: 71.5228145402149, Tokens per sec: 177241.41331909396, Loss: 2.4876952171325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12936 , TFLOPS: 69.74765922864161, Tokens per sec: 172842.38290751472, Loss: 2.485508918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12937 , TFLOPS: 67.34385948454506, Tokens per sec: 166885.5023412418, Loss: 2.47615909576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12938 , TFLOPS: 70.73317615486819, Tokens per sec: 175284.60241435582, Loss: 2.490560293197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12939 , TFLOPS: 70.57124067344502, Tokens per sec: 174883.3084527233, Loss: 2.496087074279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12940 , TFLOPS: 68.93074606206494, Tokens per sec: 170817.98209032355, Loss: 2.486098051071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12941 , TFLOPS: 69.03060457131356, Tokens per sec: 171065.44247656452, Loss: 2.482933282852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12942 , TFLOPS: 70.77151499192244, Tokens per sec: 175379.61027594778, Loss: 2.482393980026245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12943 , TFLOPS: 69.58869987344848, Tokens per sec: 172448.4641145276, Loss: 2.493583917617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12944 , TFLOPS: 69.09778670884587, Tokens per sec: 171231.927214093, Loss: 2.4962029457092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12945 , TFLOPS: 70.71052293512307, Tokens per sec: 175228.46524036923, Loss: 2.497562885284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12946 , TFLOPS: 70.5643853190927, Tokens per sec: 174866.32012945862, Loss: 2.4905569553375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12947 , TFLOPS: 67.36218376981404, Tokens per sec: 166930.911938725, Loss: 2.4829235076904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12948 , TFLOPS: 70.63733646778121, Tokens per sec: 175047.1011121412, Loss: 2.4711337089538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12949 , TFLOPS: 70.6727693056237, Tokens per sec: 175134.90758756417, Loss: 2.4722189903259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12950 , TFLOPS: 68.71987086097361, Tokens per sec: 170295.41011219815, Loss: 2.4958834648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12951 , TFLOPS: 70.70348675889772, Tokens per sec: 175211.028820584, Loss: 2.4814679622650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12952 , TFLOPS: 71.55078310283919, Tokens per sec: 177310.72249826847, Loss: 2.5028328895568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12953 , TFLOPS: 69.70671152520488, Tokens per sec: 172740.9099876366, Loss: 2.5061497688293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12954 , TFLOPS: 67.5279195498202, Tokens per sec: 167341.6234588836, Loss: 2.488551616668701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12955 , TFLOPS: 70.93729985272395, Tokens per sec: 175790.44342372398, Loss: 2.477778434753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12956 , TFLOPS: 70.68271416110568, Tokens per sec: 175159.55203496478, Loss: 2.5024526119232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12957 , TFLOPS: 68.03531992638104, Tokens per sec: 168599.0174867671, Loss: 2.48978590965271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12958 , TFLOPS: 69.65990229552932, Tokens per sec: 172624.91156003822, Loss: 2.460023880004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12959 , TFLOPS: 70.62334165298712, Tokens per sec: 175012.4204194253, Loss: 2.486060380935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12960 , TFLOPS: 70.55951956471061, Tokens per sec: 174854.2622540927, Loss: 2.4811043739318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12961 , TFLOPS: 68.26734453336154, Tokens per sec: 169174.00002248524, Loss: 2.4889700412750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12962 , TFLOPS: 70.49692314251541, Tokens per sec: 174699.14142432786, Loss: 2.494353771209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12963 , TFLOPS: 70.99105305467543, Tokens per sec: 175923.64977956377, Loss: 2.5048811435699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12964 , TFLOPS: 68.50350611194095, Tokens per sec: 169759.2344295505, Loss: 2.5047390460968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12965 , TFLOPS: 69.66144542025023, Tokens per sec: 172628.7355930859, Loss: 2.4865942001342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12966 , TFLOPS: 70.61600982914797, Tokens per sec: 174994.2513522848, Loss: 2.4927728176116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12967 , TFLOPS: 67.8936343408376, Tokens per sec: 168247.9049978347, Loss: 2.492827892303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12968 , TFLOPS: 70.66488899595309, Tokens per sec: 175115.3793121126, Loss: 2.5023653507232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12969 , TFLOPS: 69.76510161877617, Tokens per sec: 172885.6070717061, Loss: 2.4905622005462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12970 , TFLOPS: 69.88848177034178, Tokens per sec: 173191.35667873136, Loss: 2.5114288330078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12971 , TFLOPS: 70.80808094141746, Tokens per sec: 175470.2247268683, Loss: 2.481224536895752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12972 , TFLOPS: 71.01735381680018, Tokens per sec: 175988.825965377, Loss: 2.4828295707702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12973 , TFLOPS: 68.74833179267011, Tokens per sec: 170365.9394361725, Loss: 2.483769655227661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12974 , TFLOPS: 68.73003235776892, Tokens per sec: 170320.5914206382, Loss: 2.5039148330688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12975 , TFLOPS: 70.63526846778035, Tokens per sec: 175041.97638033004, Loss: 2.487755060195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12976 , TFLOPS: 70.67850458796669, Tokens per sec: 175149.12024900358, Loss: 2.5064077377319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12977 , TFLOPS: 67.22462641740485, Tokens per sec: 166590.02966626172, Loss: 2.496670961380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12978 , TFLOPS: 70.74579709497857, Tokens per sec: 175315.87849426063, Loss: 2.485774517059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12979 , TFLOPS: 71.55560502000277, Tokens per sec: 177322.67174017787, Loss: 2.5108132362365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12980 , TFLOPS: 69.7967063715881, Tokens per sec: 172963.92713072497, Loss: 2.4864532947540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12981 , TFLOPS: 68.19056188388357, Tokens per sec: 168983.72415877174, Loss: 2.5034637451171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12982 , TFLOPS: 71.56052037423845, Tokens per sec: 177334.85252944482, Loss: 2.4842493534088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12983 , TFLOPS: 70.73653629090016, Tokens per sec: 175292.92920159257, Loss: 2.4771406650543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12984 , TFLOPS: 67.07583628789918, Tokens per sec: 166221.3113347608, Loss: 2.491726875305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12985 , TFLOPS: 70.6848479897276, Tokens per sec: 175164.83989735052, Loss: 2.4957923889160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12986 , TFLOPS: 70.66353322102394, Tokens per sec: 175112.01955248727, Loss: 2.488395929336548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12987 , TFLOPS: 68.73914563584954, Tokens per sec: 170343.17512763245, Loss: 2.4902782440185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12988 , TFLOPS: 70.06613674080587, Tokens per sec: 173631.60526585195, Loss: 2.4792399406433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12989 , TFLOPS: 70.69701452979223, Tokens per sec: 175194.9899238857, Loss: 2.50838041305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12990 , TFLOPS: 70.74615708234946, Tokens per sec: 175316.77058262157, Loss: 2.5109987258911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12991 , TFLOPS: 68.74704491067372, Tokens per sec: 170362.7504008237, Loss: 2.4911043643951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12992 , TFLOPS: 70.7294487878571, Tokens per sec: 175275.36558829894, Loss: 2.5087571144104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12993 , TFLOPS: 70.71975581588765, Tokens per sec: 175251.34533598964, Loss: 2.483999490737915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12994 , TFLOPS: 68.17711769787181, Tokens per sec: 168950.40798483524, Loss: 2.4668142795562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12995 , TFLOPS: 69.7231086870206, Tokens per sec: 172781.54396091244, Loss: 2.4655356407165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12996 , TFLOPS: 70.57971693373375, Tokens per sec: 174904.3135594559, Loss: 2.480914831161499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12997 , TFLOPS: 67.9713519087947, Tokens per sec: 168440.4976925293, Loss: 2.4754176139831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12998 , TFLOPS: 70.72672480286182, Tokens per sec: 175268.61525340893, Loss: 2.473123073577881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12999 , TFLOPS: 70.51751718896774, Tokens per sec: 174750.1757400582, Loss: 2.4883885383605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13000 , TFLOPS: 69.84760218704783, Tokens per sec: 173090.05256807109, Loss: 2.5018203258514404 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/optimizer.pt +[rank0]:[2024-08-27 05:28:38,858] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006385134998708963, 'preprocessing_with_comm': 0.0012547619990073144, 'state_converting': 0.9797117940033786, : 0.9886966299964115}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011000 + eval ppl=10.197332382202148, eval loss=2.3221261501312256 +------------------------------------------------------------------ +iteration: 13001 , TFLOPS: 63.33953665210473, Tokens per sec: 156962.34924987986, Loss: 2.4949100017547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13002 , TFLOPS: 70.0726831830181, Tokens per sec: 173647.82807080398, Loss: 2.492565870285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13003 , TFLOPS: 70.61128853634567, Tokens per sec: 174982.55146862738, Loss: 2.491624116897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13004 , TFLOPS: 68.26920972230901, Tokens per sec: 169178.6221661646, Loss: 2.5013766288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13005 , TFLOPS: 70.62969570676887, Tokens per sec: 175028.1664646533, Loss: 2.4880964756011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13006 , TFLOPS: 69.6225220788544, Tokens per sec: 172532.27926534702, Loss: 2.4907681941986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13007 , TFLOPS: 68.19870480110932, Tokens per sec: 169003.9032046735, Loss: 2.4843943119049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13008 , TFLOPS: 69.8504552492746, Tokens per sec: 173097.1227705598, Loss: 2.493821859359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13009 , TFLOPS: 70.38202824209492, Tokens per sec: 174414.41920436773, Loss: 2.512759208679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13010 , TFLOPS: 67.67304124471389, Tokens per sec: 167701.25100530457, Loss: 2.4761979579925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13011 , TFLOPS: 69.76434903172405, Tokens per sec: 172883.7420780866, Loss: 2.474907875061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13012 , TFLOPS: 71.50653453350891, Tokens per sec: 177201.06966908573, Loss: 2.525212526321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13013 , TFLOPS: 68.66534934279363, Tokens per sec: 170160.29978410713, Loss: 2.4926393032073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13014 , TFLOPS: 68.87562483097621, Tokens per sec: 170681.3856075808, Loss: 2.4989991188049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13015 , TFLOPS: 71.00365550778928, Tokens per sec: 175954.88004665467, Loss: 2.507979393005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13016 , TFLOPS: 70.64852992956413, Tokens per sec: 175074.8397435009, Loss: 2.4875149726867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13017 , TFLOPS: 68.56125505532614, Tokens per sec: 169902.34267283027, Loss: 2.49477481842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13018 , TFLOPS: 70.04683902034857, Tokens per sec: 173583.78338873934, Loss: 2.485459089279175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13019 , TFLOPS: 70.62487333222076, Tokens per sec: 175016.21608929464, Loss: 2.4833598136901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13020 , TFLOPS: 69.55350259645967, Tokens per sec: 172361.2413848492, Loss: 2.4986376762390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13021 , TFLOPS: 68.30334895628211, Tokens per sec: 169263.22294869702, Loss: 2.4963810443878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13022 , TFLOPS: 71.49252215014094, Tokens per sec: 177166.34543950844, Loss: 2.4850962162017822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13023 , TFLOPS: 70.84945778864184, Tokens per sec: 175572.76111232632, Loss: 2.5080552101135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13024 , TFLOPS: 67.38351455245312, Tokens per sec: 166983.77196788433, Loss: 2.47047758102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13025 , TFLOPS: 70.62842334773472, Tokens per sec: 175025.01341880474, Loss: 2.492342472076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13026 , TFLOPS: 70.65409191581983, Tokens per sec: 175088.62295814583, Loss: 2.4837698936462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13027 , TFLOPS: 68.83334266683481, Tokens per sec: 170576.6057470735, Loss: 2.5130929946899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13028 , TFLOPS: 69.64433127670056, Tokens per sec: 172586.32486009126, Loss: 2.4781365394592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13029 , TFLOPS: 70.56394458566636, Tokens per sec: 174865.22794347734, Loss: 2.4778811931610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13030 , TFLOPS: 69.64398843409757, Tokens per sec: 172585.4752583538, Loss: 2.492337942123413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13031 , TFLOPS: 69.98709219746934, Tokens per sec: 173435.72417998876, Loss: 2.4712891578674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13032 , TFLOPS: 69.92232574606983, Tokens per sec: 173275.22577880713, Loss: 2.492020845413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13033 , TFLOPS: 69.79682922960617, Tokens per sec: 172964.2315864277, Loss: 2.4715702533721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13034 , TFLOPS: 69.595527883797, Tokens per sec: 172465.3846763387, Loss: 2.4718947410583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13035 , TFLOPS: 69.07119774269896, Tokens per sec: 171166.03682696453, Loss: 2.478849172592163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13036 , TFLOPS: 70.65162494705598, Tokens per sec: 175082.50953778997, Loss: 2.5065717697143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13037 , TFLOPS: 68.68469007790645, Tokens per sec: 170208.22825627457, Loss: 2.4889509677886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13038 , TFLOPS: 69.8574561656332, Tokens per sec: 173114.47181251098, Loss: 2.492074489593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13039 , TFLOPS: 70.45386852115791, Tokens per sec: 174592.4473297459, Loss: 2.5074944496154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13040 , TFLOPS: 70.16091115990939, Tokens per sec: 173866.46671665457, Loss: 2.4815921783447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13041 , TFLOPS: 68.12714914484899, Tokens per sec: 168826.58040595416, Loss: 2.483375072479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13042 , TFLOPS: 71.50322009138452, Tokens per sec: 177192.85611638555, Loss: 2.481623888015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13043 , TFLOPS: 70.46928449983015, Tokens per sec: 174630.64982310595, Loss: 2.4993600845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13044 , TFLOPS: 68.11576098173158, Tokens per sec: 168798.3592832395, Loss: 2.5167789459228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13045 , TFLOPS: 70.62769594414897, Tokens per sec: 175023.21083258913, Loss: 2.50461745262146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13046 , TFLOPS: 70.66611318817657, Tokens per sec: 175118.41299529833, Loss: 2.4996178150177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13047 , TFLOPS: 67.78062005804287, Tokens per sec: 167967.84315551838, Loss: 2.508746862411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13048 , TFLOPS: 69.88776666829328, Tokens per sec: 173189.58457708015, Loss: 2.496504306793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13049 , TFLOPS: 70.58621691164741, Tokens per sec: 174920.42122075762, Loss: 2.4902992248535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13050 , TFLOPS: 69.6535657795689, Tokens per sec: 172609.20897546347, Loss: 2.4699490070343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13051 , TFLOPS: 69.05891220121968, Tokens per sec: 171135.5919018147, Loss: 2.4890499114990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13052 , TFLOPS: 70.61310811731663, Tokens per sec: 174987.06059071684, Loss: 2.491914749145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13053 , TFLOPS: 70.69324050991645, Tokens per sec: 175185.63748689092, Loss: 2.506516456604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13054 , TFLOPS: 67.1905505452835, Tokens per sec: 166505.58590137784, Loss: 2.4709439277648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13055 , TFLOPS: 69.87686713221849, Tokens per sec: 173162.57432600384, Loss: 2.494326114654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13056 , TFLOPS: 70.66953689190343, Tokens per sec: 175126.89731028557, Loss: 2.4986753463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13057 , TFLOPS: 67.31719222877707, Tokens per sec: 166819.41794380802, Loss: 2.4797210693359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13058 , TFLOPS: 69.82866550587288, Tokens per sec: 173043.12538607192, Loss: 2.471226453781128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13059 , TFLOPS: 71.41539136013317, Tokens per sec: 176975.20684521127, Loss: 2.484875202178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13060 , TFLOPS: 69.70818996276064, Tokens per sec: 172744.57371876313, Loss: 2.477440595626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13061 , TFLOPS: 67.60061491631029, Tokens per sec: 167521.77058510177, Loss: 2.4868977069854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13062 , TFLOPS: 71.4740993902691, Tokens per sec: 177120.69181111152, Loss: 2.491071939468384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13063 , TFLOPS: 70.73710687625018, Tokens per sec: 175294.34317494507, Loss: 2.479326009750366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13064 , TFLOPS: 68.03296887502276, Tokens per sec: 168593.19132251161, Loss: 2.509904623031616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13065 , TFLOPS: 69.43432799422285, Tokens per sec: 172065.9135923402, Loss: 2.507584810256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13066 , TFLOPS: 70.57860410324975, Tokens per sec: 174901.55584292812, Loss: 2.5007245540618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13067 , TFLOPS: 69.79011766406131, Tokens per sec: 172947.5995876675, Loss: 2.472634792327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13068 , TFLOPS: 69.19880189688186, Tokens per sec: 171482.25397778777, Loss: 2.4815778732299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13069 , TFLOPS: 70.08833511325794, Tokens per sec: 173686.61527814076, Loss: 2.4907498359680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13070 , TFLOPS: 69.95878454053852, Tokens per sec: 173365.5746877683, Loss: 2.516329526901245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13071 , TFLOPS: 68.72933116536102, Tokens per sec: 170318.85378860869, Loss: 2.4825563430786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13072 , TFLOPS: 69.54845371129166, Tokens per sec: 172348.72969122225, Loss: 2.473130702972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13073 , TFLOPS: 70.55197124571114, Tokens per sec: 174835.55668809605, Loss: 2.477081060409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13074 , TFLOPS: 68.69810527135604, Tokens per sec: 170241.47258344817, Loss: 2.4908742904663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13075 , TFLOPS: 69.95817668146233, Tokens per sec: 173364.06834601634, Loss: 2.5123214721679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13076 , TFLOPS: 69.75429507218448, Tokens per sec: 172858.8272588125, Loss: 2.493121862411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13077 , TFLOPS: 68.68535341579513, Tokens per sec: 170209.8720806354, Loss: 2.483349561691284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13078 , TFLOPS: 69.72297413336061, Tokens per sec: 172781.2105221207, Loss: 2.4810280799865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13079 , TFLOPS: 71.51469203695137, Tokens per sec: 177221.28486123934, Loss: 2.504584550857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13080 , TFLOPS: 68.8488693532125, Tokens per sec: 170615.08258632317, Loss: 2.487971067428589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13081 , TFLOPS: 68.75690730206078, Tokens per sec: 170387.19049311295, Loss: 2.4782748222351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13082 , TFLOPS: 70.98391140442342, Tokens per sec: 175905.95198352158, Loss: 2.4803390502929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13083 , TFLOPS: 70.71894017226128, Tokens per sec: 175249.32408123295, Loss: 2.49236798286438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13084 , TFLOPS: 67.39692679712317, Tokens per sec: 167017.00898765743, Loss: 2.4813318252563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13085 , TFLOPS: 70.0569280002869, Tokens per sec: 173608.78499241945, Loss: 2.482227325439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13086 , TFLOPS: 70.65243427730981, Tokens per sec: 175084.51514731406, Loss: 2.4888134002685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13087 , TFLOPS: 69.6851386190689, Tokens per sec: 172687.44994977984, Loss: 2.478701114654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13088 , TFLOPS: 68.37427939985348, Tokens per sec: 169438.99640150025, Loss: 2.5066847801208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13089 , TFLOPS: 71.54235095669999, Tokens per sec: 177289.82671125795, Loss: 2.4885611534118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13090 , TFLOPS: 70.74775393676181, Tokens per sec: 175320.72776376345, Loss: 2.4819600582122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13091 , TFLOPS: 67.18095669615731, Tokens per sec: 166481.8112863934, Loss: 2.4655580520629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13092 , TFLOPS: 70.63139139275148, Tokens per sec: 175032.36856131343, Loss: 2.4803404808044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13093 , TFLOPS: 70.58582198971196, Tokens per sec: 174919.44256069732, Loss: 2.4826877117156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13094 , TFLOPS: 67.82988042768966, Tokens per sec: 168089.91577798084, Loss: 2.4972100257873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13095 , TFLOPS: 69.89178228104909, Tokens per sec: 173199.53570784532, Loss: 2.498990058898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13096 , TFLOPS: 70.70436625396636, Tokens per sec: 175213.20830624847, Loss: 2.4687435626983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13097 , TFLOPS: 68.46340166570636, Tokens per sec: 169659.85119391172, Loss: 2.49259877204895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13098 , TFLOPS: 69.62375638408541, Tokens per sec: 172535.3380096794, Loss: 2.4887661933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13099 , TFLOPS: 70.9050003375309, Tokens per sec: 175710.40166699022, Loss: 2.4876832962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13100 , TFLOPS: 69.73383837542553, Tokens per sec: 172808.13331075263, Loss: 2.4962470531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13101 , TFLOPS: 69.51695758126463, Tokens per sec: 172270.67881143015, Loss: 2.475013494491577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13102 , TFLOPS: 69.2000894849202, Tokens per sec: 171485.44476278627, Loss: 2.4919586181640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13103 , TFLOPS: 70.73012255746299, Tokens per sec: 175277.0352636042, Loss: 2.4915480613708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13104 , TFLOPS: 68.77770870866425, Tokens per sec: 170438.73867014045, Loss: 2.5191266536712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13105 , TFLOPS: 69.0064139237205, Tokens per sec: 171005.49538700908, Loss: 2.4915990829467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13106 , TFLOPS: 70.44933164089618, Tokens per sec: 174581.2044406787, Loss: 2.4923388957977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13107 , TFLOPS: 70.05302164597639, Tokens per sec: 173599.10461611752, Loss: 2.4881248474121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13108 , TFLOPS: 68.06918570922545, Tokens per sec: 168682.9406272795, Loss: 2.4846930503845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13109 , TFLOPS: 70.51716201370175, Tokens per sec: 174749.2955766233, Loss: 2.5083794593811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13110 , TFLOPS: 70.67864544022731, Tokens per sec: 175149.46929642156, Loss: 2.5097861289978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13111 , TFLOPS: 67.91302004347358, Tokens per sec: 168295.94490447812, Loss: 2.4820761680603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13112 , TFLOPS: 70.61465551468534, Tokens per sec: 174990.89521185635, Loss: 2.500872850418091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13113 , TFLOPS: 69.53967821555736, Tokens per sec: 172326.98304609203, Loss: 2.5007987022399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13114 , TFLOPS: 68.7839102222748, Tokens per sec: 170454.1067040205, Loss: 2.4901485443115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13115 , TFLOPS: 69.91114709108115, Tokens per sec: 173247.5238403147, Loss: 2.4880292415618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13116 , TFLOPS: 70.74437594662398, Tokens per sec: 175312.35673208555, Loss: 2.4976720809936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13117 , TFLOPS: 68.71363584359113, Tokens per sec: 170279.9590522805, Loss: 2.517112970352173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13118 , TFLOPS: 69.92429852646478, Tokens per sec: 173280.11454594554, Loss: 2.486584186553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13119 , TFLOPS: 70.72834410929448, Tokens per sec: 175272.62807313039, Loss: 2.4951930046081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13120 , TFLOPS: 69.80475906349716, Tokens per sec: 172983.88258835283, Loss: 2.4853477478027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13121 , TFLOPS: 68.59332215071242, Tokens per sec: 169981.80846767966, Loss: 2.489607334136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13122 , TFLOPS: 69.81947697110789, Tokens per sec: 173020.35518472362, Loss: 2.481585741043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13123 , TFLOPS: 70.6497737963299, Tokens per sec: 175077.9221823697, Loss: 2.4965288639068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13124 , TFLOPS: 68.63377633910446, Tokens per sec: 170082.05840291188, Loss: 2.512110948562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13125 , TFLOPS: 69.92936412215559, Tokens per sec: 173292.66765008878, Loss: 2.4792542457580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13126 , TFLOPS: 71.54195606162794, Tokens per sec: 177288.84811776815, Loss: 2.5158777236938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13127 , TFLOPS: 69.86746823754866, Tokens per sec: 173139.28282963825, Loss: 2.488449811935425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13128 , TFLOPS: 67.84720860104318, Tokens per sec: 168132.8569004067, Loss: 2.5016791820526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13129 , TFLOPS: 70.80150391860487, Tokens per sec: 175453.92613981984, Loss: 2.4809985160827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13130 , TFLOPS: 70.60021988356571, Tokens per sec: 174955.12212772592, Loss: 2.4753777980804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13131 , TFLOPS: 67.87316580365729, Tokens per sec: 168197.1817079064, Loss: 2.49269437789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13132 , TFLOPS: 70.1720838155129, Tokens per sec: 173894.15378800972, Loss: 2.5012855529785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13133 , TFLOPS: 70.6073824918593, Tokens per sec: 174972.87186010386, Loss: 2.4680888652801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13134 , TFLOPS: 69.79884619943465, Tokens per sec: 172969.22985985022, Loss: 2.485311985015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13135 , TFLOPS: 67.98856985905334, Tokens per sec: 168483.165669984, Loss: 2.479623317718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13136 , TFLOPS: 70.70051167934945, Tokens per sec: 175203.6562457313, Loss: 2.5060677528381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13137 , TFLOPS: 70.62882516315399, Tokens per sec: 175026.0091616773, Loss: 2.5221078395843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13138 , TFLOPS: 69.10297777282672, Tokens per sec: 171244.79124248045, Loss: 2.484999656677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13139 , TFLOPS: 69.70174009373444, Tokens per sec: 172728.59023280448, Loss: 2.501281499862671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13140 , TFLOPS: 70.61011734991709, Tokens per sec: 174979.6491396405, Loss: 2.48530912399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13141 , TFLOPS: 69.001040988129, Tokens per sec: 170992.18066073564, Loss: 2.490600109100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13142 , TFLOPS: 69.72716713795027, Tokens per sec: 172791.60124939147, Loss: 2.499452590942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13143 , TFLOPS: 69.54971788996575, Tokens per sec: 172351.86246523744, Loss: 2.485231637954712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13144 , TFLOPS: 68.49219367876309, Tokens per sec: 169731.20097396837, Loss: 2.4862537384033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13145 , TFLOPS: 69.98578317703434, Tokens per sec: 173432.4802831503, Loss: 2.4853568077087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13146 , TFLOPS: 70.54096132205966, Tokens per sec: 174808.27288444445, Loss: 2.5029149055480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13147 , TFLOPS: 69.69924746839956, Tokens per sec: 172722.4132326393, Loss: 2.479658365249634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13148 , TFLOPS: 68.80181959850046, Tokens per sec: 170498.48811118834, Loss: 2.4781529903411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13149 , TFLOPS: 70.65760967707351, Tokens per sec: 175097.34035804585, Loss: 2.482174873352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13150 , TFLOPS: 69.6514140313739, Tokens per sec: 172603.87670640164, Loss: 2.5263493061065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13151 , TFLOPS: 68.80829652216818, Tokens per sec: 170514.53864152898, Loss: 2.4644277095794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13152 , TFLOPS: 69.93274026962139, Tokens per sec: 173301.0341154224, Loss: 2.4735119342803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13153 , TFLOPS: 70.58593572748072, Tokens per sec: 174919.7244154176, Loss: 2.488431453704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13154 , TFLOPS: 69.5035732875767, Tokens per sec: 172237.51105726976, Loss: 2.4932479858398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13155 , TFLOPS: 68.1609809278804, Tokens per sec: 168910.41929118455, Loss: 2.4935567378997803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13156 , TFLOPS: 71.50525572833928, Tokens per sec: 177197.9006490034, Loss: 2.4870429039001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13157 , TFLOPS: 70.9300512266476, Tokens per sec: 175772.48052980468, Loss: 2.4836745262145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13158 , TFLOPS: 67.86830172946652, Tokens per sec: 168185.12799623967, Loss: 2.507417917251587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13159 , TFLOPS: 70.7175157008369, Tokens per sec: 175245.79408412485, Loss: 2.4714207649230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13160 , TFLOPS: 70.60915406149472, Tokens per sec: 174977.26200481493, Loss: 2.508498191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13161 , TFLOPS: 67.77980786517148, Tokens per sec: 167965.83045211266, Loss: 2.512166738510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13162 , TFLOPS: 69.72856293757832, Tokens per sec: 172795.06019462977, Loss: 2.482491970062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13163 , TFLOPS: 70.64918173824685, Tokens per sec: 175076.4549972193, Loss: 2.4857778549194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13164 , TFLOPS: 68.93093049856097, Tokens per sec: 170818.43914427803, Loss: 2.4774580001831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13165 , TFLOPS: 69.71615282079651, Tokens per sec: 172764.30655815723, Loss: 2.4899940490722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13166 , TFLOPS: 70.95156550177761, Tokens per sec: 175825.7952735699, Loss: 2.509359359741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13167 , TFLOPS: 69.81069383459389, Tokens per sec: 172998.58960489955, Loss: 2.506857395172119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13168 , TFLOPS: 68.4506340546555, Tokens per sec: 169628.21164725925, Loss: 2.4715735912323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13169 , TFLOPS: 69.25061530330566, Tokens per sec: 171610.65330662383, Loss: 2.489069938659668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13170 , TFLOPS: 70.56656976877683, Tokens per sec: 174871.73343641288, Loss: 2.489797830581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13171 , TFLOPS: 68.96262831951034, Tokens per sec: 170896.98983639383, Loss: 2.511040210723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13172 , TFLOPS: 68.97655079084268, Tokens per sec: 170931.4912540422, Loss: 2.503298759460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13173 , TFLOPS: 71.49351372780386, Tokens per sec: 177168.80267818843, Loss: 2.495922088623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13174 , TFLOPS: 70.04867769980706, Tokens per sec: 173588.33983898477, Loss: 2.5006802082061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13175 , TFLOPS: 67.86700706796569, Tokens per sec: 168181.91968242187, Loss: 2.4773497581481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13176 , TFLOPS: 70.42437280415693, Tokens per sec: 174519.353693796, Loss: 2.4664416313171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13177 , TFLOPS: 70.57005210561731, Tokens per sec: 174880.36305071463, Loss: 2.489502429962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13178 , TFLOPS: 68.15243712889273, Tokens per sec: 168889.24681611598, Loss: 2.48221492767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13179 , TFLOPS: 70.67982254319419, Tokens per sec: 175152.38628724212, Loss: 2.4870903491973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13180 , TFLOPS: 69.70575074802434, Tokens per sec: 172738.5290759449, Loss: 2.468562364578247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13181 , TFLOPS: 68.73194408064083, Tokens per sec: 170325.32888050846, Loss: 2.491997241973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13182 , TFLOPS: 69.80588582098687, Tokens per sec: 172986.67481753495, Loss: 2.4857311248779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13183 , TFLOPS: 69.50967856713525, Tokens per sec: 172252.64061256798, Loss: 2.49239182472229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13184 , TFLOPS: 69.8813305033858, Tokens per sec: 173173.6350514373, Loss: 2.4767138957977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13185 , TFLOPS: 69.90142035583257, Tokens per sec: 173223.41991888036, Loss: 2.500775098800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13186 , TFLOPS: 70.6732125786365, Tokens per sec: 175136.00606692044, Loss: 2.494478464126587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13187 , TFLOPS: 68.83787025135422, Tokens per sec: 170587.82560026724, Loss: 2.505253553390503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13188 , TFLOPS: 69.69974004606277, Tokens per sec: 172723.63389435064, Loss: 2.4789884090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13189 , TFLOPS: 69.71320368618964, Tokens per sec: 172756.9982777274, Loss: 2.4909510612487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13190 , TFLOPS: 70.54216096914107, Tokens per sec: 174811.24574206423, Loss: 2.492006540298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13191 , TFLOPS: 67.95682186544597, Tokens per sec: 168404.49064449474, Loss: 2.481706142425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13192 , TFLOPS: 69.90505079674075, Tokens per sec: 173232.41655137695, Loss: 2.4803624153137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13193 , TFLOPS: 71.52333568440541, Tokens per sec: 177242.7047718068, Loss: 2.4982731342315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13194 , TFLOPS: 69.79971958329097, Tokens per sec: 172971.39420125735, Loss: 2.475109100341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13195 , TFLOPS: 67.03345171313063, Tokens per sec: 166116.27768943802, Loss: 2.477597236633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13196 , TFLOPS: 71.48150580635193, Tokens per sec: 177139.04572604885, Loss: 2.493558168411255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13197 , TFLOPS: 70.61462692984371, Tokens per sec: 174990.8243754699, Loss: 2.4828364849090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13198 , TFLOPS: 67.11395417465965, Tokens per sec: 166315.77165718516, Loss: 2.510134220123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13199 , TFLOPS: 70.70106964586488, Tokens per sec: 175205.03894822166, Loss: 2.5042543411254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13200 , TFLOPS: 70.67996296681086, Tokens per sec: 175152.73427243318, Loss: 2.46927809715271 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/model.pt +[rank0]:[2024-08-27 05:54:05,824] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006386103996192105, 'preprocessing_with_comm': 0.0013299470010679215, 'state_converting': 1.0034319029946346, : 1.012525447004009}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011200 + eval ppl=9.961742401123047, eval loss=2.2987520694732666 +------------------------------------------------------------------ +iteration: 13201 , TFLOPS: 61.97992478707714, Tokens per sec: 153593.08127473042, Loss: 2.508850574493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13202 , TFLOPS: 68.27201403560161, Tokens per sec: 169185.57156342428, Loss: 2.482771635055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13203 , TFLOPS: 71.45507978589598, Tokens per sec: 177073.55913629252, Loss: 2.4976439476013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13204 , TFLOPS: 69.74537548500547, Tokens per sec: 172836.72353920923, Loss: 2.460662841796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13205 , TFLOPS: 68.00767677612043, Tokens per sec: 168530.51471527692, Loss: 2.5001001358032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13206 , TFLOPS: 70.43119342795103, Tokens per sec: 174536.2559509115, Loss: 2.4808990955352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13207 , TFLOPS: 69.68635654366075, Tokens per sec: 172690.46810108394, Loss: 2.488112449645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13208 , TFLOPS: 68.95262201783822, Tokens per sec: 170872.1931185652, Loss: 2.49259090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13209 , TFLOPS: 69.59308588969577, Tokens per sec: 172459.333145948, Loss: 2.489901542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13210 , TFLOPS: 70.597448362674, Tokens per sec: 174948.25399364802, Loss: 2.4887216091156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13211 , TFLOPS: 70.00272822643363, Tokens per sec: 173474.47198221303, Loss: 2.4747633934020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13212 , TFLOPS: 68.83166503584224, Tokens per sec: 170572.44839266763, Loss: 2.4885568618774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13213 , TFLOPS: 70.45790393938624, Tokens per sec: 174602.4475406537, Loss: 2.4703614711761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13214 , TFLOPS: 69.99576471878866, Tokens per sec: 173457.2156431751, Loss: 2.4890856742858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13215 , TFLOPS: 69.29416566776669, Tokens per sec: 171718.5759072269, Loss: 2.5023112297058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13216 , TFLOPS: 70.646612641631, Tokens per sec: 175070.08849279585, Loss: 2.478271245956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13217 , TFLOPS: 68.64367427841361, Tokens per sec: 170106.58658686798, Loss: 2.4794487953186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13218 , TFLOPS: 69.54304713004039, Tokens per sec: 172335.33158729752, Loss: 2.5036892890930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13219 , TFLOPS: 69.75671453371105, Tokens per sec: 172864.822951001, Loss: 2.5069963932037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13220 , TFLOPS: 69.61054307267068, Tokens per sec: 172502.59396842727, Loss: 2.488002061843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13221 , TFLOPS: 68.84855298277812, Tokens per sec: 170614.29858553567, Loss: 2.476142644882202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13222 , TFLOPS: 69.8972345229943, Tokens per sec: 173213.04696399957, Loss: 2.4804983139038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13223 , TFLOPS: 71.49199641564566, Tokens per sec: 177165.0426115147, Loss: 2.4913034439086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13224 , TFLOPS: 68.66104877972919, Tokens per sec: 170149.6425150873, Loss: 2.483247995376587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13225 , TFLOPS: 68.07592762713128, Tokens per sec: 168699.6478425337, Loss: 2.4795172214508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13226 , TFLOPS: 71.51442405963695, Tokens per sec: 177220.62078393405, Loss: 2.5100865364074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13227 , TFLOPS: 70.66292446534068, Tokens per sec: 175110.51098884415, Loss: 2.4769489765167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13228 , TFLOPS: 67.11820737793441, Tokens per sec: 166326.31156342902, Loss: 2.4813950061798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13229 , TFLOPS: 70.74340785711405, Tokens per sec: 175309.95769963608, Loss: 2.4899110794067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13230 , TFLOPS: 71.55105820860541, Tokens per sec: 177311.40424066258, Loss: 2.4959425926208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13231 , TFLOPS: 68.12415878839985, Tokens per sec: 168819.16997325822, Loss: 2.487483263015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13232 , TFLOPS: 68.85381368933966, Tokens per sec: 170627.33519590984, Loss: 2.491218090057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13233 , TFLOPS: 71.50644017710238, Tokens per sec: 177200.83584351614, Loss: 2.494194984436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13234 , TFLOPS: 69.86992188192727, Tokens per sec: 173145.3632306997, Loss: 2.5019967555999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13235 , TFLOPS: 68.68982344495781, Tokens per sec: 170220.94930531486, Loss: 2.4794347286224365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13236 , TFLOPS: 70.67138495591377, Tokens per sec: 175131.47701648428, Loss: 2.519624710083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13237 , TFLOPS: 69.68990367644282, Tokens per sec: 172699.25828686683, Loss: 2.495699644088745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13238 , TFLOPS: 69.6387579172844, Tokens per sec: 172572.5134615018, Loss: 2.4821832180023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13239 , TFLOPS: 69.64398594220131, Tokens per sec: 172585.4690831604, Loss: 2.4844605922698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13240 , TFLOPS: 70.73647688251724, Tokens per sec: 175292.7819810753, Loss: 2.483818292617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13241 , TFLOPS: 69.8007027289463, Tokens per sec: 172973.8305444937, Loss: 2.484940528869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13242 , TFLOPS: 69.0867093434129, Tokens per sec: 171204.47628227755, Loss: 2.4926369190216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13243 , TFLOPS: 70.49412615094748, Tokens per sec: 174692.21017110845, Loss: 2.5115575790405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13244 , TFLOPS: 69.69256535240531, Tokens per sec: 172705.8542131097, Loss: 2.478095054626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13245 , TFLOPS: 68.86310522270529, Tokens per sec: 170650.36063913675, Loss: 2.4876294136047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13246 , TFLOPS: 69.83699961452453, Tokens per sec: 173063.77822538855, Loss: 2.4617228507995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13247 , TFLOPS: 70.70187990158163, Tokens per sec: 175207.0468511448, Loss: 2.4926154613494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13248 , TFLOPS: 68.62831609028859, Tokens per sec: 170068.52730485008, Loss: 2.4864208698272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13249 , TFLOPS: 70.14510411706608, Tokens per sec: 173827.29512319935, Loss: 2.495509386062622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13250 , TFLOPS: 70.4214413996704, Tokens per sec: 174512.08935055754, Loss: 2.488477945327759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13251 , TFLOPS: 69.72068050167947, Tokens per sec: 172775.52665015042, Loss: 2.479069709777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13252 , TFLOPS: 68.31972906106323, Tokens per sec: 169303.8146527621, Loss: 2.478398084640503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13253 , TFLOPS: 71.54533250220811, Tokens per sec: 177297.21530947144, Loss: 2.5011205673217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13254 , TFLOPS: 68.6396277163244, Tokens per sec: 170096.55876024527, Loss: 2.4962105751037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13255 , TFLOPS: 68.7334894742702, Tokens per sec: 170329.1585361612, Loss: 2.50258731842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13256 , TFLOPS: 70.64966742310172, Tokens per sec: 175077.65857779223, Loss: 2.4846770763397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13257 , TFLOPS: 69.71539675441971, Tokens per sec: 172762.43294238797, Loss: 2.4932801723480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13258 , TFLOPS: 69.65296201261532, Tokens per sec: 172607.71277444207, Loss: 2.4786887168884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13259 , TFLOPS: 69.11068720026785, Tokens per sec: 171263.89605294308, Loss: 2.498927116394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13260 , TFLOPS: 70.72037983445516, Tokens per sec: 175252.89172274122, Loss: 2.4784913063049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13261 , TFLOPS: 68.89256714233191, Tokens per sec: 170723.3705214699, Loss: 2.4985411167144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13262 , TFLOPS: 69.7209388770958, Tokens per sec: 172776.16693289453, Loss: 2.505467653274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13263 , TFLOPS: 71.01585170421512, Tokens per sec: 175985.10356491987, Loss: 2.4909369945526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13264 , TFLOPS: 69.71614611912422, Tokens per sec: 172764.28995067504, Loss: 2.5013036727905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13265 , TFLOPS: 68.6137094766428, Tokens per sec: 170032.33050718447, Loss: 2.485842704772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13266 , TFLOPS: 70.08597508314747, Tokens per sec: 173680.76686354826, Loss: 2.520563840866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13267 , TFLOPS: 70.65119758585537, Tokens per sec: 175081.45048965715, Loss: 2.504648447036743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13268 , TFLOPS: 67.90054740799452, Tokens per sec: 168265.03633978678, Loss: 2.5174343585968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13269 , TFLOPS: 70.01692978815561, Tokens per sec: 173509.66501659126, Loss: 2.495126962661743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13270 , TFLOPS: 71.52606842413216, Tokens per sec: 177249.47680188584, Loss: 2.4976487159729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13271 , TFLOPS: 69.67546453460395, Tokens per sec: 172663.47650278732, Loss: 2.505288600921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13272 , TFLOPS: 66.68514393308904, Tokens per sec: 165253.1326412255, Loss: 2.4852170944213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13273 , TFLOPS: 71.5113216946945, Tokens per sec: 177212.93278185296, Loss: 2.5080153942108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13274 , TFLOPS: 69.8497433397022, Tokens per sec: 173095.35858021604, Loss: 2.4845142364501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13275 , TFLOPS: 68.84785624604876, Tokens per sec: 170612.57199516843, Loss: 2.5063252449035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13276 , TFLOPS: 69.68481939594712, Tokens per sec: 172686.65887971874, Loss: 2.499891996383667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13277 , TFLOPS: 70.58693804054582, Tokens per sec: 174922.20825760678, Loss: 2.501826763153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13278 , TFLOPS: 69.98959374029062, Tokens per sec: 173441.92327866756, Loss: 2.4657249450683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13279 , TFLOPS: 68.76446714235554, Tokens per sec: 170405.9246101485, Loss: 2.49570369720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13280 , TFLOPS: 70.50673932803052, Tokens per sec: 174723.4670133764, Loss: 2.4955828189849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13281 , TFLOPS: 69.31487086435094, Tokens per sec: 171769.8856652284, Loss: 2.4893720149993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13282 , TFLOPS: 69.7277212708014, Tokens per sec: 172792.9744516393, Loss: 2.4833662509918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13283 , TFLOPS: 69.70134371075221, Tokens per sec: 172727.60795210933, Loss: 2.475456476211548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13284 , TFLOPS: 69.61574594559222, Tokens per sec: 172515.48726067063, Loss: 2.474245309829712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13285 , TFLOPS: 69.6780192377432, Tokens per sec: 172669.80733859015, Loss: 2.480290412902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13286 , TFLOPS: 69.73954945610639, Tokens per sec: 172822.28599780728, Loss: 2.495351552963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13287 , TFLOPS: 69.58694728266457, Tokens per sec: 172444.1210015013, Loss: 2.489919424057007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13288 , TFLOPS: 69.74774183361312, Tokens per sec: 172842.58761173385, Loss: 2.4918792247772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13289 , TFLOPS: 69.26579094715467, Tokens per sec: 171648.26022381685, Loss: 2.5024943351745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13290 , TFLOPS: 71.53485747416087, Tokens per sec: 177271.25703605075, Loss: 2.489591121673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13291 , TFLOPS: 68.88319820955253, Tokens per sec: 170700.15327396945, Loss: 2.4851653575897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13292 , TFLOPS: 68.8824200326529, Tokens per sec: 170698.2248658889, Loss: 2.4885029792785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13293 , TFLOPS: 70.92195945485214, Tokens per sec: 175752.4282279133, Loss: 2.4905245304107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13294 , TFLOPS: 69.55758868165873, Tokens per sec: 172371.36715409192, Loss: 2.48183274269104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13295 , TFLOPS: 69.14650855258982, Tokens per sec: 171352.66531004463, Loss: 2.472612142562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13296 , TFLOPS: 70.13379777492023, Tokens per sec: 173799.2767618655, Loss: 2.470646858215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13297 , TFLOPS: 71.53780719236028, Tokens per sec: 177278.5667626872, Loss: 2.480973243713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13298 , TFLOPS: 68.18763953445307, Tokens per sec: 168976.48225495958, Loss: 2.469133138656616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13299 , TFLOPS: 68.748443194396, Tokens per sec: 170366.2155019212, Loss: 2.5086214542388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13300 , TFLOPS: 71.51319767508268, Tokens per sec: 177217.58166791114, Loss: 2.4725639820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13301 , TFLOPS: 69.8061954685177, Tokens per sec: 172987.44215822683, Loss: 2.4985907077789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13302 , TFLOPS: 68.2266516785556, Tokens per sec: 169073.1586455881, Loss: 2.507448673248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13303 , TFLOPS: 70.68709512413886, Tokens per sec: 175170.40854396392, Loss: 2.4910361766815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13304 , TFLOPS: 69.77094910288481, Tokens per sec: 172900.0977814808, Loss: 2.484750270843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13305 , TFLOPS: 68.55536750701177, Tokens per sec: 169887.75267954028, Loss: 2.484475612640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13306 , TFLOPS: 69.71355360881026, Tokens per sec: 172757.86542452846, Loss: 2.469925880432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13307 , TFLOPS: 70.60284063899378, Tokens per sec: 174961.61664837666, Loss: 2.484959602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13308 , TFLOPS: 69.8501092040843, Tokens per sec: 173096.2652324582, Loss: 2.4675841331481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13309 , TFLOPS: 67.92875607111294, Tokens per sec: 168334.940514435, Loss: 2.4942963123321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13310 , TFLOPS: 71.51137883787935, Tokens per sec: 177213.07438895985, Loss: 2.48270845413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13311 , TFLOPS: 69.81998174624184, Tokens per sec: 173021.60607311112, Loss: 2.5002493858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13312 , TFLOPS: 68.84387659888594, Tokens per sec: 170602.70999110464, Loss: 2.5038976669311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13313 , TFLOPS: 69.59640518013342, Tokens per sec: 172467.5587133025, Loss: 2.486544370651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13314 , TFLOPS: 70.61181445050157, Tokens per sec: 174983.8547418399, Loss: 2.466400623321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13315 , TFLOPS: 68.89716908388041, Tokens per sec: 170734.77463957277, Loss: 2.495436906814575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13316 , TFLOPS: 69.92359171368848, Tokens per sec: 173278.36298602403, Loss: 2.4716365337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13317 , TFLOPS: 70.52773700538191, Tokens per sec: 174775.5015709382, Loss: 2.4959354400634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13318 , TFLOPS: 69.80286130001149, Tokens per sec: 172979.17972137945, Loss: 2.492025852203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13319 , TFLOPS: 68.17403326231329, Tokens per sec: 168942.76441374258, Loss: 2.501835346221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13320 , TFLOPS: 70.39462891644051, Tokens per sec: 174445.64506347498, Loss: 2.5178818702697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13321 , TFLOPS: 69.33434164736768, Tokens per sec: 171818.1364104277, Loss: 2.4873502254486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13322 , TFLOPS: 68.87082302531766, Tokens per sec: 170669.48620419635, Loss: 2.473663091659546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13323 , TFLOPS: 70.6858853518982, Tokens per sec: 175167.4105950841, Loss: 2.4716248512268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13324 , TFLOPS: 69.16707872789549, Tokens per sec: 171403.64047044393, Loss: 2.493608236312866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13325 , TFLOPS: 70.05975180677983, Tokens per sec: 173615.78269597687, Loss: 2.466494560241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13326 , TFLOPS: 69.07118129747117, Tokens per sec: 171165.99607387846, Loss: 2.489835739135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13327 , TFLOPS: 71.53522206651137, Tokens per sec: 177272.16053605845, Loss: 2.510530471801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13328 , TFLOPS: 68.298713925499, Tokens per sec: 169251.7368318257, Loss: 2.475850820541382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13329 , TFLOPS: 69.67287613077806, Tokens per sec: 172657.06215297055, Loss: 2.519343137741089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13330 , TFLOPS: 70.90678103342407, Tokens per sec: 175714.81442757338, Loss: 2.5035948753356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13331 , TFLOPS: 69.87893140594679, Tokens per sec: 173167.68982370108, Loss: 2.486191987991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13332 , TFLOPS: 68.78302392567022, Tokens per sec: 170451.91036340073, Loss: 2.4864344596862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13333 , TFLOPS: 70.10575560529071, Tokens per sec: 173729.78517642358, Loss: 2.510572910308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13334 , TFLOPS: 70.58547661884518, Tokens per sec: 174918.58669364374, Loss: 2.4992120265960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13335 , TFLOPS: 67.78107781144804, Tokens per sec: 167968.97751888214, Loss: 2.4754958152770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13336 , TFLOPS: 69.83131944462399, Tokens per sec: 173049.70213865265, Loss: 2.471933603286743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13337 , TFLOPS: 71.52723759823402, Tokens per sec: 177252.37414410542, Loss: 2.4970688819885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13338 , TFLOPS: 69.90052919730718, Tokens per sec: 173221.21152988524, Loss: 2.4803833961486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13339 , TFLOPS: 66.70312432499401, Tokens per sec: 165297.6900330697, Loss: 2.466592788696289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13340 , TFLOPS: 71.50058992711827, Tokens per sec: 177186.33827959726, Loss: 2.502800941467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13341 , TFLOPS: 69.69190126063803, Tokens per sec: 172704.20852055465, Loss: 2.4981579780578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13342 , TFLOPS: 68.56984017882586, Tokens per sec: 169923.61755459203, Loss: 2.4708878993988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13343 , TFLOPS: 70.68769291422572, Tokens per sec: 175171.88993365117, Loss: 2.4862616062164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13344 , TFLOPS: 70.72476248237838, Tokens per sec: 175263.75240708352, Loss: 2.4995529651641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13345 , TFLOPS: 69.81750798307536, Tokens per sec: 173015.47581547705, Loss: 2.478081464767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13346 , TFLOPS: 67.89253264364034, Tokens per sec: 168245.17487081795, Loss: 2.50329327583313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13347 , TFLOPS: 71.48019750629315, Tokens per sec: 177135.80361438202, Loss: 2.505430221557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13348 , TFLOPS: 68.70339465782408, Tokens per sec: 170254.58026579, Loss: 2.508819103240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13349 , TFLOPS: 69.48794844587275, Tokens per sec: 172198.79097830897, Loss: 2.473222017288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13350 , TFLOPS: 69.52411602810207, Tokens per sec: 172288.41823126134, Loss: 2.479921817779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13351 , TFLOPS: 69.86799562351803, Tokens per sec: 173140.58975016675, Loss: 2.4982168674468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13352 , TFLOPS: 69.72050114303492, Tokens per sec: 172775.08217967127, Loss: 2.480318307876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13353 , TFLOPS: 69.95793886908062, Tokens per sec: 173363.47902073726, Loss: 2.4731783866882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13354 , TFLOPS: 69.64632337218201, Tokens per sec: 172591.2614921417, Loss: 2.4898290634155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13355 , TFLOPS: 68.75340874975136, Tokens per sec: 170378.52069509286, Loss: 2.47603178024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13356 , TFLOPS: 69.6902961866356, Tokens per sec: 172700.23097036313, Loss: 2.4973275661468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13357 , TFLOPS: 70.56584723128857, Tokens per sec: 174869.9429089229, Loss: 2.481334686279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13358 , TFLOPS: 69.89579974884033, Tokens per sec: 173209.49143559264, Loss: 2.478475332260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13359 , TFLOPS: 68.07056399289077, Tokens per sec: 168686.3561660326, Loss: 2.512047052383423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13360 , TFLOPS: 71.53792052919881, Tokens per sec: 177278.84762385816, Loss: 2.4847869873046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13361 , TFLOPS: 69.60311723785979, Tokens per sec: 172484.19193174265, Loss: 2.4947054386138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13362 , TFLOPS: 68.17144046387932, Tokens per sec: 168936.3391736029, Loss: 2.4863438606262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13363 , TFLOPS: 70.65711828181288, Tokens per sec: 175096.12262645835, Loss: 2.4795191287994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13364 , TFLOPS: 71.52710049439705, Tokens per sec: 177252.03438569416, Loss: 2.4879863262176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13365 , TFLOPS: 68.70424439080651, Tokens per sec: 170256.68599772625, Loss: 2.481196165084839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13366 , TFLOPS: 69.04795055668549, Tokens per sec: 171108.42773913417, Loss: 2.487700939178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13367 , TFLOPS: 71.55272600325436, Tokens per sec: 177315.53721952543, Loss: 2.4742462635040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13368 , TFLOPS: 68.70498160867281, Tokens per sec: 170258.51290481022, Loss: 2.4687554836273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13369 , TFLOPS: 69.06685622200912, Tokens per sec: 171155.27806043712, Loss: 2.49859619140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13370 , TFLOPS: 70.62576501417615, Tokens per sec: 175018.425775407, Loss: 2.4963536262512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13371 , TFLOPS: 69.87056154185497, Tokens per sec: 173146.94837846488, Loss: 2.475569009780884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13372 , TFLOPS: 68.68835865849718, Tokens per sec: 170217.31940310585, Loss: 2.4883248805999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13373 , TFLOPS: 69.99574333258802, Tokens per sec: 173457.16264581404, Loss: 2.485825300216675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13374 , TFLOPS: 70.62813675818882, Tokens per sec: 175024.30321833954, Loss: 2.480165719985962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13375 , TFLOPS: 69.76997079477864, Tokens per sec: 172897.6734262242, Loss: 2.5042641162872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13376 , TFLOPS: 68.79577343146667, Tokens per sec: 170483.50504323834, Loss: 2.477663278579712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13377 , TFLOPS: 71.57626332856164, Tokens per sec: 177373.8653044873, Loss: 2.4843740463256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13378 , TFLOPS: 69.91236746348976, Tokens per sec: 173250.54805757836, Loss: 2.499739170074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13379 , TFLOPS: 67.88787789132202, Tokens per sec: 168233.63988180985, Loss: 2.5147531032562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13380 , TFLOPS: 71.02474624995527, Tokens per sec: 176007.14522907903, Loss: 2.508894920349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13381 , TFLOPS: 70.61796243642198, Tokens per sec: 174999.09012820723, Loss: 2.497452735900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13382 , TFLOPS: 68.8178912066808, Tokens per sec: 170538.31532669303, Loss: 2.477243661880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13383 , TFLOPS: 69.18137883441747, Tokens per sec: 171439.07770969224, Loss: 2.499083995819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13384 , TFLOPS: 71.55037126076738, Tokens per sec: 177309.70190824638, Loss: 2.474794864654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13385 , TFLOPS: 69.84782431248568, Tokens per sec: 173090.60301937524, Loss: 2.529244899749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13386 , TFLOPS: 68.34105172228188, Tokens per sec: 169356.65455614697, Loss: 2.4892914295196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13387 , TFLOPS: 70.4847957246761, Tokens per sec: 174669.08834697667, Loss: 2.5008115768432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13388 , TFLOPS: 69.64971792052748, Tokens per sec: 172599.67355688216, Loss: 2.493849039077759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13389 , TFLOPS: 69.00790427865748, Tokens per sec: 171009.18865071904, Loss: 2.5117993354797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13390 , TFLOPS: 70.56579725122782, Tokens per sec: 174869.81905282658, Loss: 2.490877151489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13391 , TFLOPS: 68.9395205742601, Tokens per sec: 170839.72629813544, Loss: 2.4749507904052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13392 , TFLOPS: 70.71528914934186, Tokens per sec: 175240.27644424522, Loss: 2.4766993522644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13393 , TFLOPS: 69.07345464374463, Tokens per sec: 171171.62967637385, Loss: 2.4848885536193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13394 , TFLOPS: 69.68396342217858, Tokens per sec: 172684.537682427, Loss: 2.4865996837615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13395 , TFLOPS: 69.66490282336147, Tokens per sec: 172637.30341885993, Loss: 2.481182813644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13396 , TFLOPS: 69.83313906174322, Tokens per sec: 173054.21135032148, Loss: 2.4879980087280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13397 , TFLOPS: 70.86344332037301, Tokens per sec: 175607.41880058963, Loss: 2.47819447517395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13398 , TFLOPS: 69.62997026948631, Tokens per sec: 172550.7367022188, Loss: 2.506254196166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13399 , TFLOPS: 69.6921938707695, Tokens per sec: 172704.93364069407, Loss: 2.4912962913513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13400 , TFLOPS: 70.00490679011492, Tokens per sec: 173479.8707030039, Loss: 2.489039182662964 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/model.pt +[rank0]:[2024-08-27 06:19:32,030] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006428165012039244, 'preprocessing_with_comm': 0.0012447430053725839, 'state_converting': 0.9989480399963213, : 1.0079754760081414}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400, took 4.42s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011400 + eval ppl=10.528016090393066, eval loss=2.3540399074554443 +------------------------------------------------------------------ +iteration: 13401 , TFLOPS: 64.64106034339025, Tokens per sec: 160187.66833155585, Loss: 2.4914629459381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13402 , TFLOPS: 68.75806696632782, Tokens per sec: 170390.06426891423, Loss: 2.4911680221557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13403 , TFLOPS: 70.1521288183536, Tokens per sec: 173844.70310682454, Loss: 2.5046050548553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13404 , TFLOPS: 71.51543929437284, Tokens per sec: 177223.13664744693, Loss: 2.4838755130767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13405 , TFLOPS: 68.66542134445947, Tokens per sec: 170160.47821216672, Loss: 2.5022542476654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13406 , TFLOPS: 68.23309186512411, Tokens per sec: 169089.11813735, Loss: 2.4650919437408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13407 , TFLOPS: 71.50955096578673, Tokens per sec: 177208.54472055763, Loss: 2.4873385429382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13408 , TFLOPS: 70.7070386857899, Tokens per sec: 175219.8308866999, Loss: 2.500973701477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13409 , TFLOPS: 67.38459755321377, Tokens per sec: 166986.45576307157, Loss: 2.4696178436279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13410 , TFLOPS: 70.56690770271278, Tokens per sec: 174872.57087393268, Loss: 2.5151281356811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13411 , TFLOPS: 70.53793458844808, Tokens per sec: 174800.7723051327, Loss: 2.4930596351623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13412 , TFLOPS: 68.72936158112748, Tokens per sec: 170318.92916222857, Loss: 2.5109174251556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13413 , TFLOPS: 69.73975357839927, Tokens per sec: 172822.79183533494, Loss: 2.481626510620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13414 , TFLOPS: 70.71432252631256, Tokens per sec: 175237.8810458963, Loss: 2.4838051795959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13415 , TFLOPS: 69.65362234743202, Tokens per sec: 172609.34915685974, Loss: 2.4854581356048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13416 , TFLOPS: 68.69398532591663, Tokens per sec: 170231.26290479986, Loss: 2.477220058441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13417 , TFLOPS: 71.51920446121613, Tokens per sec: 177232.467145652, Loss: 2.500473976135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13418 , TFLOPS: 69.73708774563461, Tokens per sec: 172816.1856080784, Loss: 2.4849915504455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13419 , TFLOPS: 68.99345579219256, Tokens per sec: 170973.38370962683, Loss: 2.5114126205444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13420 , TFLOPS: 69.79388014775309, Tokens per sec: 172956.92343672752, Loss: 2.4800047874450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13421 , TFLOPS: 70.70139270555988, Tokens per sec: 175205.83952573384, Loss: 2.4688985347747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13422 , TFLOPS: 68.32279870968877, Tokens per sec: 169311.42158020593, Loss: 2.4918644428253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13423 , TFLOPS: 69.81769670752765, Tokens per sec: 173015.94349545956, Loss: 2.4863922595977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13424 , TFLOPS: 70.49454806584839, Tokens per sec: 174693.25572271066, Loss: 2.4680519104003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13425 , TFLOPS: 69.78756781558197, Tokens per sec: 172941.28078224673, Loss: 2.512083053588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13426 , TFLOPS: 68.14410969981421, Tokens per sec: 168868.61052951735, Loss: 2.474508762359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13427 , TFLOPS: 70.293515895209, Tokens per sec: 174195.07585834447, Loss: 2.4965429306030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13428 , TFLOPS: 70.66796550656709, Tokens per sec: 175123.00324431976, Loss: 2.4852521419525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13429 , TFLOPS: 68.03654544122101, Tokens per sec: 168602.05444754215, Loss: 2.5083043575286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13430 , TFLOPS: 70.65030544090567, Tokens per sec: 175079.2396561939, Loss: 2.4858014583587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13431 , TFLOPS: 69.45861581173477, Tokens per sec: 172126.1014796587, Loss: 2.4897494316101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13432 , TFLOPS: 69.65942402882804, Tokens per sec: 172623.72636246707, Loss: 2.500147819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13433 , TFLOPS: 69.53574184709719, Tokens per sec: 172317.22829142094, Loss: 2.5094590187072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13434 , TFLOPS: 71.51844006383328, Tokens per sec: 177230.57288473198, Loss: 2.490971326828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13435 , TFLOPS: 68.20152845202446, Tokens per sec: 169010.90052269207, Loss: 2.4823291301727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13436 , TFLOPS: 68.77231222411841, Tokens per sec: 170425.365586964, Loss: 2.493039131164551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13437 , TFLOPS: 71.49135728663015, Tokens per sec: 177163.45877940842, Loss: 2.495283603668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13438 , TFLOPS: 68.98399621200787, Tokens per sec: 170949.94182787038, Loss: 2.496634006500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13439 , TFLOPS: 69.68630560947628, Tokens per sec: 172690.34188056394, Loss: 2.5053353309631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13440 , TFLOPS: 70.17564239630418, Tokens per sec: 173902.97234322052, Loss: 2.488607883453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13441 , TFLOPS: 70.6680488814565, Tokens per sec: 175123.2098564802, Loss: 2.491152763366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13442 , TFLOPS: 68.88317503332148, Tokens per sec: 170700.0958407158, Loss: 2.487661600112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13443 , TFLOPS: 68.9538593117929, Tokens per sec: 170875.25926928408, Loss: 2.5008814334869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13444 , TFLOPS: 71.51278333860944, Tokens per sec: 177216.55489648756, Loss: 2.503316640853882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13445 , TFLOPS: 69.6932980157, Tokens per sec: 172707.66983346344, Loss: 2.497987985610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13446 , TFLOPS: 67.26046483363295, Tokens per sec: 166678.84120960243, Loss: 2.4974961280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13447 , TFLOPS: 71.54195883623535, Tokens per sec: 177288.85499355095, Loss: 2.497772693634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13448 , TFLOPS: 70.73092585362757, Tokens per sec: 175279.02591999038, Loss: 2.4839234352111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13449 , TFLOPS: 67.21389989969829, Tokens per sec: 166563.44817376143, Loss: 2.4780189990997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13450 , TFLOPS: 70.06221815942949, Tokens per sec: 173621.89458953388, Loss: 2.4956729412078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13451 , TFLOPS: 71.49355414069629, Tokens per sec: 177168.90282578775, Loss: 2.485538959503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13452 , TFLOPS: 69.13964312739637, Tokens per sec: 171335.65203012625, Loss: 2.4901342391967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13453 , TFLOPS: 68.58775913406343, Tokens per sec: 169968.0226996066, Loss: 2.4908576011657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13454 , TFLOPS: 71.52845577680868, Tokens per sec: 177255.39292480686, Loss: 2.498809576034546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13455 , TFLOPS: 69.86284620909886, Tokens per sec: 173127.8289339812, Loss: 2.4891180992126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13456 , TFLOPS: 69.03734577611911, Tokens per sec: 171082.14792467767, Loss: 2.5003061294555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13457 , TFLOPS: 69.98652711596479, Tokens per sec: 173434.3238457716, Loss: 2.495788335800171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13458 , TFLOPS: 70.60886121543192, Tokens per sec: 174976.53630001162, Loss: 2.4878029823303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13459 , TFLOPS: 68.94543582034757, Tokens per sec: 170854.38492955835, Loss: 2.509392023086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13460 , TFLOPS: 69.82517345744603, Tokens per sec: 173034.4717053893, Loss: 2.4641220569610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13461 , TFLOPS: 69.69373868336284, Tokens per sec: 172708.76185647552, Loss: 2.5018324851989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13462 , TFLOPS: 69.90791994529704, Tokens per sec: 173239.52661756222, Loss: 2.4944894313812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13463 , TFLOPS: 68.83035430767099, Tokens per sec: 170569.20026387065, Loss: 2.4996230602264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13464 , TFLOPS: 70.44708128146164, Tokens per sec: 174575.62780210233, Loss: 2.4912776947021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13465 , TFLOPS: 69.70629793263234, Tokens per sec: 172739.88505968088, Loss: 2.506065845489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13466 , TFLOPS: 68.00705682818433, Tokens per sec: 168528.97841599837, Loss: 2.4977805614471436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13467 , TFLOPS: 71.55785721974526, Tokens per sec: 177328.25293924002, Loss: 2.4880588054656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13468 , TFLOPS: 70.54393090300385, Tokens per sec: 174815.6318331505, Loss: 2.4872472286224365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13469 , TFLOPS: 68.90247770950472, Tokens per sec: 170747.92999866322, Loss: 2.50134015083313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13470 , TFLOPS: 69.84348001463117, Tokens per sec: 173079.83737072782, Loss: 2.4897170066833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13471 , TFLOPS: 71.51474958165399, Tokens per sec: 177221.4274633515, Loss: 2.4869089126586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13472 , TFLOPS: 68.7280923429636, Tokens per sec: 170315.78385024038, Loss: 2.4970016479492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13473 , TFLOPS: 68.95007613064854, Tokens per sec: 170865.88412965654, Loss: 2.4929168224334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13474 , TFLOPS: 71.5489684448925, Tokens per sec: 177306.2255759755, Loss: 2.47662615776062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13475 , TFLOPS: 70.23604024514776, Tokens per sec: 174052.6448660272, Loss: 2.4873974323272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13476 , TFLOPS: 67.92604508126425, Tokens per sec: 168328.22238294975, Loss: 2.483842611312866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13477 , TFLOPS: 70.71879533342495, Tokens per sec: 175248.9651546412, Loss: 2.4912750720977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13478 , TFLOPS: 70.69073170206347, Tokens per sec: 175179.42038466327, Loss: 2.4851794242858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13479 , TFLOPS: 68.81479626602136, Tokens per sec: 170530.6457228031, Loss: 2.4818663597106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13480 , TFLOPS: 69.04822776172128, Tokens per sec: 171109.11468374953, Loss: 2.4928510189056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13481 , TFLOPS: 71.51986412608991, Tokens per sec: 177234.1018678773, Loss: 2.4994771480560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13482 , TFLOPS: 69.09634856816945, Tokens per sec: 171228.36334307256, Loss: 2.4848039150238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13483 , TFLOPS: 67.73934461804403, Tokens per sec: 167865.5580683362, Loss: 2.484809637069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13484 , TFLOPS: 71.49184567067007, Tokens per sec: 177164.66904885927, Loss: 2.4558849334716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13485 , TFLOPS: 69.77043541893939, Tokens per sec: 172898.8248160771, Loss: 2.5002079010009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13486 , TFLOPS: 67.88375012946607, Tokens per sec: 168223.41083322032, Loss: 2.489149808883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13487 , TFLOPS: 70.88211504636962, Tokens per sec: 175653.68939446966, Loss: 2.4774527549743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13488 , TFLOPS: 70.6135974502001, Tokens per sec: 174988.27321150654, Loss: 2.4778106212615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13489 , TFLOPS: 68.83413771443075, Tokens per sec: 170578.575962598, Loss: 2.4756906032562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13490 , TFLOPS: 68.99099042602161, Tokens per sec: 170967.2742606727, Loss: 2.4769437313079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13491 , TFLOPS: 71.51660972473385, Tokens per sec: 177226.03710281508, Loss: 2.495239734649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13492 , TFLOPS: 69.81618601083403, Tokens per sec: 173012.19982263577, Loss: 2.462752342224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13493 , TFLOPS: 68.64785523358728, Tokens per sec: 170116.94745435857, Loss: 2.5037002563476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13494 , TFLOPS: 70.57157365719938, Tokens per sec: 174884.1336231478, Loss: 2.5109410285949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13495 , TFLOPS: 70.61615116708089, Tokens per sec: 174994.60160325223, Loss: 2.496159076690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13496 , TFLOPS: 68.1477337503895, Tokens per sec: 168877.59132606507, Loss: 2.477461576461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13497 , TFLOPS: 70.77504594137689, Tokens per sec: 175388.3603576621, Loss: 2.4771218299865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13498 , TFLOPS: 69.70791206824168, Tokens per sec: 172743.88506553255, Loss: 2.4922537803649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13499 , TFLOPS: 69.64074535205118, Tokens per sec: 172577.4385437879, Loss: 2.50351619720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13500 , TFLOPS: 68.78213491780238, Tokens per sec: 170449.70730397193, Loss: 2.522230386734009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13501 , TFLOPS: 70.58401539870216, Tokens per sec: 174914.96562916265, Loss: 2.514585494995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13502 , TFLOPS: 69.00214150750156, Tokens per sec: 170994.90786897304, Loss: 2.4923224449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13503 , TFLOPS: 68.99001320010555, Tokens per sec: 170964.8525872024, Loss: 2.479752779006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13504 , TFLOPS: 71.47162952385918, Tokens per sec: 177114.57121006947, Loss: 2.4924564361572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13505 , TFLOPS: 69.59767366432192, Tokens per sec: 172470.70215685669, Loss: 2.494229793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13506 , TFLOPS: 68.8766104316884, Tokens per sec: 170683.82803471852, Loss: 2.500701904296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13507 , TFLOPS: 70.03532133220804, Tokens per sec: 173555.24128874904, Loss: 2.4852795600891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13508 , TFLOPS: 70.58116079407385, Tokens per sec: 174907.891604433, Loss: 2.491914987564087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13509 , TFLOPS: 67.8885590072061, Tokens per sec: 168235.32776200172, Loss: 2.5168681144714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13510 , TFLOPS: 69.92196227383704, Tokens per sec: 173274.32505457452, Loss: 2.497903823852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13511 , TFLOPS: 71.56019230768324, Tokens per sec: 177334.0395443815, Loss: 2.511157751083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13512 , TFLOPS: 68.79115651196378, Tokens per sec: 170472.0638081145, Loss: 2.4994113445281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13513 , TFLOPS: 68.77397854746536, Tokens per sec: 170429.49491977843, Loss: 2.482940912246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13514 , TFLOPS: 71.53488613220502, Tokens per sec: 177271.3280538411, Loss: 2.500389337539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13515 , TFLOPS: 70.59552556229974, Tokens per sec: 174943.48908250662, Loss: 2.479069471359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13516 , TFLOPS: 67.10874825442788, Tokens per sec: 166302.87081337135, Loss: 2.483593225479126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13517 , TFLOPS: 70.65834156342004, Tokens per sec: 175099.15405303647, Loss: 2.516972780227661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13518 , TFLOPS: 71.52514049070865, Tokens per sec: 177247.1772806421, Loss: 2.4797768592834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13519 , TFLOPS: 69.30797433547032, Tokens per sec: 171752.79530694688, Loss: 2.4792096614837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13520 , TFLOPS: 68.59956476416161, Tokens per sec: 169997.2783514874, Loss: 2.4731228351593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13521 , TFLOPS: 71.54246462700029, Tokens per sec: 177290.1083987839, Loss: 2.5274505615234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13522 , TFLOPS: 69.72907471030246, Tokens per sec: 172796.3284238166, Loss: 2.5061452388763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13523 , TFLOPS: 68.08831677111338, Tokens per sec: 168730.34950609822, Loss: 2.4652767181396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13524 , TFLOPS: 70.91285477574984, Tokens per sec: 175729.86583013457, Loss: 2.490387439727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13525 , TFLOPS: 70.72285914615833, Tokens per sec: 175259.03573026042, Loss: 2.4935660362243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13526 , TFLOPS: 69.030353132149, Tokens per sec: 171064.81938261644, Loss: 2.5022807121276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13527 , TFLOPS: 68.86026285306768, Tokens per sec: 170643.3169340619, Loss: 2.523977518081665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13528 , TFLOPS: 70.63087391247495, Tokens per sec: 175031.08618818305, Loss: 2.4847028255462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13529 , TFLOPS: 69.75309996123744, Tokens per sec: 172855.86564223337, Loss: 2.5043063163757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13530 , TFLOPS: 69.17696115659483, Tokens per sec: 171428.1302174003, Loss: 2.497274875640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13531 , TFLOPS: 70.4309725985789, Tokens per sec: 174535.70871140104, Loss: 2.4803032875061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13532 , TFLOPS: 69.74135394012302, Tokens per sec: 172826.7577079808, Loss: 2.488856792449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13533 , TFLOPS: 68.61892081825349, Tokens per sec: 170045.24478577243, Loss: 2.4852263927459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13534 , TFLOPS: 71.5309388309639, Tokens per sec: 177261.5462065345, Loss: 2.500519275665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13535 , TFLOPS: 69.58589716448452, Tokens per sec: 172441.5186929711, Loss: 2.4753494262695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13536 , TFLOPS: 68.70869253631625, Tokens per sec: 170267.7089923031, Loss: 2.505247116088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13537 , TFLOPS: 70.19273302997476, Tokens per sec: 173945.32481614494, Loss: 2.4839987754821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13538 , TFLOPS: 70.37839181819668, Tokens per sec: 174405.40774536273, Loss: 2.4830474853515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13539 , TFLOPS: 69.90769451120052, Tokens per sec: 173238.9679670374, Loss: 2.505401849746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13540 , TFLOPS: 68.25463438717709, Tokens per sec: 169142.502880102, Loss: 2.512559652328491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13541 , TFLOPS: 71.47548825214474, Tokens per sec: 177124.1335638351, Loss: 2.5074846744537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13542 , TFLOPS: 69.67196453108645, Tokens per sec: 172654.80310851376, Loss: 2.499934196472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13543 , TFLOPS: 68.24106249918187, Tokens per sec: 169108.87024658293, Loss: 2.496394157409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13544 , TFLOPS: 70.6855317721781, Tokens per sec: 175166.53438558773, Loss: 2.506089687347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13545 , TFLOPS: 70.60543892496266, Tokens per sec: 174968.05548723237, Loss: 2.5068538188934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13546 , TFLOPS: 68.85682777272791, Tokens per sec: 170634.80442657458, Loss: 2.4899308681488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13547 , TFLOPS: 69.31762496483428, Tokens per sec: 171776.71062961535, Loss: 2.48081111907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13548 , TFLOPS: 70.61541423973405, Tokens per sec: 174992.77541610753, Loss: 2.5102529525756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13549 , TFLOPS: 69.84118660139367, Tokens per sec: 173074.15404008498, Loss: 2.487912893295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13550 , TFLOPS: 67.87789597332483, Tokens per sec: 168208.90358941373, Loss: 2.4987893104553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13551 , TFLOPS: 71.51991785318309, Tokens per sec: 177234.2350095327, Loss: 2.4858615398406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13552 , TFLOPS: 69.8640425577854, Tokens per sec: 173130.79361781513, Loss: 2.4962143898010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13553 , TFLOPS: 69.327880301835, Tokens per sec: 171802.12448442174, Loss: 2.479825973510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13554 , TFLOPS: 70.89002299932456, Tokens per sec: 175673.28617302494, Loss: 2.472377061843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13555 , TFLOPS: 70.6413016735084, Tokens per sec: 175056.9273287366, Loss: 2.4875171184539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13556 , TFLOPS: 68.82120716894893, Tokens per sec: 170546.53264647687, Loss: 2.4849958419799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13557 , TFLOPS: 68.81623070670042, Tokens per sec: 170534.20042482248, Loss: 2.5006799697875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13558 , TFLOPS: 71.5174209376325, Tokens per sec: 177228.04737774038, Loss: 2.5106582641601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13559 , TFLOPS: 69.88201706417185, Tokens per sec: 173175.33642469608, Loss: 2.4911749362945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13560 , TFLOPS: 67.28636272645124, Tokens per sec: 166743.01904089458, Loss: 2.501156806945801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13561 , TFLOPS: 71.52011752616731, Tokens per sec: 177234.72982118343, Loss: 2.474393606185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13562 , TFLOPS: 70.52636340490263, Tokens per sec: 174772.09763763598, Loss: 2.494955539703369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13563 , TFLOPS: 68.02756212346526, Tokens per sec: 168579.79279654927, Loss: 2.4994122982025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13564 , TFLOPS: 69.53231545203245, Tokens per sec: 172308.73730700192, Loss: 2.4759724140167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13565 , TFLOPS: 70.64178905026104, Tokens per sec: 175058.13510201866, Loss: 2.4753899574279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13566 , TFLOPS: 69.60229822404224, Tokens per sec: 172482.16232528092, Loss: 2.509053945541382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13567 , TFLOPS: 68.94473272186444, Tokens per sec: 170852.64257406446, Loss: 2.481773853302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13568 , TFLOPS: 70.53260690903237, Tokens per sec: 174787.56972864823, Loss: 2.480282783508301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13569 , TFLOPS: 69.98598880790877, Tokens per sec: 173432.98985910928, Loss: 2.501093864440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13570 , TFLOPS: 68.1954037983878, Tokens per sec: 168995.7229562941, Loss: 2.481935739517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13571 , TFLOPS: 71.5646909451485, Tokens per sec: 177345.18766358626, Loss: 2.4858062267303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13572 , TFLOPS: 69.67459499083125, Tokens per sec: 172661.32167753045, Loss: 2.497392416000366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13573 , TFLOPS: 69.21281584726637, Tokens per sec: 171516.98209060464, Loss: 2.502379894256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13574 , TFLOPS: 70.62943001496363, Tokens per sec: 175027.50805109154, Loss: 2.491935968399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13575 , TFLOPS: 69.56773416388913, Tokens per sec: 172396.508776676, Loss: 2.4929232597351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13576 , TFLOPS: 68.62350174060042, Tokens per sec: 170056.59681597838, Loss: 2.4700722694396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13577 , TFLOPS: 69.76329599770816, Tokens per sec: 172881.13254379394, Loss: 2.478055477142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13578 , TFLOPS: 71.46305195230596, Tokens per sec: 177093.31504286305, Loss: 2.504979372024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13579 , TFLOPS: 69.43408887637318, Tokens per sec: 172065.32103196767, Loss: 2.472886085510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13580 , TFLOPS: 68.1653635149069, Tokens per sec: 168921.27982461805, Loss: 2.501944065093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13581 , TFLOPS: 71.54422098591552, Tokens per sec: 177294.46084965474, Loss: 2.4820635318756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13582 , TFLOPS: 70.68930485582884, Tokens per sec: 175175.8845025137, Loss: 2.4947659969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13583 , TFLOPS: 67.05245386505919, Tokens per sec: 166163.3671151768, Loss: 2.480278491973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13584 , TFLOPS: 70.59874207714454, Tokens per sec: 174951.4599606204, Loss: 2.486166477203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13585 , TFLOPS: 71.56200217622786, Tokens per sec: 177338.52459800857, Loss: 2.516810894012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13586 , TFLOPS: 68.95995998487881, Tokens per sec: 170890.37740923557, Loss: 2.496598243713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13587 , TFLOPS: 69.03415082602899, Tokens per sec: 171074.23048640156, Loss: 2.485456705093384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13588 , TFLOPS: 71.57082051333808, Tokens per sec: 177360.3774087877, Loss: 2.4757230281829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13589 , TFLOPS: 69.82999096260532, Tokens per sec: 173046.41001386603, Loss: 2.5061442852020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13590 , TFLOPS: 67.85372931645591, Tokens per sec: 168149.01595151558, Loss: 2.489886999130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13591 , TFLOPS: 70.88169445800501, Tokens per sec: 175652.64713017055, Loss: 2.4961283206939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13592 , TFLOPS: 70.55933840331174, Tokens per sec: 174853.8133161897, Loss: 2.482481002807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13593 , TFLOPS: 68.78570932767872, Tokens per sec: 170458.56508539926, Loss: 2.456202983856201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13594 , TFLOPS: 69.66357179247221, Tokens per sec: 172634.0049776912, Loss: 2.4813225269317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13595 , TFLOPS: 70.72507553597089, Tokens per sec: 175264.52818837142, Loss: 2.5009448528289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13596 , TFLOPS: 69.699404374569, Tokens per sec: 172722.8020634118, Loss: 2.4936861991882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13597 , TFLOPS: 67.97925811112371, Tokens per sec: 168460.09013284068, Loss: 2.5079457759857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13598 , TFLOPS: 71.50526748151994, Tokens per sec: 177197.92977467977, Loss: 2.469003677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13599 , TFLOPS: 69.6771673190828, Tokens per sec: 172667.69619030313, Loss: 2.4770617485046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13600 , TFLOPS: 68.1950366603958, Tokens per sec: 168994.8131479064, Loss: 2.484280824661255 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/model.pt +[rank0]:[2024-08-27 06:44:57,120] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006429215005482547, 'preprocessing_with_comm': 0.00125640700571239, 'state_converting': 0.9806077609973727, : 0.9896481390023837}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600, took 4.37s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011600 + eval ppl=9.887173652648926, eval loss=2.291238307952881 +------------------------------------------------------------------ +iteration: 13601 , TFLOPS: 64.7202038207744, Tokens per sec: 160383.79458688755, Loss: 2.4753129482269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13602 , TFLOPS: 70.66661632917317, Tokens per sec: 175119.65983411344, Loss: 2.477262496948242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13603 , TFLOPS: 69.68806090626326, Tokens per sec: 172694.691699364, Loss: 2.509063243865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13604 , TFLOPS: 69.88146104787131, Tokens per sec: 173173.9585550531, Loss: 2.4836769104003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13605 , TFLOPS: 70.43864409119128, Tokens per sec: 174554.71951517917, Loss: 2.4857373237609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13606 , TFLOPS: 68.93863476047498, Tokens per sec: 170837.5311539955, Loss: 2.4824671745300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13607 , TFLOPS: 69.73792650618017, Tokens per sec: 172818.26414910742, Loss: 2.489689350128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13608 , TFLOPS: 70.43137475639934, Tokens per sec: 174536.70530278137, Loss: 2.4892995357513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13609 , TFLOPS: 69.70812673458408, Tokens per sec: 172744.41703237646, Loss: 2.5053436756134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13610 , TFLOPS: 68.99651465473363, Tokens per sec: 170980.96390796514, Loss: 2.478179454803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13611 , TFLOPS: 71.50349576813747, Tokens per sec: 177193.53927374768, Loss: 2.4766807556152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13612 , TFLOPS: 70.41614385225347, Tokens per sec: 174498.96144448535, Loss: 2.484917640686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13613 , TFLOPS: 70.0682720246294, Tokens per sec: 173636.89673438756, Loss: 2.4921624660491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13614 , TFLOPS: 69.02409520618063, Tokens per sec: 171049.31155269936, Loss: 2.4900026321411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13615 , TFLOPS: 71.49784018041056, Tokens per sec: 177179.524104346, Loss: 2.4938459396362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13616 , TFLOPS: 68.11629269046621, Tokens per sec: 168799.67691605646, Loss: 2.4824612140655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13617 , TFLOPS: 68.93822708134728, Tokens per sec: 170836.52088020745, Loss: 2.491224527359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13618 , TFLOPS: 71.51723809902086, Tokens per sec: 177227.5942835194, Loss: 2.4957115650177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13619 , TFLOPS: 69.77207152331368, Tokens per sec: 172902.8792629484, Loss: 2.5130953788757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13620 , TFLOPS: 67.15764093697683, Tokens per sec: 166424.0322667013, Loss: 2.4798781871795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13621 , TFLOPS: 71.4956974003536, Tokens per sec: 177174.21405931812, Loss: 2.4630470275878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13622 , TFLOPS: 70.71752799936095, Tokens per sec: 175245.82456122225, Loss: 2.4812638759613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13623 , TFLOPS: 67.77581601301424, Tokens per sec: 167955.9382027282, Loss: 2.482893228530884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13624 , TFLOPS: 69.78503796662517, Tokens per sec: 172935.01153784592, Loss: 2.487668752670288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13625 , TFLOPS: 71.52366203136633, Tokens per sec: 177243.51349552607, Loss: 2.485356330871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13626 , TFLOPS: 69.79804221876397, Tokens per sec: 172967.23750718246, Loss: 2.510669469833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13627 , TFLOPS: 67.39473339107528, Tokens per sec: 167011.57348584716, Loss: 2.494685173034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13628 , TFLOPS: 71.55532244246398, Tokens per sec: 177321.97148190823, Loss: 2.504000186920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13629 , TFLOPS: 70.66610601987846, Tokens per sec: 175118.39523146593, Loss: 2.4889273643493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13630 , TFLOPS: 68.18777382165214, Tokens per sec: 168976.81503343183, Loss: 2.4814627170562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13631 , TFLOPS: 70.3814887356333, Tokens per sec: 174413.0822479243, Loss: 2.5054714679718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13632 , TFLOPS: 71.5166363684199, Tokens per sec: 177226.10312880413, Loss: 2.486265182495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13633 , TFLOPS: 69.19362655851255, Tokens per sec: 171469.42891919802, Loss: 2.5074000358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13634 , TFLOPS: 68.87997316081925, Tokens per sec: 170692.16124793992, Loss: 2.488100528717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13635 , TFLOPS: 70.58711834616278, Tokens per sec: 174922.65507478794, Loss: 2.492403984069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13636 , TFLOPS: 68.98238871504601, Tokens per sec: 170945.9582733185, Loss: 2.4997286796569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13637 , TFLOPS: 68.72558747073496, Tokens per sec: 170309.57650092331, Loss: 2.496112108230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13638 , TFLOPS: 70.45755074275378, Tokens per sec: 174601.57228049063, Loss: 2.5066022872924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13639 , TFLOPS: 69.6874894342353, Tokens per sec: 172693.27552872588, Loss: 2.503047466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13640 , TFLOPS: 68.96472899709815, Tokens per sec: 170902.19554686514, Loss: 2.5091564655303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13641 , TFLOPS: 70.98227058205212, Tokens per sec: 175901.88584493398, Loss: 2.4958534240722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13642 , TFLOPS: 69.64948232890583, Tokens per sec: 172599.08973489117, Loss: 2.4947726726531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13643 , TFLOPS: 69.88760000126385, Tokens per sec: 173189.17155782133, Loss: 2.4904873371124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13644 , TFLOPS: 69.11103482628202, Tokens per sec: 171264.75750850033, Loss: 2.4968011379241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13645 , TFLOPS: 70.52493599937002, Tokens per sec: 174768.56036948448, Loss: 2.5008630752563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13646 , TFLOPS: 69.6711228548353, Tokens per sec: 172652.71734204513, Loss: 2.4928200244903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13647 , TFLOPS: 67.90982985087754, Tokens per sec: 168288.0392557964, Loss: 2.4920237064361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13648 , TFLOPS: 71.490398320025, Tokens per sec: 177161.08235452193, Loss: 2.4927399158477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13649 , TFLOPS: 69.6352661341223, Tokens per sec: 172563.86043817445, Loss: 2.5022389888763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13650 , TFLOPS: 68.74417051845208, Tokens per sec: 170355.62734025804, Loss: 2.494533061981201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13651 , TFLOPS: 71.00954355800896, Tokens per sec: 175969.47128372133, Loss: 2.485541343688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13652 , TFLOPS: 71.53181646538798, Tokens per sec: 177263.72108131676, Loss: 2.50390887260437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13653 , TFLOPS: 67.93269699449336, Tokens per sec: 168344.70655670023, Loss: 2.481882095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13654 , TFLOPS: 68.83426063482915, Tokens per sec: 170578.880572886, Loss: 2.48879337310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13655 , TFLOPS: 71.55539555514243, Tokens per sec: 177322.15266317944, Loss: 2.502321720123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13656 , TFLOPS: 69.10542380933609, Tokens per sec: 171250.8527904038, Loss: 2.4941961765289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13657 , TFLOPS: 67.77179176029392, Tokens per sec: 167945.96566117922, Loss: 2.494197368621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13658 , TFLOPS: 71.52506240550035, Tokens per sec: 177246.9837768941, Loss: 2.457279920578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13659 , TFLOPS: 69.81789253166882, Tokens per sec: 173016.42876925317, Loss: 2.473604202270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13660 , TFLOPS: 68.85584966033622, Tokens per sec: 170632.38055632001, Loss: 2.482585906982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13661 , TFLOPS: 70.00733468897347, Tokens per sec: 173485.8873038305, Loss: 2.4868032932281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13662 , TFLOPS: 70.58113094732698, Tokens per sec: 174907.8176409064, Loss: 2.523519992828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13663 , TFLOPS: 69.8284017357073, Tokens per sec: 173042.47173454496, Loss: 2.4721431732177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13664 , TFLOPS: 67.78060700700019, Tokens per sec: 167967.81081359694, Loss: 2.507991313934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13665 , TFLOPS: 71.5352497679894, Tokens per sec: 177272.2291833726, Loss: 2.4657225608825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13666 , TFLOPS: 69.79398954458637, Tokens per sec: 172957.19453413147, Loss: 2.4827568531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13667 , TFLOPS: 67.98475071024295, Tokens per sec: 168473.70139851785, Loss: 2.4883644580841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13668 , TFLOPS: 70.4826583810221, Tokens per sec: 174663.7917739542, Loss: 2.4942986965179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13669 , TFLOPS: 70.57245723925715, Tokens per sec: 174886.3232368215, Loss: 2.4782745838165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13670 , TFLOPS: 68.72688535248207, Tokens per sec: 170312.79279486614, Loss: 2.4859235286712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13671 , TFLOPS: 70.67728449531857, Tokens per sec: 175146.09672501724, Loss: 2.496971607208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13672 , TFLOPS: 70.51348192662063, Tokens per sec: 174740.17591544124, Loss: 2.4968433380126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13673 , TFLOPS: 68.90596705643571, Tokens per sec: 170756.5769847415, Loss: 2.4920084476470947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13674 , TFLOPS: 68.82689019633963, Tokens per sec: 170560.61581438815, Loss: 2.501298666000366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13675 , TFLOPS: 70.41517873141987, Tokens per sec: 174496.56976874283, Loss: 2.5010688304901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13676 , TFLOPS: 69.73970647948309, Tokens per sec: 172822.67511903227, Loss: 2.4903030395507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13677 , TFLOPS: 68.35066628848124, Tokens per sec: 169380.48051032086, Loss: 2.4774603843688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13678 , TFLOPS: 71.53010828257052, Tokens per sec: 177259.48801612324, Loss: 2.4731156826019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13679 , TFLOPS: 69.71671236033974, Tokens per sec: 172765.69315878372, Loss: 2.5107195377349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13680 , TFLOPS: 69.04077905764676, Tokens per sec: 171090.65597451036, Loss: 2.485527992248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13681 , TFLOPS: 69.70028121789568, Tokens per sec: 172724.9749777676, Loss: 2.4926393032073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13682 , TFLOPS: 70.5622198512265, Tokens per sec: 174860.95386154018, Loss: 2.4942924976348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13683 , TFLOPS: 69.25906226369446, Tokens per sec: 171631.58580497678, Loss: 2.5140066146850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13684 , TFLOPS: 68.79936244617603, Tokens per sec: 170492.39901705072, Loss: 2.477428436279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13685 , TFLOPS: 71.53533651658307, Tokens per sec: 177272.44415594387, Loss: 2.4632885456085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13686 , TFLOPS: 68.70802086536564, Tokens per sec: 170266.0445176968, Loss: 2.489535331726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13687 , TFLOPS: 67.95113370240952, Tokens per sec: 168390.39474988752, Loss: 2.4862003326416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13688 , TFLOPS: 71.5082830922195, Tokens per sec: 177205.4027901902, Loss: 2.463644027709961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13689 , TFLOPS: 70.5342591646946, Tokens per sec: 174791.66420019025, Loss: 2.508657455444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13690 , TFLOPS: 69.64564937820612, Tokens per sec: 172589.59126082287, Loss: 2.481672525405884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13691 , TFLOPS: 70.04952286931214, Tokens per sec: 173590.43426212145, Loss: 2.484943151473999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13692 , TFLOPS: 71.53487410837818, Tokens per sec: 177271.29825747368, Loss: 2.508585214614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13693 , TFLOPS: 69.80013670554358, Tokens per sec: 172972.42787614913, Loss: 2.51755428314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13694 , TFLOPS: 67.22692172612454, Tokens per sec: 166595.71769411827, Loss: 2.492992401123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13695 , TFLOPS: 71.51778114023759, Tokens per sec: 177228.93999947538, Loss: 2.480379343032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13696 , TFLOPS: 70.06474520575654, Tokens per sec: 173628.15688870943, Loss: 2.488856792449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13697 , TFLOPS: 66.97163895745467, Tokens per sec: 165963.09887163583, Loss: 2.4738917350769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13698 , TFLOPS: 71.50172649879896, Tokens per sec: 177189.15482942574, Loss: 2.4973912239074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13699 , TFLOPS: 71.55509989441869, Tokens per sec: 177321.41998333632, Loss: 2.5102667808532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13700 , TFLOPS: 69.34895006393482, Tokens per sec: 171854.337675931, Loss: 2.489973545074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13701 , TFLOPS: 68.18119035497449, Tokens per sec: 168960.5004777759, Loss: 2.4955930709838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13702 , TFLOPS: 71.52704900798096, Tokens per sec: 177251.90679668338, Loss: 2.5196869373321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13703 , TFLOPS: 68.85512776475692, Tokens per sec: 170630.59161955112, Loss: 2.507368326187134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13704 , TFLOPS: 68.87760456422747, Tokens per sec: 170686.291604663, Loss: 2.4942004680633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13705 , TFLOPS: 70.60868821902217, Tokens per sec: 174976.10759585112, Loss: 2.502009868621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13706 , TFLOPS: 69.79585645254264, Tokens per sec: 172961.82093770397, Loss: 2.468090534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13707 , TFLOPS: 68.88119733672623, Tokens per sec: 170695.19489069152, Loss: 2.5070316791534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13708 , TFLOPS: 70.87138798683226, Tokens per sec: 175627.10655925295, Loss: 2.496472120285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13709 , TFLOPS: 69.50880778146448, Tokens per sec: 172250.48270975082, Loss: 2.470092296600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13710 , TFLOPS: 69.73303874824, Tokens per sec: 172806.15174650066, Loss: 2.4926977157592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13711 , TFLOPS: 69.69629321299315, Tokens per sec: 172715.09226230413, Loss: 2.474609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13712 , TFLOPS: 70.53774607598893, Tokens per sec: 174800.3051504926, Loss: 2.49143648147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13713 , TFLOPS: 69.81645598606241, Tokens per sec: 173012.86885099192, Loss: 2.4905481338500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13714 , TFLOPS: 67.94258135839007, Tokens per sec: 168369.20109928885, Loss: 2.496746063232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13715 , TFLOPS: 71.51591342030495, Tokens per sec: 177224.311583736, Loss: 2.494584321975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13716 , TFLOPS: 70.4720200207415, Tokens per sec: 174637.42874526675, Loss: 2.5020670890808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13717 , TFLOPS: 68.09136019479615, Tokens per sec: 168737.89144524492, Loss: 2.492865562438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13718 , TFLOPS: 70.90846313610653, Tokens per sec: 175718.98286331943, Loss: 2.4676411151885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13719 , TFLOPS: 70.43534812291998, Tokens per sec: 174546.551742722, Loss: 2.4733617305755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13720 , TFLOPS: 69.00145091232724, Tokens per sec: 170993.19649805574, Loss: 2.4823215007781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13721 , TFLOPS: 68.86834145756018, Tokens per sec: 170663.33660592607, Loss: 2.492156982421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13722 , TFLOPS: 71.5397852131621, Tokens per sec: 177283.4685161311, Loss: 2.5077338218688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13723 , TFLOPS: 69.4328185262537, Tokens per sec: 172062.172964431, Loss: 2.463771343231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13724 , TFLOPS: 68.48223772567427, Tokens per sec: 169706.5290254755, Loss: 2.475977659225464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13725 , TFLOPS: 71.57458566984796, Tokens per sec: 177369.70788138543, Loss: 2.4838199615478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13726 , TFLOPS: 70.7012607413201, Tokens per sec: 175205.51250381058, Loss: 2.4694085121154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13727 , TFLOPS: 68.11576090279236, Tokens per sec: 168798.3590876194, Loss: 2.493839979171753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13728 , TFLOPS: 69.9101327496834, Tokens per sec: 173245.01019059215, Loss: 2.500171661376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13729 , TFLOPS: 71.52363023244307, Tokens per sec: 177243.43469429127, Loss: 2.482774257659912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13730 , TFLOPS: 69.23009487044258, Tokens per sec: 171559.80141347685, Loss: 2.5093536376953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13731 , TFLOPS: 68.79093987652607, Tokens per sec: 170471.5269616355, Loss: 2.506415843963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13732 , TFLOPS: 71.52392649297961, Tokens per sec: 177244.16886053665, Loss: 2.497744083404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13733 , TFLOPS: 69.8825948764903, Tokens per sec: 173176.7683072725, Loss: 2.4790077209472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13734 , TFLOPS: 67.78191672794003, Tokens per sec: 167971.05644636354, Loss: 2.493009090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13735 , TFLOPS: 71.49023448231189, Tokens per sec: 177160.67634662063, Loss: 2.473541498184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13736 , TFLOPS: 70.59448451580617, Tokens per sec: 174940.90925461482, Loss: 2.486605405807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13737 , TFLOPS: 68.0821268884361, Tokens per sec: 168715.01029495202, Loss: 2.4778084754943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13738 , TFLOPS: 69.74958221124957, Tokens per sec: 172847.1482702513, Loss: 2.507858991622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13739 , TFLOPS: 71.4696090473407, Tokens per sec: 177109.5642466828, Loss: 2.463758945465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13740 , TFLOPS: 69.08306075646577, Tokens per sec: 171195.43468189752, Loss: 2.4966347217559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13741 , TFLOPS: 68.20533271965533, Tokens per sec: 169020.32791695604, Loss: 2.5254099369049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13742 , TFLOPS: 70.47507113862399, Tokens per sec: 174644.98975148794, Loss: 2.4724555015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13743 , TFLOPS: 69.75973231781586, Tokens per sec: 172872.3013524491, Loss: 2.492062568664551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13744 , TFLOPS: 67.5838167716869, Tokens per sec: 167480.14293225663, Loss: 2.5076346397399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13745 , TFLOPS: 71.55268694500178, Tokens per sec: 177315.44042887288, Loss: 2.4884889125823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13746 , TFLOPS: 69.52379215238777, Tokens per sec: 172287.61563156344, Loss: 2.516176462173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13747 , TFLOPS: 69.07328626858204, Tokens per sec: 171171.21242417287, Loss: 2.471118927001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13748 , TFLOPS: 69.65766027618348, Tokens per sec: 172619.35558912042, Loss: 2.461395025253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13749 , TFLOPS: 70.43754705330052, Tokens per sec: 174552.00093443715, Loss: 2.4627914428710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13750 , TFLOPS: 68.83082804972051, Tokens per sec: 170570.3742488563, Loss: 2.482760429382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13751 , TFLOPS: 69.67648120136181, Tokens per sec: 172665.99591500842, Loss: 2.498352527618408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13752 , TFLOPS: 71.55236830123134, Tokens per sec: 177314.65079450826, Loss: 2.4719185829162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13753 , TFLOPS: 68.90574853137187, Tokens per sec: 170756.03545556078, Loss: 2.497270107269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13754 , TFLOPS: 68.10620156347463, Tokens per sec: 168774.66999174192, Loss: 2.449775218963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13755 , TFLOPS: 71.5062799354539, Tokens per sec: 177200.43874705912, Loss: 2.498594284057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13756 , TFLOPS: 69.68278138628934, Tokens per sec: 172681.60846728145, Loss: 2.4985647201538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13757 , TFLOPS: 68.82771532927971, Tokens per sec: 170562.66058470958, Loss: 2.4998960494995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13758 , TFLOPS: 70.09431058439705, Tokens per sec: 173701.42315387595, Loss: 2.4971823692321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13759 , TFLOPS: 71.54931508017094, Tokens per sec: 177307.08457638053, Loss: 2.4760780334472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13760 , TFLOPS: 68.81649967663125, Tokens per sec: 170534.86696194057, Loss: 2.492039918899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13761 , TFLOPS: 68.28535550042713, Tokens per sec: 169218.63318294557, Loss: 2.4787704944610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13762 , TFLOPS: 71.53468849339369, Tokens per sec: 177270.83828309493, Loss: 2.502955675125122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13763 , TFLOPS: 70.70364013662184, Tokens per sec: 175211.40890748033, Loss: 2.4883317947387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13764 , TFLOPS: 66.64341531555291, Tokens per sec: 165149.7245301843, Loss: 2.480902910232544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13765 , TFLOPS: 71.55914855595165, Tokens per sec: 177331.4530126119, Loss: 2.5075812339782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13766 , TFLOPS: 71.50561689011731, Tokens per sec: 177198.7956476746, Loss: 2.4854400157928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13767 , TFLOPS: 69.07172719766953, Tokens per sec: 171167.34887470584, Loss: 2.4910128116607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13768 , TFLOPS: 68.97986535345761, Tokens per sec: 170939.7051053312, Loss: 2.504359722137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13769 , TFLOPS: 71.55265622253114, Tokens per sec: 177315.3642952063, Loss: 2.4980931282043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13770 , TFLOPS: 68.96228313711744, Tokens per sec: 170896.13443639924, Loss: 2.5207676887512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13771 , TFLOPS: 68.64975384211179, Tokens per sec: 170121.6524154313, Loss: 2.4792511463165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13772 , TFLOPS: 71.53135475988645, Tokens per sec: 177262.57692422293, Loss: 2.504049301147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13773 , TFLOPS: 69.7587556661857, Tokens per sec: 172869.8811021225, Loss: 2.4687018394470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13774 , TFLOPS: 68.8263327787638, Tokens per sec: 170559.2344722306, Loss: 2.487377882003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13775 , TFLOPS: 69.83863718394265, Tokens per sec: 173067.8363027998, Loss: 2.484135389328003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13776 , TFLOPS: 70.76315853531754, Tokens per sec: 175358.90205594114, Loss: 2.503844738006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13777 , TFLOPS: 69.77430874789597, Tokens per sec: 172908.42335191395, Loss: 2.486356258392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13778 , TFLOPS: 69.18991803246817, Tokens per sec: 171460.23878313217, Loss: 2.500518798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13779 , TFLOPS: 70.48538959139765, Tokens per sec: 174670.5600141325, Loss: 2.5040316581726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13780 , TFLOPS: 69.79508120191637, Tokens per sec: 172959.8997812512, Loss: 2.47762131690979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13781 , TFLOPS: 68.21805907118133, Tokens per sec: 169051.86521796085, Loss: 2.4919161796569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13782 , TFLOPS: 71.53384925595905, Tokens per sec: 177268.75856028227, Loss: 2.501950979232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13783 , TFLOPS: 69.6524704988731, Tokens per sec: 172606.4947492442, Loss: 2.510770082473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13784 , TFLOPS: 67.8682803100446, Tokens per sec: 168185.0749165526, Loss: 2.4733493328094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13785 , TFLOPS: 70.56142151107598, Tokens per sec: 174858.97548670275, Loss: 2.479099988937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13786 , TFLOPS: 70.55129133288818, Tokens per sec: 174833.87178922215, Loss: 2.486204147338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13787 , TFLOPS: 68.87245338750162, Tokens per sec: 170673.52642128745, Loss: 2.4969522953033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13788 , TFLOPS: 69.00024878877787, Tokens per sec: 170990.21750347558, Loss: 2.4914169311523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13789 , TFLOPS: 71.47986558203809, Tokens per sec: 177134.9810695135, Loss: 2.4880831241607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13790 , TFLOPS: 67.85157194752252, Tokens per sec: 168143.66975364482, Loss: 2.508382558822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13791 , TFLOPS: 69.59233401802203, Tokens per sec: 172457.46992511497, Loss: 2.5003280639648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13792 , TFLOPS: 71.53109022407085, Tokens per sec: 177261.92137533083, Loss: 2.4707508087158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13793 , TFLOPS: 69.4982089691181, Tokens per sec: 172224.2176852009, Loss: 2.4972615242004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13794 , TFLOPS: 69.92634699380133, Tokens per sec: 173285.19087366282, Loss: 2.4738712310791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13795 , TFLOPS: 69.98472361412887, Tokens per sec: 173429.85456955008, Loss: 2.5035715103149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13796 , TFLOPS: 70.74902544621173, Tokens per sec: 175323.87870424867, Loss: 2.4956631660461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13797 , TFLOPS: 68.62100489526107, Tokens per sec: 170050.40935817725, Loss: 2.48130202293396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13798 , TFLOPS: 68.83623827724432, Tokens per sec: 170583.78138864614, Loss: 2.477642297744751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13799 , TFLOPS: 71.47600309080525, Tokens per sec: 177125.40939075002, Loss: 2.4974472522735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13800 , TFLOPS: 69.83870339424462, Tokens per sec: 173068.00037922175, Loss: 2.494095802307129 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/optimizer.pt +[rank0]:[2024-08-27 07:10:22,102] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006429344997741282, 'preprocessing_with_comm': 0.0013171210011932999, 'state_converting': 0.9826521630020579, : 0.9917558760062093}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800, took 4.30s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0011800 + eval ppl=10.087128639221191, eval loss=2.311260223388672 +------------------------------------------------------------------ +iteration: 13801 , TFLOPS: 63.71186296702437, Tokens per sec: 157885.0148733802, Loss: 2.4990062713623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13802 , TFLOPS: 70.6932786355169, Tokens per sec: 175185.73196632872, Loss: 2.5048630237579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13803 , TFLOPS: 70.67566013871156, Tokens per sec: 175142.07139040745, Loss: 2.4969491958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13804 , TFLOPS: 67.38584745814684, Tokens per sec: 166989.55316518416, Loss: 2.504254102706909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13805 , TFLOPS: 69.96853459891246, Tokens per sec: 173389.73640647653, Loss: 2.4869117736816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13806 , TFLOPS: 71.47777677109617, Tokens per sec: 177129.80476589792, Loss: 2.4704155921936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13807 , TFLOPS: 69.75332825903254, Tokens per sec: 172856.43138931872, Loss: 2.492865800857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13808 , TFLOPS: 67.76519385351685, Tokens per sec: 167929.6153213662, Loss: 2.5102643966674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13809 , TFLOPS: 71.47304140803718, Tokens per sec: 177118.07001459444, Loss: 2.4924964904785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13810 , TFLOPS: 70.62225567935602, Tokens per sec: 175009.72925713757, Loss: 2.4865505695343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13811 , TFLOPS: 67.53947317097739, Tokens per sec: 167370.25460484758, Loss: 2.5043447017669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13812 , TFLOPS: 70.3988947560734, Tokens per sec: 174456.2162840063, Loss: 2.5031490325927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13813 , TFLOPS: 70.62611667718329, Tokens per sec: 175019.29723507757, Loss: 2.4847049713134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13814 , TFLOPS: 69.01079153042436, Tokens per sec: 171016.34357865443, Loss: 2.4878835678100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13815 , TFLOPS: 69.83070882500155, Tokens per sec: 173048.18895596295, Loss: 2.465874195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13816 , TFLOPS: 69.6137146470091, Tokens per sec: 172510.45347901562, Loss: 2.485480546951294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13817 , TFLOPS: 70.56584306467263, Tokens per sec: 174869.9325835896, Loss: 2.492163896560669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13818 , TFLOPS: 68.34689745110197, Tokens per sec: 169371.14091612335, Loss: 2.47467041015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13819 , TFLOPS: 70.38376713139385, Tokens per sec: 174418.72836360495, Loss: 2.508819818496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13820 , TFLOPS: 69.78329108134368, Tokens per sec: 172930.68256368305, Loss: 2.482635736465454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13821 , TFLOPS: 68.80735559363339, Tokens per sec: 170512.20691696796, Loss: 2.4903368949890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13822 , TFLOPS: 70.60505912628163, Tokens per sec: 174967.11430426253, Loss: 2.4920804500579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13823 , TFLOPS: 69.5079817962924, Tokens per sec: 172248.43582750464, Loss: 2.493955373764038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13824 , TFLOPS: 68.75980659416439, Tokens per sec: 170394.3752583291, Loss: 2.501495599746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13825 , TFLOPS: 69.87630816390595, Tokens per sec: 173161.1891409499, Loss: 2.509016513824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13826 , TFLOPS: 70.40629129504259, Tokens per sec: 174474.545722368, Loss: 2.492521286010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13827 , TFLOPS: 69.795043707725, Tokens per sec: 172959.80686651464, Loss: 2.5023131370544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13828 , TFLOPS: 68.64663865216741, Tokens per sec: 170113.9326315826, Loss: 2.51936674118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13829 , TFLOPS: 71.52057189829081, Tokens per sec: 177235.85580535867, Loss: 2.521374464035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13830 , TFLOPS: 69.60308130383855, Tokens per sec: 172484.10288327956, Loss: 2.478058338165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13831 , TFLOPS: 68.05626097026719, Tokens per sec: 168650.9117003634, Loss: 2.4801101684570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13832 , TFLOPS: 71.02483754829127, Tokens per sec: 176007.371476413, Loss: 2.4640238285064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13833 , TFLOPS: 70.64550831488906, Tokens per sec: 175067.35184947768, Loss: 2.4973912239074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13834 , TFLOPS: 69.66978647976575, Tokens per sec: 172649.40565740888, Loss: 2.4790782928466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13835 , TFLOPS: 68.99348276480976, Tokens per sec: 170973.45055074352, Loss: 2.495145797729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13836 , TFLOPS: 71.50032961645563, Tokens per sec: 177185.69320109967, Loss: 2.496575117111206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13837 , TFLOPS: 70.0237302926548, Tokens per sec: 173526.5174158777, Loss: 2.50920033454895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13838 , TFLOPS: 67.85153012099357, Tokens per sec: 168143.56610289853, Loss: 2.4820456504821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13839 , TFLOPS: 71.5314874441824, Tokens per sec: 177262.90573052468, Loss: 2.4866979122161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13840 , TFLOPS: 69.90782184039502, Tokens per sec: 173239.28350280784, Loss: 2.4743354320526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13841 , TFLOPS: 68.29533128418538, Tokens per sec: 169243.35427402184, Loss: 2.4876925945281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13842 , TFLOPS: 70.61565261554745, Tokens per sec: 174993.3661376324, Loss: 2.506840229034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13843 , TFLOPS: 70.55413242722454, Tokens per sec: 174840.91233396006, Loss: 2.4906959533691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13844 , TFLOPS: 69.75087484795691, Tokens per sec: 172850.35156640774, Loss: 2.484848737716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13845 , TFLOPS: 68.02122925150942, Tokens per sec: 168564.09924221926, Loss: 2.5144710540771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13846 , TFLOPS: 71.50987849101294, Tokens per sec: 177209.35636414803, Loss: 2.4727818965911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13847 , TFLOPS: 69.78975961278017, Tokens per sec: 172946.71229715025, Loss: 2.4930381774902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13848 , TFLOPS: 69.64594684972509, Tokens per sec: 172590.3284280161, Loss: 2.4790186882019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13849 , TFLOPS: 69.86873369489258, Tokens per sec: 173142.41877233816, Loss: 2.495244026184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13850 , TFLOPS: 70.5435602626484, Tokens per sec: 174814.7133455203, Loss: 2.4727182388305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13851 , TFLOPS: 68.00041164727428, Tokens per sec: 168512.51092567615, Loss: 2.4840636253356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13852 , TFLOPS: 70.64726911891242, Tokens per sec: 175071.71531581617, Loss: 2.518672466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13853 , TFLOPS: 69.23878668650595, Tokens per sec: 171581.34069116443, Loss: 2.4994568824768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13854 , TFLOPS: 70.03263094740774, Tokens per sec: 173548.57421884354, Loss: 2.50006365776062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13855 , TFLOPS: 68.4369794015182, Tokens per sec: 169594.3739126008, Loss: 2.4993655681610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13856 , TFLOPS: 70.36673061804424, Tokens per sec: 174376.51000679776, Loss: 2.498137950897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13857 , TFLOPS: 69.67078889834981, Tokens per sec: 172651.8897610857, Loss: 2.4814844131469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13858 , TFLOPS: 67.80638076908825, Tokens per sec: 168031.68103526544, Loss: 2.4945027828216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13859 , TFLOPS: 71.49978030300271, Tokens per sec: 177184.33194185144, Loss: 2.507674217224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13860 , TFLOPS: 69.36761094026393, Tokens per sec: 171900.58138313974, Loss: 2.4798154830932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13861 , TFLOPS: 68.29234506740093, Tokens per sec: 169235.95409987104, Loss: 2.4917526245117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13862 , TFLOPS: 70.7523875231941, Tokens per sec: 175332.21030137435, Loss: 2.5133748054504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13863 , TFLOPS: 69.95628714067574, Tokens per sec: 173359.38585579707, Loss: 2.5017597675323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13864 , TFLOPS: 70.72611155659544, Tokens per sec: 175267.09556160617, Loss: 2.513871908187866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13865 , TFLOPS: 68.19396376439842, Tokens per sec: 168992.1543934356, Loss: 2.501486301422119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13866 , TFLOPS: 71.53558912739416, Tokens per sec: 177273.07015336113, Loss: 2.4886538982391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13867 , TFLOPS: 68.81158819919155, Tokens per sec: 170522.69577980108, Loss: 2.4836843013763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13868 , TFLOPS: 69.65087590902074, Tokens per sec: 172602.5431799314, Loss: 2.4950320720672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13869 , TFLOPS: 70.67698452638137, Tokens per sec: 175145.35336894658, Loss: 2.5087473392486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13870 , TFLOPS: 70.61344458541028, Tokens per sec: 174987.89439571789, Loss: 2.4898228645324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13871 , TFLOPS: 68.50966012637517, Tokens per sec: 169774.4847552392, Loss: 2.5052971839904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13872 , TFLOPS: 70.00172340632707, Tokens per sec: 173471.9819272989, Loss: 2.49474835395813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13873 , TFLOPS: 71.52588031349694, Tokens per sec: 177249.01064300965, Loss: 2.472334384918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13874 , TFLOPS: 69.75513205003557, Tokens per sec: 172860.9013821282, Loss: 2.5038092136383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13875 , TFLOPS: 67.96833174195086, Tokens per sec: 168433.01338639142, Loss: 2.498185396194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13876 , TFLOPS: 71.40416324638329, Tokens per sec: 176947.3823424602, Loss: 2.480409860610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13877 , TFLOPS: 70.55567073423198, Tokens per sec: 174844.72442818296, Loss: 2.4913346767425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13878 , TFLOPS: 67.33156287664053, Tokens per sec: 166855.0299922712, Loss: 2.4714818000793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13879 , TFLOPS: 71.5447867703703, Tokens per sec: 177295.86292586, Loss: 2.485114336013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13880 , TFLOPS: 70.65902042935681, Tokens per sec: 175100.83635761106, Loss: 2.518362522125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13881 , TFLOPS: 68.91314090211772, Tokens per sec: 170774.35456460583, Loss: 2.4786784648895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13882 , TFLOPS: 69.03087461661266, Tokens per sec: 171066.11167856364, Loss: 2.479510545730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13883 , TFLOPS: 71.50437335688379, Tokens per sec: 177195.71403533543, Loss: 2.502134323120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13884 , TFLOPS: 69.9855286725243, Tokens per sec: 173431.84959293844, Loss: 2.493056058883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13885 , TFLOPS: 68.09042384107893, Tokens per sec: 168735.57105758577, Loss: 2.5114212036132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13886 , TFLOPS: 70.43162780111031, Tokens per sec: 174537.33237545035, Loss: 2.469442367553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13887 , TFLOPS: 69.57120742082769, Tokens per sec: 172405.11588998057, Loss: 2.4914302825927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13888 , TFLOPS: 69.56440069795634, Tokens per sec: 172388.24808088297, Loss: 2.47403883934021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13889 , TFLOPS: 70.26264595592626, Tokens per sec: 174118.57674819735, Loss: 2.485860586166382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13890 , TFLOPS: 69.60317912254126, Tokens per sec: 172484.34528880048, Loss: 2.480825185775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13891 , TFLOPS: 68.91768507315636, Tokens per sec: 170785.61552102136, Loss: 2.5069470405578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13892 , TFLOPS: 69.85923642940374, Tokens per sec: 173118.8835022466, Loss: 2.4685680866241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13893 , TFLOPS: 70.42986925998473, Tokens per sec: 174532.97451682188, Loss: 2.4942688941955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13894 , TFLOPS: 69.74156612082318, Tokens per sec: 172827.28351512944, Loss: 2.4932923316955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13895 , TFLOPS: 67.96717394058157, Tokens per sec: 168430.14422705595, Loss: 2.5215351581573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13896 , TFLOPS: 71.4850164099852, Tokens per sec: 177147.74538857702, Loss: 2.4864020347595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13897 , TFLOPS: 69.64825144538358, Tokens per sec: 172596.03946993002, Loss: 2.4545862674713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13898 , TFLOPS: 68.09803874637518, Tokens per sec: 168754.44163176094, Loss: 2.4869236946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13899 , TFLOPS: 70.92116082966328, Tokens per sec: 175750.44914671947, Loss: 2.5075297355651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13900 , TFLOPS: 70.55740142986215, Tokens per sec: 174849.0132826064, Loss: 2.5033814907073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13901 , TFLOPS: 69.74583282637654, Tokens per sec: 172837.8568815071, Loss: 2.4696645736694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13902 , TFLOPS: 68.99904730957417, Tokens per sec: 170987.24010565493, Loss: 2.5040550231933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13903 , TFLOPS: 71.50354175468398, Tokens per sec: 177193.65323347575, Loss: 2.47971773147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13904 , TFLOPS: 68.6213553771167, Tokens per sec: 170051.27789082428, Loss: 2.5082032680511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13905 , TFLOPS: 67.98467421886012, Tokens per sec: 168473.511844445, Loss: 2.4954235553741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13906 , TFLOPS: 71.49338266143457, Tokens per sec: 177168.47788128722, Loss: 2.456472396850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13907 , TFLOPS: 69.79834312849809, Tokens per sec: 172967.98319465155, Loss: 2.497925043106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13908 , TFLOPS: 68.46200114233503, Tokens per sec: 169656.38054271718, Loss: 2.489722490310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13909 , TFLOPS: 70.69251683926392, Tokens per sec: 175183.84415129505, Loss: 2.4753708839416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13910 , TFLOPS: 70.63133970850006, Tokens per sec: 175032.24048204493, Loss: 2.510571241378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13911 , TFLOPS: 68.86360463880801, Tokens per sec: 170651.5982472545, Loss: 2.4958043098449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13912 , TFLOPS: 69.61591084270202, Tokens per sec: 172515.89589387362, Loss: 2.508254289627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13913 , TFLOPS: 71.49880448340828, Tokens per sec: 177181.91375340105, Loss: 2.486086130142212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13914 , TFLOPS: 69.77513923589132, Tokens per sec: 172910.4813926522, Loss: 2.4596409797668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13915 , TFLOPS: 67.11507299067584, Tokens per sec: 166318.54420652104, Loss: 2.509033679962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13916 , TFLOPS: 70.9546145624102, Tokens per sec: 175833.35118169925, Loss: 2.505831718444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13917 , TFLOPS: 70.62651702473363, Tokens per sec: 175020.28934040922, Loss: 2.4802331924438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13918 , TFLOPS: 68.03924520906112, Tokens per sec: 168608.74476965395, Loss: 2.498816967010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13919 , TFLOPS: 69.5770219424961, Tokens per sec: 172419.52491519737, Loss: 2.4854886531829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13920 , TFLOPS: 71.51335622804893, Tokens per sec: 177217.97457962763, Loss: 2.4909284114837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13921 , TFLOPS: 70.07412503343502, Tokens per sec: 173651.40113497007, Loss: 2.4840874671936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13922 , TFLOPS: 68.27965194926057, Tokens per sec: 169204.4991548568, Loss: 2.496694326400757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13923 , TFLOPS: 70.46374691009112, Tokens per sec: 174616.92706571674, Loss: 2.4926559925079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13924 , TFLOPS: 69.71219611788553, Tokens per sec: 172754.501412477, Loss: 2.499584913253784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13925 , TFLOPS: 67.92112955246698, Tokens per sec: 168316.0411610999, Loss: 2.4838249683380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13926 , TFLOPS: 71.51720473087548, Tokens per sec: 177227.51159357937, Loss: 2.5008347034454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13927 , TFLOPS: 69.6662356558391, Tokens per sec: 172640.60632456312, Loss: 2.4930684566497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13928 , TFLOPS: 68.92614036988381, Tokens per sec: 170806.56867773982, Loss: 2.480086326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13929 , TFLOPS: 70.71436717281726, Tokens per sec: 175237.99168485316, Loss: 2.4963009357452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13930 , TFLOPS: 69.60748289722959, Tokens per sec: 172495.0105165781, Loss: 2.489325523376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13931 , TFLOPS: 69.74624393882802, Tokens per sec: 172838.8756634496, Loss: 2.4943816661834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13932 , TFLOPS: 69.20496502447716, Tokens per sec: 171497.52688689344, Loss: 2.4905452728271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13933 , TFLOPS: 71.47778246468815, Tokens per sec: 177129.81887524604, Loss: 2.488590955734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13934 , TFLOPS: 68.70760709977425, Tokens per sec: 170265.0191609813, Loss: 2.4768426418304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13935 , TFLOPS: 68.78104335283685, Tokens per sec: 170447.00228574153, Loss: 2.4884440898895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13936 , TFLOPS: 70.60467171423704, Tokens per sec: 174966.15425453908, Loss: 2.497995138168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13937 , TFLOPS: 69.57587597252157, Tokens per sec: 172416.68507536143, Loss: 2.504112482070923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13938 , TFLOPS: 68.91247300472284, Tokens per sec: 170772.6994412864, Loss: 2.4884772300720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13939 , TFLOPS: 70.24869480619842, Tokens per sec: 174084.0042623257, Loss: 2.5128042697906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13940 , TFLOPS: 71.53541569698663, Tokens per sec: 177272.64037370632, Loss: 2.510026216506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13941 , TFLOPS: 68.82197439973316, Tokens per sec: 170548.43392887764, Loss: 2.4958291053771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13942 , TFLOPS: 67.88131581497393, Tokens per sec: 168217.3783337469, Loss: 2.484869956970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13943 , TFLOPS: 71.5009886144914, Tokens per sec: 177187.32627082695, Loss: 2.4647765159606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13944 , TFLOPS: 70.60168447673502, Tokens per sec: 174958.75155093768, Loss: 2.5036818981170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13945 , TFLOPS: 67.19035332171411, Tokens per sec: 166505.09715964692, Loss: 2.517429828643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13946 , TFLOPS: 71.00619752814788, Tokens per sec: 175961.17945312997, Loss: 2.5028817653656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13947 , TFLOPS: 70.72196307438284, Tokens per sec: 175256.81516569035, Loss: 2.490556478500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13948 , TFLOPS: 69.7663965407895, Tokens per sec: 172888.81603110256, Loss: 2.4874444007873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13949 , TFLOPS: 68.21344080222741, Tokens per sec: 169040.42063874865, Loss: 2.5010900497436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13950 , TFLOPS: 70.67128720589892, Tokens per sec: 175131.23478117943, Loss: 2.4696054458618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13951 , TFLOPS: 70.6201546883572, Tokens per sec: 175004.52276999922, Loss: 2.4989800453186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13952 , TFLOPS: 66.8306525589766, Tokens per sec: 165613.71904527326, Loss: 2.492914915084839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13953 , TFLOPS: 71.47793402845384, Tokens per sec: 177130.19446695354, Loss: 2.5069432258605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13954 , TFLOPS: 69.71208778530318, Tokens per sec: 172754.23295240416, Loss: 2.506974458694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13955 , TFLOPS: 68.8133186228715, Tokens per sec: 170526.9839603019, Loss: 2.4801764488220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13956 , TFLOPS: 69.62001194494574, Tokens per sec: 172526.05887700728, Loss: 2.506288528442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13957 , TFLOPS: 70.70355965190862, Tokens per sec: 175211.20945749484, Loss: 2.5015058517456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13958 , TFLOPS: 68.79872040834225, Tokens per sec: 170490.80797657234, Loss: 2.4886345863342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13959 , TFLOPS: 69.92615184455582, Tokens per sec: 173284.70727233498, Loss: 2.50034499168396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13960 , TFLOPS: 70.46258612718783, Tokens per sec: 174614.0505178116, Loss: 2.486368179321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13961 , TFLOPS: 69.74797089579933, Tokens per sec: 172843.15525306453, Loss: 2.488124370574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13962 , TFLOPS: 68.03212318182979, Tokens per sec: 168591.09560161858, Loss: 2.4833168983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13963 , TFLOPS: 71.50514808668682, Tokens per sec: 177197.63390113087, Loss: 2.4945831298828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13964 , TFLOPS: 69.72764369979697, Tokens per sec: 172792.7822221452, Loss: 2.496166467666626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13965 , TFLOPS: 68.04399869629931, Tokens per sec: 168620.52443467, Loss: 2.4508659839630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13966 , TFLOPS: 70.74656574039498, Tokens per sec: 175317.7832822758, Loss: 2.501945972442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13967 , TFLOPS: 69.61114630609823, Tokens per sec: 172504.0888473124, Loss: 2.4784157276153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13968 , TFLOPS: 70.72012234913923, Tokens per sec: 175252.25364576405, Loss: 2.5144386291503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13969 , TFLOPS: 68.8923807147912, Tokens per sec: 170722.90853348735, Loss: 2.497791290283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13970 , TFLOPS: 71.5575483949587, Tokens per sec: 177327.48763739908, Loss: 2.4926412105560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13971 , TFLOPS: 69.04984694695732, Tokens per sec: 171113.1272031324, Loss: 2.500952959060669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13972 , TFLOPS: 68.1311638864011, Tokens per sec: 168836.5293777806, Loss: 2.4840734004974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13973 , TFLOPS: 71.52124434183068, Tokens per sec: 177237.52219452625, Loss: 2.5005977153778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13974 , TFLOPS: 69.67325302705461, Tokens per sec: 172657.99614346243, Loss: 2.4946446418762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13975 , TFLOPS: 68.20316756396505, Tokens per sec: 169014.96242264393, Loss: 2.471970558166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13976 , TFLOPS: 70.59160357575972, Tokens per sec: 174933.769967818, Loss: 2.4935789108276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13977 , TFLOPS: 70.71336160607537, Tokens per sec: 175235.49977969442, Loss: 2.4935173988342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13978 , TFLOPS: 68.50948082745978, Tokens per sec: 169774.04043277542, Loss: 2.469420909881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13979 , TFLOPS: 69.93209045497125, Tokens per sec: 173299.42380313633, Loss: 2.5105464458465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13980 , TFLOPS: 71.49374695687432, Tokens per sec: 177169.38064551735, Loss: 2.46882963180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13981 , TFLOPS: 69.77288678575559, Tokens per sec: 172904.899573088, Loss: 2.486947536468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13982 , TFLOPS: 68.7192213864166, Tokens per sec: 170293.80064270034, Loss: 2.490617275238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13983 , TFLOPS: 70.99820960485857, Tokens per sec: 175941.38449927667, Loss: 2.49562931060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13984 , TFLOPS: 70.68778084700021, Tokens per sec: 175172.10784075296, Loss: 2.4802355766296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13985 , TFLOPS: 67.97196639735996, Tokens per sec: 168442.0204628855, Loss: 2.502525806427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13986 , TFLOPS: 70.40746401604973, Tokens per sec: 174477.45185420956, Loss: 2.4778435230255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13987 , TFLOPS: 71.57133346576279, Tokens per sec: 177361.64856140252, Loss: 2.482822895050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13988 , TFLOPS: 70.01747346132622, Tokens per sec: 173511.0122985986, Loss: 2.4643425941467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13989 , TFLOPS: 67.97963000826142, Tokens per sec: 168461.01173491578, Loss: 2.466068983078003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13990 , TFLOPS: 71.52094147538915, Tokens per sec: 177236.77165812062, Loss: 2.5038297176361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13991 , TFLOPS: 69.7640529563689, Tokens per sec: 172883.00837074115, Loss: 2.510371685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13992 , TFLOPS: 68.08780472821485, Tokens per sec: 168729.08060738948, Loss: 2.4964468479156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13993 , TFLOPS: 70.43953825539431, Tokens per sec: 174556.93535257457, Loss: 2.4704198837280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13994 , TFLOPS: 70.58707791647204, Tokens per sec: 174922.5548855606, Loss: 2.466587781906128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13995 , TFLOPS: 68.0096389893167, Tokens per sec: 168535.3772957329, Loss: 2.484649658203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13996 , TFLOPS: 70.58953189024646, Tokens per sec: 174928.63610290113, Loss: 2.495915651321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13997 , TFLOPS: 69.70104513248178, Tokens per sec: 172726.8680422641, Loss: 2.4889795780181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13998 , TFLOPS: 69.68615478853897, Tokens per sec: 172689.968129667, Loss: 2.4785284996032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13999 , TFLOPS: 69.14851562067263, Tokens per sec: 171357.63904585064, Loss: 2.4695217609405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14000 , TFLOPS: 71.51789330273472, Tokens per sec: 177229.21795049903, Loss: 2.474944829940796 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/model.pt +[rank0]:[2024-08-27 07:35:47,271] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006396492011845112, 'preprocessing_with_comm': 0.0012864460004493594, 'state_converting': 0.9864355400059139, : 0.9955305889889132}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012000 + eval ppl=10.096779823303223, eval loss=2.3122165203094482 +------------------------------------------------------------------ +iteration: 14001 , TFLOPS: 65.30875248744391, Tokens per sec: 161842.28301688243, Loss: 2.4645602703094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14002 , TFLOPS: 69.66468858815928, Tokens per sec: 172636.77252042893, Loss: 2.495237112045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14003 , TFLOPS: 70.66694553525748, Tokens per sec: 175120.47564305543, Loss: 2.468592643737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14004 , TFLOPS: 69.21928635919237, Tokens per sec: 171533.01673194297, Loss: 2.47967267036438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14005 , TFLOPS: 68.7630142014618, Tokens per sec: 170402.32406255917, Loss: 2.5212929248809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14006 , TFLOPS: 70.64990413532631, Tokens per sec: 175078.2451767608, Loss: 2.4659175872802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14007 , TFLOPS: 71.50242136810017, Tokens per sec: 177190.87679210032, Loss: 2.4876322746276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14008 , TFLOPS: 68.76588434644115, Tokens per sec: 170409.4365979906, Loss: 2.522172212600708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14009 , TFLOPS: 67.9303055806121, Tokens per sec: 168338.7803696665, Loss: 2.494420289993286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14010 , TFLOPS: 71.50452271579192, Tokens per sec: 177196.084163163, Loss: 2.5062527656555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14011 , TFLOPS: 69.74162128828942, Tokens per sec: 172827.42022618797, Loss: 2.4859354496002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14012 , TFLOPS: 68.44144023441297, Tokens per sec: 169605.42834791535, Loss: 2.487961769104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14013 , TFLOPS: 70.14282872172352, Tokens per sec: 173821.656442885, Loss: 2.4834959506988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14014 , TFLOPS: 70.5869594975611, Tokens per sec: 174922.26143045432, Loss: 2.4751954078674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14015 , TFLOPS: 69.80156857645383, Tokens per sec: 172975.9762099983, Loss: 2.4911224842071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14016 , TFLOPS: 69.21597267386521, Tokens per sec: 171524.8050546697, Loss: 2.4818811416625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14017 , TFLOPS: 71.55528584755007, Tokens per sec: 177321.88079568028, Loss: 2.4881389141082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14018 , TFLOPS: 70.06562023108395, Tokens per sec: 173630.32529786276, Loss: 2.4890871047973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14019 , TFLOPS: 67.13390282672793, Tokens per sec: 166365.20661453577, Loss: 2.5040557384490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14020 , TFLOPS: 71.48611407983002, Tokens per sec: 177150.46553537075, Loss: 2.4703738689422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14021 , TFLOPS: 69.77134456782169, Tokens per sec: 172901.07778715846, Loss: 2.4728622436523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14022 , TFLOPS: 68.7737608767028, Tokens per sec: 170428.95550765033, Loss: 2.4900639057159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14023 , TFLOPS: 69.64182447094586, Tokens per sec: 172580.11271928367, Loss: 2.4765007495880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14024 , TFLOPS: 70.70033728021154, Tokens per sec: 175203.22406545596, Loss: 2.49863862991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14025 , TFLOPS: 68.88962786106119, Tokens per sec: 170716.08665869248, Loss: 2.4837417602539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14026 , TFLOPS: 70.13992105222805, Tokens per sec: 173814.450917569, Loss: 2.483794927597046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14027 , TFLOPS: 70.38455406451662, Tokens per sec: 174420.67847057103, Loss: 2.5113182067871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14028 , TFLOPS: 69.87109149195759, Tokens per sec: 173148.26165319802, Loss: 2.484576463699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14029 , TFLOPS: 68.18152323158313, Tokens per sec: 168961.32538268148, Loss: 2.488307476043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14030 , TFLOPS: 69.81904752837559, Tokens per sec: 173019.29097832597, Loss: 2.488067150115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14031 , TFLOPS: 70.58122118404788, Tokens per sec: 174908.04125744116, Loss: 2.48477840423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14032 , TFLOPS: 69.05093748716202, Tokens per sec: 171115.82968189256, Loss: 2.5008983612060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14033 , TFLOPS: 70.74088218813296, Tokens per sec: 175303.69881367552, Loss: 2.499342441558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14034 , TFLOPS: 70.43681073760173, Tokens per sec: 174550.1762630237, Loss: 2.4741363525390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14035 , TFLOPS: 69.35697593911576, Tokens per sec: 171874.2266787525, Loss: 2.4830682277679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14036 , TFLOPS: 68.55151252712842, Tokens per sec: 169878.19961472828, Loss: 2.495729923248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14037 , TFLOPS: 70.44676329757783, Tokens per sec: 174574.83980300944, Loss: 2.4768903255462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14038 , TFLOPS: 69.69034746625499, Tokens per sec: 172700.3580469089, Loss: 2.4888617992401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14039 , TFLOPS: 68.14176805051588, Tokens per sec: 168862.8076646021, Loss: 2.4989755153656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14040 , TFLOPS: 71.51417824426905, Tokens per sec: 177220.01162637363, Loss: 2.508622169494629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14041 , TFLOPS: 67.80083576875741, Tokens per sec: 168017.9399136134, Loss: 2.503419876098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14042 , TFLOPS: 68.85831151206379, Tokens per sec: 170638.48129609632, Loss: 2.4883644580841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14043 , TFLOPS: 70.6435076756952, Tokens per sec: 175062.3940451667, Loss: 2.49219012260437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14044 , TFLOPS: 70.58483993133659, Tokens per sec: 174917.0089118605, Loss: 2.4936447143554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14045 , TFLOPS: 68.88615221489765, Tokens per sec: 170707.47362462519, Loss: 2.473306179046631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14046 , TFLOPS: 69.13428567171835, Tokens per sec: 171322.3756647836, Loss: 2.490821361541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14047 , TFLOPS: 71.52526956701932, Tokens per sec: 177247.49714595886, Loss: 2.483182907104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14048 , TFLOPS: 69.84127120621905, Tokens per sec: 173074.36370016224, Loss: 2.4929466247558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14049 , TFLOPS: 68.58300751822142, Tokens per sec: 169956.2476721164, Loss: 2.4940342903137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14050 , TFLOPS: 70.60534060395811, Tokens per sec: 174967.81183695217, Loss: 2.488584280014038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14051 , TFLOPS: 70.54417409784926, Tokens per sec: 174816.23449676755, Loss: 2.51200270652771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14052 , TFLOPS: 68.60800028759225, Tokens per sec: 170018.1825077981, Loss: 2.4728662967681885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14053 , TFLOPS: 70.71565560811742, Tokens per sec: 175241.18456945982, Loss: 2.522266387939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14054 , TFLOPS: 71.52995198431255, Tokens per sec: 177259.1006918223, Loss: 2.4966771602630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14055 , TFLOPS: 69.05735303131908, Tokens per sec: 171131.7281070443, Loss: 2.4836981296539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14056 , TFLOPS: 67.67624616739865, Tokens per sec: 167709.1931567688, Loss: 2.497223377227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14057 , TFLOPS: 71.54618763347196, Tokens per sec: 177299.3344189437, Loss: 2.496616840362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14058 , TFLOPS: 69.77638781418808, Tokens per sec: 172913.57550721383, Loss: 2.4896433353424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14059 , TFLOPS: 67.98760467273397, Tokens per sec: 168480.77383196054, Loss: 2.494865894317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14060 , TFLOPS: 70.42484382438533, Tokens per sec: 174520.52093380914, Loss: 2.5084915161132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14061 , TFLOPS: 69.82403173024588, Tokens per sec: 173031.6423796157, Loss: 2.476763963699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14062 , TFLOPS: 69.90584773735091, Tokens per sec: 173234.39145799918, Loss: 2.473878860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14063 , TFLOPS: 69.85801003936022, Tokens per sec: 173115.84437262078, Loss: 2.4717421531677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14064 , TFLOPS: 69.6042920975408, Tokens per sec: 172487.1033634537, Loss: 2.4901676177978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14065 , TFLOPS: 70.7416207062924, Tokens per sec: 175305.52894302912, Loss: 2.50545334815979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14066 , TFLOPS: 68.30947912522194, Tokens per sec: 169278.41418262376, Loss: 2.506279945373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14067 , TFLOPS: 70.42042641542571, Tokens per sec: 174509.5741077894, Loss: 2.4854114055633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14068 , TFLOPS: 69.79753717177493, Tokens per sec: 172965.98594510814, Loss: 2.4863710403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14069 , TFLOPS: 68.99504741379427, Tokens per sec: 170977.32792328746, Loss: 2.4733352661132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14070 , TFLOPS: 70.60021083611878, Tokens per sec: 174955.0997071557, Loss: 2.5033910274505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14071 , TFLOPS: 69.10047427041543, Tokens per sec: 171238.5872877217, Loss: 2.490892171859741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14072 , TFLOPS: 68.05785782347526, Tokens per sec: 168654.8688785209, Loss: 2.4930500984191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14073 , TFLOPS: 70.72929070588373, Tokens per sec: 175274.97384375473, Loss: 2.4928927421569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14074 , TFLOPS: 70.42123988994852, Tokens per sec: 174511.58998726858, Loss: 2.490145444869995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14075 , TFLOPS: 69.85132096766358, Tokens per sec: 173099.26811609443, Loss: 2.503288984298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14076 , TFLOPS: 68.29220601638998, Tokens per sec: 169235.60951614816, Loss: 2.4678499698638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14077 , TFLOPS: 71.53581795275218, Tokens per sec: 177273.63720780547, Loss: 2.4926908016204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14078 , TFLOPS: 68.83248486018998, Tokens per sec: 170574.48000771244, Loss: 2.4913601875305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14079 , TFLOPS: 68.88263981668891, Tokens per sec: 170698.76951494132, Loss: 2.4870378971099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14080 , TFLOPS: 71.01143061144816, Tokens per sec: 175974.14761001978, Loss: 2.506216526031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14081 , TFLOPS: 70.69058298638555, Tokens per sec: 175179.05185083096, Loss: 2.510411262512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14082 , TFLOPS: 68.97629871195632, Tokens per sec: 170930.866574793, Loss: 2.477268934249878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14083 , TFLOPS: 68.99030180705898, Tokens per sec: 170965.56778702568, Loss: 2.482365846633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14084 , TFLOPS: 71.52980000649865, Tokens per sec: 177258.7240740577, Loss: 2.483776569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14085 , TFLOPS: 70.07899485606856, Tokens per sec: 173663.4690919112, Loss: 2.4953808784484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14086 , TFLOPS: 67.98647808043687, Tokens per sec: 168477.98201214394, Loss: 2.5031137466430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14087 , TFLOPS: 71.50293939522354, Tokens per sec: 177192.16052037742, Loss: 2.498802661895752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14088 , TFLOPS: 69.7267144491362, Tokens per sec: 172790.4794366424, Loss: 2.4955406188964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14089 , TFLOPS: 68.64849434304188, Tokens per sec: 170118.53123799077, Loss: 2.484753370285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14090 , TFLOPS: 70.69060515777527, Tokens per sec: 175179.10679397706, Loss: 2.4755778312683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14091 , TFLOPS: 70.57473723489075, Tokens per sec: 174891.97331716382, Loss: 2.491027593612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14092 , TFLOPS: 69.74225184862861, Tokens per sec: 172828.98282417044, Loss: 2.506486654281616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14093 , TFLOPS: 68.54785912614734, Tokens per sec: 169869.1460846418, Loss: 2.489125967025757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14094 , TFLOPS: 71.48508151258761, Tokens per sec: 177147.9067199974, Loss: 2.480236768722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14095 , TFLOPS: 69.78451326731724, Tokens per sec: 172933.71127516043, Loss: 2.4935500621795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14096 , TFLOPS: 68.66989557542284, Tokens per sec: 170171.56584937117, Loss: 2.489260196685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14097 , TFLOPS: 70.06270626606558, Tokens per sec: 173623.1041715475, Loss: 2.5147836208343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14098 , TFLOPS: 70.56471675224917, Tokens per sec: 174867.1414573307, Loss: 2.491610050201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14099 , TFLOPS: 68.31996534359531, Tokens per sec: 169304.4001869053, Loss: 2.4900689125061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14100 , TFLOPS: 70.48338575596067, Tokens per sec: 174665.59428917896, Loss: 2.485337495803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14101 , TFLOPS: 70.40100876761302, Tokens per sec: 174461.45503747909, Loss: 2.473040819168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14102 , TFLOPS: 69.20918993675852, Tokens per sec: 171507.99668493913, Loss: 2.490128993988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14103 , TFLOPS: 68.93320902709286, Tokens per sec: 170824.0855889807, Loss: 2.454991102218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14104 , TFLOPS: 70.51943695381222, Tokens per sec: 174754.9331288224, Loss: 2.4842324256896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14105 , TFLOPS: 69.77995130132513, Tokens per sec: 172922.40622088997, Loss: 2.487025260925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14106 , TFLOPS: 67.954404330772, Tokens per sec: 168398.4997272592, Loss: 2.490614652633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14107 , TFLOPS: 71.49766528206163, Tokens per sec: 177179.09068697068, Loss: 2.502488851547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14108 , TFLOPS: 68.71875063325902, Tokens per sec: 170292.63406451812, Loss: 2.5051796436309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14109 , TFLOPS: 68.84750938244935, Tokens per sec: 170611.71242895888, Loss: 2.4924113750457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14110 , TFLOPS: 70.80574672617406, Tokens per sec: 175464.4402843601, Loss: 2.4815666675567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14111 , TFLOPS: 70.52101491021959, Tokens per sec: 174758.84347862608, Loss: 2.47601580619812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14112 , TFLOPS: 70.65744505140756, Tokens per sec: 175096.93239751062, Loss: 2.4973645210266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14113 , TFLOPS: 69.03343512435947, Tokens per sec: 171072.45689882344, Loss: 2.4922425746917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14114 , TFLOPS: 71.52135339434244, Tokens per sec: 177237.7924386635, Loss: 2.484060287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14115 , TFLOPS: 68.73326378791417, Tokens per sec: 170328.59926050954, Loss: 2.5024609565734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14116 , TFLOPS: 68.7055617506156, Tokens per sec: 170259.9505604524, Loss: 2.50000262260437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14117 , TFLOPS: 70.70962466380358, Tokens per sec: 175226.23922508676, Loss: 2.5019571781158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14118 , TFLOPS: 70.64928565870137, Tokens per sec: 175076.71252355337, Loss: 2.479780912399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14119 , TFLOPS: 67.08443730902393, Tokens per sec: 166242.62561258866, Loss: 2.483001232147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14120 , TFLOPS: 70.66839819238733, Tokens per sec: 175124.07548744656, Loss: 2.5087904930114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14121 , TFLOPS: 71.48662038447172, Tokens per sec: 177151.72021404697, Loss: 2.4608542919158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14122 , TFLOPS: 69.31644217443997, Tokens per sec: 171773.77954472313, Loss: 2.4994542598724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14123 , TFLOPS: 67.81912259375919, Tokens per sec: 168063.2566804279, Loss: 2.4695687294006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14124 , TFLOPS: 71.52476981703542, Tokens per sec: 177246.2587104467, Loss: 2.482541084289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14125 , TFLOPS: 69.81196396547828, Tokens per sec: 173001.73712914754, Loss: 2.5149381160736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14126 , TFLOPS: 67.1590123359887, Tokens per sec: 166427.43074452475, Loss: 2.495272636413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14127 , TFLOPS: 71.55350719654018, Tokens per sec: 177317.4731025432, Loss: 2.4794721603393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14128 , TFLOPS: 70.71760132737028, Tokens per sec: 175246.0062761071, Loss: 2.4993960857391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14129 , TFLOPS: 69.17124131231829, Tokens per sec: 171413.95581319017, Loss: 2.4925618171691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14130 , TFLOPS: 69.02522912809309, Tokens per sec: 171052.12153611024, Loss: 2.4758119583129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14131 , TFLOPS: 71.52514528349087, Tokens per sec: 177247.1891576844, Loss: 2.5058555603027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14132 , TFLOPS: 70.03388800757706, Tokens per sec: 173551.68935241972, Loss: 2.477306365966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14133 , TFLOPS: 68.71464347056529, Tokens per sec: 170282.45606292173, Loss: 2.4865975379943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14134 , TFLOPS: 70.39276956031983, Tokens per sec: 174441.03737418295, Loss: 2.4849510192871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14135 , TFLOPS: 69.87193110904933, Tokens per sec: 173150.3423168428, Loss: 2.477789878845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14136 , TFLOPS: 68.79501335524367, Tokens per sec: 170481.62149062843, Loss: 2.513873338699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14137 , TFLOPS: 70.60742785080781, Tokens per sec: 174972.98426457494, Loss: 2.5084478855133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14138 , TFLOPS: 69.7923306995444, Tokens per sec: 172953.0837333806, Loss: 2.481794834136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14139 , TFLOPS: 67.90435143207947, Tokens per sec: 168274.46313051714, Loss: 2.5090761184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14140 , TFLOPS: 70.65652848630214, Tokens per sec: 175094.66104821136, Loss: 2.4894561767578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14141 , TFLOPS: 70.42140697642031, Tokens per sec: 174512.00404595176, Loss: 2.4981985092163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14142 , TFLOPS: 69.76067666857367, Tokens per sec: 172874.64155765573, Loss: 2.4903855323791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14143 , TFLOPS: 68.05671606787864, Tokens per sec: 168652.0394823777, Loss: 2.4960227012634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14144 , TFLOPS: 71.49260928739517, Tokens per sec: 177166.56137522342, Loss: 2.510150671005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14145 , TFLOPS: 68.85352627701079, Tokens per sec: 170626.62295649783, Loss: 2.4919989109039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14146 , TFLOPS: 68.81694010474904, Tokens per sec: 170535.95839133358, Loss: 2.4946117401123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14147 , TFLOPS: 70.84813779435943, Tokens per sec: 175569.49002108513, Loss: 2.4838664531707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14148 , TFLOPS: 70.40392641259818, Tokens per sec: 174468.68528315745, Loss: 2.481050729751587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14149 , TFLOPS: 69.78364928697162, Tokens per sec: 172931.5702366887, Loss: 2.494298219680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14150 , TFLOPS: 69.17955305117067, Tokens per sec: 171434.55321768002, Loss: 2.47959041595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14151 , TFLOPS: 71.5259711942866, Tokens per sec: 177249.2358556178, Loss: 2.5014231204986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14152 , TFLOPS: 68.82812564835092, Tokens per sec: 170563.6774005684, Loss: 2.4788668155670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14153 , TFLOPS: 68.72573635706293, Tokens per sec: 170309.9454576452, Loss: 2.4564461708068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14154 , TFLOPS: 71.5270156115556, Tokens per sec: 177251.8240366624, Loss: 2.4964821338653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14155 , TFLOPS: 69.75233136714446, Tokens per sec: 172853.96098140333, Loss: 2.4832770824432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14156 , TFLOPS: 67.94617496191005, Tokens per sec: 168378.10644467943, Loss: 2.480207681655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14157 , TFLOPS: 70.67856227608269, Tokens per sec: 175149.2632065099, Loss: 2.497988224029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14158 , TFLOPS: 70.669794159856, Tokens per sec: 175127.5348486124, Loss: 2.4930031299591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14159 , TFLOPS: 68.4971521942632, Tokens per sec: 169743.48872160324, Loss: 2.4971981048583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14160 , TFLOPS: 69.05618612297346, Tokens per sec: 171128.8363796183, Loss: 2.47895884513855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14161 , TFLOPS: 71.51500622027397, Tokens per sec: 177222.0634421229, Loss: 2.4795384407043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14162 , TFLOPS: 69.78492296863726, Tokens per sec: 172934.7265601638, Loss: 2.478280782699585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14163 , TFLOPS: 67.86139158187636, Tokens per sec: 168168.00388929568, Loss: 2.5005152225494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14164 , TFLOPS: 70.9726382824138, Tokens per sec: 175878.0159453448, Loss: 2.494037389755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14165 , TFLOPS: 70.71068884717592, Tokens per sec: 175228.8763887129, Loss: 2.4915733337402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14166 , TFLOPS: 68.3800643560334, Tokens per sec: 169453.33216017965, Loss: 2.4994306564331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14167 , TFLOPS: 69.61842109278822, Tokens per sec: 172522.1165701141, Loss: 2.4799108505249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14168 , TFLOPS: 71.51603246478624, Tokens per sec: 177224.60658907445, Loss: 2.4827003479003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14169 , TFLOPS: 69.18600299669052, Tokens per sec: 171450.53689319835, Loss: 2.4817702770233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14170 , TFLOPS: 68.79215588430318, Tokens per sec: 170474.5403628615, Loss: 2.4551079273223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14171 , TFLOPS: 70.58685789662748, Tokens per sec: 174922.00965214858, Loss: 2.4971728324890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14172 , TFLOPS: 69.74670383670222, Tokens per sec: 172840.01534104382, Loss: 2.5041818618774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14173 , TFLOPS: 68.99353994696403, Tokens per sec: 170973.59225442095, Loss: 2.4785215854644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14174 , TFLOPS: 71.50816447915123, Tokens per sec: 177205.10885394074, Loss: 2.5105419158935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14175 , TFLOPS: 68.69230215494872, Tokens per sec: 170227.0918217248, Loss: 2.4961330890655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14176 , TFLOPS: 68.70947990082891, Tokens per sec: 170269.66016830082, Loss: 2.4701457023620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14177 , TFLOPS: 70.60198740311102, Tokens per sec: 174959.50223586755, Loss: 2.4954729080200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14178 , TFLOPS: 69.7308681639936, Tokens per sec: 172800.7727996285, Loss: 2.5027236938476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14179 , TFLOPS: 69.80162090327829, Tokens per sec: 172976.10588163364, Loss: 2.4961800575256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14180 , TFLOPS: 69.20149884193825, Tokens per sec: 171488.93730473047, Loss: 2.4791762828826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14181 , TFLOPS: 71.51827315075035, Tokens per sec: 177230.15925572545, Loss: 2.483612537384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14182 , TFLOPS: 69.66207241443097, Tokens per sec: 172630.28935373484, Loss: 2.4954769611358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14183 , TFLOPS: 68.61820503829597, Tokens per sec: 170043.47100418803, Loss: 2.505514144897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14184 , TFLOPS: 70.61573635675822, Tokens per sec: 174993.57365757757, Loss: 2.477461099624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14185 , TFLOPS: 69.73877355440086, Tokens per sec: 172820.36322790832, Loss: 2.4925942420959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14186 , TFLOPS: 68.03834208849528, Tokens per sec: 168606.50673740485, Loss: 2.485990047454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14187 , TFLOPS: 70.74653475141169, Tokens per sec: 175317.70648816146, Loss: 2.485604763031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14188 , TFLOPS: 71.5049236196675, Tokens per sec: 177197.0776471299, Loss: 2.4840571880340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14189 , TFLOPS: 68.67875065602333, Tokens per sec: 170193.50971456684, Loss: 2.4794840812683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14190 , TFLOPS: 68.74610687956066, Tokens per sec: 170360.4258563929, Loss: 2.489753007888794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14191 , TFLOPS: 71.47296498987122, Tokens per sec: 177117.88064196103, Loss: 2.4865002632141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14192 , TFLOPS: 69.74421525276647, Tokens per sec: 172833.84835591086, Loss: 2.496992826461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14193 , TFLOPS: 68.74146727717319, Tokens per sec: 170348.92841058233, Loss: 2.472534656524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14194 , TFLOPS: 71.04205251749724, Tokens per sec: 176050.0320664908, Loss: 2.4768242835998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14195 , TFLOPS: 69.79254901417828, Tokens per sec: 172953.62474109267, Loss: 2.4918644428253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14196 , TFLOPS: 69.77605023604684, Tokens per sec: 172912.73895139256, Loss: 2.495633125305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14197 , TFLOPS: 68.88009119073469, Tokens per sec: 170692.45373907246, Loss: 2.493459463119507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14198 , TFLOPS: 70.75679980381476, Tokens per sec: 175343.14441880514, Loss: 2.46726655960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14199 , TFLOPS: 70.57915807717386, Tokens per sec: 174902.92865133737, Loss: 2.493725299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14200 , TFLOPS: 67.3377788272673, Tokens per sec: 166870.43380266783, Loss: 2.5024116039276123 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/model.pt +[rank0]:[2024-08-27 08:01:12,654] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006356514000799507, 'preprocessing_with_comm': 0.001224170991918072, 'state_converting': 1.000940336001804, : 1.0099179470125819}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012200 + eval ppl=10.64647102355957, eval loss=2.3652284145355225 +------------------------------------------------------------------ +iteration: 14201 , TFLOPS: 66.97244770596059, Tokens per sec: 165965.1030395245, Loss: 2.487231731414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14202 , TFLOPS: 69.12775290318677, Tokens per sec: 171306.18674471855, Loss: 2.497277021408081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14203 , TFLOPS: 67.51987445064681, Tokens per sec: 167321.68681689113, Loss: 2.4747374057769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14204 , TFLOPS: 71.47605799684094, Tokens per sec: 177125.54545395484, Loss: 2.4726123809814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14205 , TFLOPS: 69.75836877244467, Tokens per sec: 172868.92233681236, Loss: 2.49889874458313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14206 , TFLOPS: 68.85761856273523, Tokens per sec: 170636.76409132543, Loss: 2.465646743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14207 , TFLOPS: 69.79400755030578, Tokens per sec: 172957.23915428764, Loss: 2.493046522140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14208 , TFLOPS: 70.56039846276074, Tokens per sec: 174856.44026027955, Loss: 2.482281446456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14209 , TFLOPS: 69.6194210726233, Tokens per sec: 172524.59463030228, Loss: 2.5098326206207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14210 , TFLOPS: 69.86522595681895, Tokens per sec: 173133.72621098245, Loss: 2.5037600994110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14211 , TFLOPS: 70.39308747807407, Tokens per sec: 174441.82520939948, Loss: 2.4687674045562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14212 , TFLOPS: 69.71549185943996, Tokens per sec: 172762.6686231048, Loss: 2.487870931625366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14213 , TFLOPS: 68.63920436243889, Tokens per sec: 170095.50964268073, Loss: 2.5040690898895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14214 , TFLOPS: 71.47552514166256, Tokens per sec: 177124.22498012395, Loss: 2.487109661102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14215 , TFLOPS: 68.76901643401031, Tokens per sec: 170417.1982560149, Loss: 2.4882314205169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14216 , TFLOPS: 68.16885625399138, Tokens per sec: 168929.93521682636, Loss: 2.4707517623901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14217 , TFLOPS: 70.6282635971941, Tokens per sec: 175024.61753936714, Loss: 2.507711410522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14218 , TFLOPS: 70.43686605481061, Tokens per sec: 174550.313345161, Loss: 2.4867665767669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14219 , TFLOPS: 68.70186165417526, Tokens per sec: 170250.78131387394, Loss: 2.501918315887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14220 , TFLOPS: 68.92010695610398, Tokens per sec: 170791.61721375797, Loss: 2.4938127994537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14221 , TFLOPS: 71.5062083717779, Tokens per sec: 177200.2614043866, Loss: 2.486238479614258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14222 , TFLOPS: 68.86004401776012, Tokens per sec: 170642.77463606306, Loss: 2.4893739223480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14223 , TFLOPS: 68.27879754090998, Tokens per sec: 169202.38183684318, Loss: 2.4718525409698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14224 , TFLOPS: 71.48900003658827, Tokens per sec: 177157.61725413208, Loss: 2.4912569522857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14225 , TFLOPS: 70.55744753343983, Tokens per sec: 174849.1275323506, Loss: 2.483736038208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14226 , TFLOPS: 67.90946400429291, Tokens per sec: 168287.13264765835, Loss: 2.4915060997009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14227 , TFLOPS: 70.02313408513757, Tokens per sec: 173525.0399479723, Loss: 2.4669363498687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14228 , TFLOPS: 70.7183573638093, Tokens per sec: 175247.8798176873, Loss: 2.4833133220672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14229 , TFLOPS: 69.6277996320024, Tokens per sec: 172545.35762341818, Loss: 2.4965760707855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14230 , TFLOPS: 68.87543499173866, Tokens per sec: 170680.91516503762, Loss: 2.4769692420959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14231 , TFLOPS: 71.48195537401588, Tokens per sec: 177140.1598042441, Loss: 2.496281623840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14232 , TFLOPS: 69.71872910902387, Tokens per sec: 172770.69088418622, Loss: 2.4704275131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14233 , TFLOPS: 67.59960488633493, Tokens per sec: 167519.26761955884, Loss: 2.4884226322174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14234 , TFLOPS: 71.565277010298, Tokens per sec: 177346.63999758713, Loss: 2.472184419631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14235 , TFLOPS: 69.88223898412609, Tokens per sec: 173175.8863667892, Loss: 2.474604606628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14236 , TFLOPS: 68.92734740560238, Tokens per sec: 170809.55984521637, Loss: 2.4999046325683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14237 , TFLOPS: 69.56246677227905, Tokens per sec: 172383.45560001512, Loss: 2.500035285949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14238 , TFLOPS: 71.56118836521824, Tokens per sec: 177336.50788467802, Loss: 2.4774935245513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14239 , TFLOPS: 69.79592027231801, Tokens per sec: 172961.9790901377, Loss: 2.4948670864105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14240 , TFLOPS: 67.38355541795647, Tokens per sec: 166983.87323710334, Loss: 2.485201358795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14241 , TFLOPS: 71.53875446846114, Tokens per sec: 177280.9142172162, Loss: 2.508993625640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14242 , TFLOPS: 69.84570883805478, Tokens per sec: 173085.3606406967, Loss: 2.4972589015960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14243 , TFLOPS: 68.84446692200146, Tokens per sec: 170604.17287681435, Loss: 2.4830758571624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14244 , TFLOPS: 69.79484217834157, Tokens per sec: 172959.3074545022, Loss: 2.4893407821655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14245 , TFLOPS: 70.56756860667802, Tokens per sec: 174874.2086667631, Loss: 2.4907243251800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14246 , TFLOPS: 68.8746310871516, Tokens per sec: 170678.9230009135, Loss: 2.4845056533813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14247 , TFLOPS: 69.86652016349923, Tokens per sec: 173136.9333977048, Loss: 2.5170106887817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14248 , TFLOPS: 70.54498245329088, Tokens per sec: 174818.23769059958, Loss: 2.492872714996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14249 , TFLOPS: 68.92733657444246, Tokens per sec: 170809.533004409, Loss: 2.516922950744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14250 , TFLOPS: 69.92601535162171, Tokens per sec: 173284.36902780834, Loss: 2.5012240409851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14251 , TFLOPS: 71.52819178415193, Tokens per sec: 177254.73872192224, Loss: 2.473583936691284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14252 , TFLOPS: 68.13694728850045, Tokens per sec: 168850.8612852771, Loss: 2.502725601196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14253 , TFLOPS: 68.3163156002386, Tokens per sec: 169295.35572081493, Loss: 2.5134971141815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14254 , TFLOPS: 71.44145622878942, Tokens per sec: 177039.79846102375, Loss: 2.4872896671295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14255 , TFLOPS: 68.87477432118845, Tokens per sec: 170679.27795063533, Loss: 2.489638090133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14256 , TFLOPS: 69.58495239977313, Tokens per sec: 172439.1774619418, Loss: 2.499746799468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14257 , TFLOPS: 70.04496108983747, Tokens per sec: 173579.12967006172, Loss: 2.503364324569702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14258 , TFLOPS: 71.49412195284708, Tokens per sec: 177170.30992684708, Loss: 2.49395489692688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14259 , TFLOPS: 68.74366787920682, Tokens per sec: 170354.381744838, Loss: 2.5045528411865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14260 , TFLOPS: 68.1141960071948, Tokens per sec: 168794.48110394122, Loss: 2.5172128677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14261 , TFLOPS: 71.49539800411777, Tokens per sec: 177173.47212246453, Loss: 2.5016133785247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14262 , TFLOPS: 69.80968216707072, Tokens per sec: 172996.08258133277, Loss: 2.4889636039733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14263 , TFLOPS: 67.99649220634859, Tokens per sec: 168502.79811930022, Loss: 2.494995355606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14264 , TFLOPS: 70.66636393252551, Tokens per sec: 175119.03436741678, Loss: 2.491614580154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14265 , TFLOPS: 71.51289563991124, Tokens per sec: 177216.83319148442, Loss: 2.487447500228882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14266 , TFLOPS: 68.06122416447272, Tokens per sec: 168663.21104234568, Loss: 2.4864606857299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14267 , TFLOPS: 68.86682118547745, Tokens per sec: 170659.56920423327, Loss: 2.4818828105926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14268 , TFLOPS: 71.47293313941076, Tokens per sec: 177117.80171301137, Loss: 2.503943681716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14269 , TFLOPS: 69.76288932639852, Tokens per sec: 172880.1247674901, Loss: 2.493767023086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14270 , TFLOPS: 68.9778506831456, Tokens per sec: 170934.7125303638, Loss: 2.489315986633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14271 , TFLOPS: 71.50965815729633, Tokens per sec: 177208.81035292652, Loss: 2.4876904487609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14272 , TFLOPS: 69.66216293796197, Tokens per sec: 172630.5136810167, Loss: 2.4977657794952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14273 , TFLOPS: 69.53492996174047, Tokens per sec: 172315.21635007043, Loss: 2.476755142211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14274 , TFLOPS: 69.7463870814608, Tokens per sec: 172839.23038666212, Loss: 2.504810333251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14275 , TFLOPS: 70.7433573415736, Tokens per sec: 175309.8325165621, Loss: 2.476177215576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14276 , TFLOPS: 69.06817432415437, Tokens per sec: 171158.54446275395, Loss: 2.5003585815429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14277 , TFLOPS: 68.98904992717674, Tokens per sec: 170962.46549077152, Loss: 2.49285626411438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14278 , TFLOPS: 71.51959046782153, Tokens per sec: 177233.4237125423, Loss: 2.4687952995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14279 , TFLOPS: 69.7310070095876, Tokens per sec: 172801.11687430553, Loss: 2.4953339099884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14280 , TFLOPS: 67.76448968477861, Tokens per sec: 167927.87031366228, Loss: 2.4926702976226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14281 , TFLOPS: 70.50156265628931, Tokens per sec: 174710.6386505442, Loss: 2.5070714950561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14282 , TFLOPS: 69.74115051579315, Tokens per sec: 172826.25360008236, Loss: 2.4884305000305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14283 , TFLOPS: 68.82888082831192, Tokens per sec: 170565.54881970168, Loss: 2.49497652053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14284 , TFLOPS: 70.66421319590934, Tokens per sec: 175113.7046051587, Loss: 2.4802186489105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14285 , TFLOPS: 70.52959994691466, Tokens per sec: 174780.11814527636, Loss: 2.5016915798187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14286 , TFLOPS: 68.8679436662101, Tokens per sec: 170662.35083514024, Loss: 2.4888510704040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14287 , TFLOPS: 69.19577988604759, Tokens per sec: 171474.76510203845, Loss: 2.515407085418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14288 , TFLOPS: 71.54675063536132, Tokens per sec: 177300.72959964533, Loss: 2.4854443073272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14289 , TFLOPS: 69.56535819516915, Tokens per sec: 172390.62086445335, Loss: 2.4828779697418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14290 , TFLOPS: 68.79004961817616, Tokens per sec: 170469.32080337437, Loss: 2.482292413711548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14291 , TFLOPS: 71.46796439092867, Tokens per sec: 177105.48860691994, Loss: 2.48813796043396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14292 , TFLOPS: 69.7025930630125, Tokens per sec: 172730.70398463783, Loss: 2.506871461868286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14293 , TFLOPS: 69.22093560559321, Tokens per sec: 171537.10374620373, Loss: 2.483983039855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14294 , TFLOPS: 70.55850819426284, Tokens per sec: 174851.75596671147, Loss: 2.4939191341400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14295 , TFLOPS: 71.54794169967644, Tokens per sec: 177303.6811882248, Loss: 2.486758232116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14296 , TFLOPS: 68.03157981336128, Tokens per sec: 168589.74907469665, Loss: 2.474421977996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14297 , TFLOPS: 69.90728849001596, Tokens per sec: 173237.96180181514, Loss: 2.493968963623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14298 , TFLOPS: 71.5311736669038, Tokens per sec: 177262.12815586285, Loss: 2.471248149871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14299 , TFLOPS: 69.73075426288253, Tokens per sec: 172800.49054012803, Loss: 2.503826856613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14300 , TFLOPS: 67.10744644536086, Tokens per sec: 166299.6447870972, Loss: 2.4967257976531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14301 , TFLOPS: 71.45121932659708, Tokens per sec: 177063.99249288518, Loss: 2.483715295791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14302 , TFLOPS: 69.71885437363674, Tokens per sec: 172771.00130369593, Loss: 2.5020978450775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14303 , TFLOPS: 68.71221669009127, Tokens per sec: 170276.44223357516, Loss: 2.4836761951446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14304 , TFLOPS: 69.7528891257572, Tokens per sec: 172855.34316868792, Loss: 2.4859323501586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14305 , TFLOPS: 71.50893983180018, Tokens per sec: 177207.030263217, Loss: 2.4878885746002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14306 , TFLOPS: 69.83364515823969, Tokens per sec: 173055.46551319087, Loss: 2.504171133041382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14307 , TFLOPS: 68.02423381804586, Tokens per sec: 168571.54488907685, Loss: 2.4889707565307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14308 , TFLOPS: 71.51083947722961, Tokens per sec: 177211.73779385432, Loss: 2.491431951522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14309 , TFLOPS: 69.69768139085801, Tokens per sec: 172718.53231997363, Loss: 2.4903624057769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14310 , TFLOPS: 69.54400168251078, Tokens per sec: 172337.69707347182, Loss: 2.4698855876922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14311 , TFLOPS: 69.94022243551646, Tokens per sec: 173319.57574673902, Loss: 2.508991003036499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14312 , TFLOPS: 70.59037570498221, Tokens per sec: 174930.7271687702, Loss: 2.490476131439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14313 , TFLOPS: 68.76071166221615, Tokens per sec: 170396.61811666284, Loss: 2.5004754066467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14314 , TFLOPS: 69.48825377429498, Tokens per sec: 172199.54761577398, Loss: 2.4965527057647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14315 , TFLOPS: 71.47710169010259, Tokens per sec: 177128.1318408295, Loss: 2.5002503395080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14316 , TFLOPS: 68.08889603794186, Tokens per sec: 168731.78499311072, Loss: 2.482499599456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14317 , TFLOPS: 69.92575553719722, Tokens per sec: 173283.72517904348, Loss: 2.4910295009613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14318 , TFLOPS: 70.59504546008351, Tokens per sec: 174942.29933632718, Loss: 2.497238874435425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14319 , TFLOPS: 69.77747867271556, Tokens per sec: 172916.2787748129, Loss: 2.503159284591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14320 , TFLOPS: 67.94471511593329, Tokens per sec: 168374.48878553376, Loss: 2.4893910884857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14321 , TFLOPS: 71.55235595251537, Tokens per sec: 177314.62019302972, Loss: 2.4879579544067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14322 , TFLOPS: 68.68394173879766, Tokens per sec: 170206.3737895266, Loss: 2.494320869445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14323 , TFLOPS: 69.07858374659546, Tokens per sec: 171184.34015825554, Loss: 2.490968942642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14324 , TFLOPS: 70.26006088450846, Tokens per sec: 174112.1706564549, Loss: 2.4807662963867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14325 , TFLOPS: 71.52953894195994, Tokens per sec: 177258.07712737218, Loss: 2.5144920349121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14326 , TFLOPS: 69.66286860004924, Tokens per sec: 172632.262389404, Loss: 2.480757713317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14327 , TFLOPS: 67.9509737894529, Tokens per sec: 168389.99846796514, Loss: 2.516242742538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14328 , TFLOPS: 71.51496699064462, Tokens per sec: 177221.96622677988, Loss: 2.513190269470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14329 , TFLOPS: 69.55561706556001, Tokens per sec: 172366.48127220786, Loss: 2.477644681930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14330 , TFLOPS: 69.66004981318828, Tokens per sec: 172625.27712504755, Loss: 2.4782979488372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14331 , TFLOPS: 70.63277861427468, Tokens per sec: 175035.806249063, Loss: 2.493475914001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14332 , TFLOPS: 71.53106270934411, Tokens per sec: 177261.85319080696, Loss: 2.482464075088501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14333 , TFLOPS: 67.37277731176688, Tokens per sec: 166957.16390265911, Loss: 2.4602489471435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14334 , TFLOPS: 70.48589491051196, Tokens per sec: 174671.81225056323, Loss: 2.490049362182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14335 , TFLOPS: 71.51320106552724, Tokens per sec: 177217.5900698063, Loss: 2.50038480758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14336 , TFLOPS: 68.80599806709175, Tokens per sec: 170508.8428166539, Loss: 2.511044979095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14337 , TFLOPS: 68.37593875633458, Tokens per sec: 169443.10846965364, Loss: 2.4827637672424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14338 , TFLOPS: 71.51363154833184, Tokens per sec: 177218.65685361766, Loss: 2.488760232925415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14339 , TFLOPS: 69.73760058963396, Tokens per sec: 172817.4564920032, Loss: 2.4940848350524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14340 , TFLOPS: 68.75903322075216, Tokens per sec: 170392.45875381946, Loss: 2.4752771854400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14341 , TFLOPS: 69.85059503229954, Tokens per sec: 173097.46916829393, Loss: 2.473651647567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14342 , TFLOPS: 70.60444041762784, Tokens per sec: 174965.58107606205, Loss: 2.5160841941833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14343 , TFLOPS: 69.11025944096745, Tokens per sec: 171262.83601827512, Loss: 2.4848670959472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14344 , TFLOPS: 69.81341367469973, Tokens per sec: 173005.32966829694, Loss: 2.485743522644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14345 , TFLOPS: 71.54457862561716, Tokens per sec: 177295.3471202327, Loss: 2.4690191745758057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14346 , TFLOPS: 69.19950551991792, Tokens per sec: 171483.99763318154, Loss: 2.484766960144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14347 , TFLOPS: 67.97660897480489, Tokens per sec: 168453.52528121616, Loss: 2.4824368953704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14348 , TFLOPS: 70.41607974553426, Tokens per sec: 174498.80258097322, Loss: 2.4793193340301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14349 , TFLOPS: 69.6756141171662, Tokens per sec: 172663.84718485447, Loss: 2.4768471717834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14350 , TFLOPS: 69.59755002421875, Tokens per sec: 172470.39576306092, Loss: 2.507096767425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14351 , TFLOPS: 69.68696088075464, Tokens per sec: 172691.9657149758, Loss: 2.4811434745788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14352 , TFLOPS: 71.51992891539363, Tokens per sec: 177234.26242290906, Loss: 2.478217363357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14353 , TFLOPS: 68.78193157309144, Tokens per sec: 170449.20339337795, Loss: 2.5216403007507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14354 , TFLOPS: 69.91760038353502, Tokens per sec: 173263.51580990426, Loss: 2.4940741062164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14355 , TFLOPS: 70.11173852246104, Tokens per sec: 173744.61150424439, Loss: 2.5027453899383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14356 , TFLOPS: 69.71929448317104, Tokens per sec: 172772.091943604, Loss: 2.510894775390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14357 , TFLOPS: 68.30378984620188, Tokens per sec: 169264.31552248623, Loss: 2.4811129570007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14358 , TFLOPS: 71.50685174548786, Tokens per sec: 177201.8557553134, Loss: 2.491041898727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14359 , TFLOPS: 69.71950171862883, Tokens per sec: 172772.60549589727, Loss: 2.485318660736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14360 , TFLOPS: 69.08091458288263, Tokens per sec: 171190.11622733795, Loss: 2.5194976329803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14361 , TFLOPS: 70.14375968266984, Tokens per sec: 173823.9634666638, Loss: 2.4723012447357178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14362 , TFLOPS: 70.61190418188504, Tokens per sec: 174984.07710609288, Loss: 2.4989209175109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14363 , TFLOPS: 67.76203224282987, Tokens per sec: 167921.7805018037, Loss: 2.478496789932251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14364 , TFLOPS: 69.82673907179436, Tokens per sec: 173038.3514702111, Loss: 2.493134021759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14365 , TFLOPS: 71.52430200865278, Tokens per sec: 177245.0994297412, Loss: 2.496715545654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14366 , TFLOPS: 68.45146004096809, Tokens per sec: 169630.25853233176, Loss: 2.4905965328216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14367 , TFLOPS: 68.09176869962384, Tokens per sec: 168738.90376520844, Loss: 2.5055956840515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14368 , TFLOPS: 71.43598449693684, Tokens per sec: 177026.23890673203, Loss: 2.5030224323272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14369 , TFLOPS: 69.76408936005726, Tokens per sec: 172883.09858309108, Loss: 2.491431713104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14370 , TFLOPS: 69.36326582931422, Tokens per sec: 171889.81371955288, Loss: 2.4928648471832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14371 , TFLOPS: 70.08463597575697, Tokens per sec: 173677.44840791973, Loss: 2.5121312141418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14372 , TFLOPS: 71.4722089133486, Tokens per sec: 177116.00700104926, Loss: 2.4836723804473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14373 , TFLOPS: 69.58291163099561, Tokens per sec: 172434.12021210187, Loss: 2.5080013275146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14374 , TFLOPS: 67.85710088936183, Tokens per sec: 168157.37108058587, Loss: 2.4830563068389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14375 , TFLOPS: 71.4619440424455, Tokens per sec: 177090.5695201825, Loss: 2.5207629203796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14376 , TFLOPS: 69.6497875301056, Tokens per sec: 172599.84605708488, Loss: 2.4914169311523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14377 , TFLOPS: 68.82370823556163, Tokens per sec: 170552.73056505836, Loss: 2.500253915786743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14378 , TFLOPS: 70.23392130583802, Tokens per sec: 174047.39390099724, Loss: 2.507126569747925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14379 , TFLOPS: 70.71908720296028, Tokens per sec: 175249.68843950183, Loss: 2.483271837234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14380 , TFLOPS: 68.80944660757369, Tokens per sec: 170517.3886798562, Loss: 2.473698854446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14381 , TFLOPS: 68.96014496385467, Tokens per sec: 170890.8358075149, Loss: 2.4815876483917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14382 , TFLOPS: 71.56340447764258, Tokens per sec: 177341.99965539045, Loss: 2.503797769546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14383 , TFLOPS: 68.86211418656981, Tokens per sec: 170647.9047424214, Loss: 2.4945781230926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14384 , TFLOPS: 69.02811246581679, Tokens per sec: 171059.26676461546, Loss: 2.4968366622924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14385 , TFLOPS: 70.45417612480787, Tokens per sec: 174593.2096054758, Loss: 2.510286808013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14386 , TFLOPS: 69.76324487741684, Tokens per sec: 172881.00586208067, Loss: 2.4720494747161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14387 , TFLOPS: 68.18991736955064, Tokens per sec: 168982.1269812556, Loss: 2.4769232273101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14388 , TFLOPS: 70.38648655623467, Tokens per sec: 174425.4673979299, Loss: 2.484408378601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14389 , TFLOPS: 70.61187280218748, Tokens per sec: 174983.99934374556, Loss: 2.461516857147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14390 , TFLOPS: 69.83777049954892, Tokens per sec: 173065.68856339902, Loss: 2.489497661590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14391 , TFLOPS: 69.68678266780691, Tokens per sec: 172691.52408365955, Loss: 2.4972898960113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14392 , TFLOPS: 70.01251230491822, Tokens per sec: 173498.71800650307, Loss: 2.488426685333252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14393 , TFLOPS: 69.73459704645613, Tokens per sec: 172810.01338114098, Loss: 2.4957354068756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14394 , TFLOPS: 68.6743592458411, Tokens per sec: 170182.62731638152, Loss: 2.487156391143799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14395 , TFLOPS: 71.48524238300986, Tokens per sec: 177148.30537462508, Loss: 2.4779133796691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14396 , TFLOPS: 68.85389122670485, Tokens per sec: 170627.5273420422, Loss: 2.4972620010375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14397 , TFLOPS: 68.86720740816753, Tokens per sec: 170660.5263066052, Loss: 2.4806113243103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14398 , TFLOPS: 70.64465802541135, Tokens per sec: 175065.2447384848, Loss: 2.5186538696289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14399 , TFLOPS: 71.40934236011584, Tokens per sec: 176960.21675681596, Loss: 2.515375852584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14400 , TFLOPS: 67.3230337371342, Tokens per sec: 166833.89384501224, Loss: 2.50498628616333 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/optimizer.pt +[rank0]:[2024-08-27 08:26:37,433] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063700280006742105, 'preprocessing_with_comm': 0.001207908004289493, 'state_converting': 0.9779591230035294, : 0.9869007829984184}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400, took 4.44s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012400 + eval ppl=10.468620300292969, eval loss=2.3483822345733643 +------------------------------------------------------------------ +iteration: 14401 , TFLOPS: 65.13015464154195, Tokens per sec: 161399.69788056047, Loss: 2.4644291400909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14402 , TFLOPS: 71.52891536697558, Tokens per sec: 177256.53183986846, Loss: 2.5019891262054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14403 , TFLOPS: 69.10320054514928, Tokens per sec: 171245.34329683613, Loss: 2.4937944412231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14404 , TFLOPS: 67.85871278065233, Tokens per sec: 168161.3655247682, Loss: 2.4721450805664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14405 , TFLOPS: 71.52180171473725, Tokens per sec: 177238.90342598842, Loss: 2.480117082595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14406 , TFLOPS: 70.57609383826507, Tokens per sec: 174895.3351297676, Loss: 2.5033936500549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14407 , TFLOPS: 66.99672219800823, Tokens per sec: 166025.2579048746, Loss: 2.4827561378479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14408 , TFLOPS: 71.48737112868402, Tokens per sec: 177153.58064090632, Loss: 2.4717581272125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14409 , TFLOPS: 70.41786182780572, Tokens per sec: 174503.21877715446, Loss: 2.5041098594665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14410 , TFLOPS: 68.9162419951236, Tokens per sec: 170782.039414688, Loss: 2.489091157913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14411 , TFLOPS: 69.55867583520381, Tokens per sec: 172374.06124033648, Loss: 2.4890618324279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14412 , TFLOPS: 71.51615155816302, Tokens per sec: 177224.9017155813, Loss: 2.492499589920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14413 , TFLOPS: 69.53286795340546, Tokens per sec: 172310.1064662673, Loss: 2.4944450855255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14414 , TFLOPS: 68.57908445758436, Tokens per sec: 169946.52589569692, Loss: 2.495506763458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14415 , TFLOPS: 71.49216974165017, Tokens per sec: 177165.47213244732, Loss: 2.4751336574554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14416 , TFLOPS: 70.58834144515201, Tokens per sec: 174925.6860488187, Loss: 2.4887468814849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14417 , TFLOPS: 68.41042961996126, Tokens per sec: 169528.58062920283, Loss: 2.4792771339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14418 , TFLOPS: 70.37405171071075, Tokens per sec: 174394.65248091027, Loss: 2.466327667236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14419 , TFLOPS: 71.50138532639436, Tokens per sec: 177188.3093666239, Loss: 2.4865946769714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14420 , TFLOPS: 68.95308438702132, Tokens per sec: 170873.33892033508, Loss: 2.506880760192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14421 , TFLOPS: 70.10303232116401, Tokens per sec: 173723.0365783631, Loss: 2.49403977394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14422 , TFLOPS: 70.22193479177648, Tokens per sec: 174017.68999872924, Loss: 2.499617099761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14423 , TFLOPS: 69.6473089172195, Tokens per sec: 172593.70378131137, Loss: 2.496616840362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14424 , TFLOPS: 69.38229129130609, Tokens per sec: 171936.9609101964, Loss: 2.4859213829040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14425 , TFLOPS: 71.52609644700121, Tokens per sec: 177249.5462456423, Loss: 2.482819080352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14426 , TFLOPS: 69.55554656592427, Tokens per sec: 172366.30656634434, Loss: 2.4838106632232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14427 , TFLOPS: 67.7458637907358, Tokens per sec: 167881.71329641255, Loss: 2.4946508407592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14428 , TFLOPS: 71.48228618850825, Tokens per sec: 177140.9795989984, Loss: 2.4649956226348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14429 , TFLOPS: 69.59656286799407, Tokens per sec: 172467.94948119193, Loss: 2.494127035140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14430 , TFLOPS: 69.64131714770475, Tokens per sec: 172578.85551640592, Loss: 2.4941582679748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14431 , TFLOPS: 70.15125077582384, Tokens per sec: 173842.5272207114, Loss: 2.475641965866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14432 , TFLOPS: 71.41321726196514, Tokens per sec: 176969.8191904542, Loss: 2.4793660640716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14433 , TFLOPS: 69.51359954524983, Tokens per sec: 172262.3572282673, Loss: 2.4980289936065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14434 , TFLOPS: 68.91856138633241, Tokens per sec: 170787.78712160548, Loss: 2.485015869140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14435 , TFLOPS: 71.5740296995078, Tokens per sec: 177368.33012563732, Loss: 2.493774652481079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14436 , TFLOPS: 70.54786912128263, Tokens per sec: 174825.39117187585, Loss: 2.4866933822631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14437 , TFLOPS: 68.84380779429175, Tokens per sec: 170602.53948574074, Loss: 2.5245509147644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14438 , TFLOPS: 70.97388508233205, Tokens per sec: 175881.10565288857, Loss: 2.512162208557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14439 , TFLOPS: 71.53086742085858, Tokens per sec: 177261.36924442704, Loss: 2.4844510555267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14440 , TFLOPS: 69.22730719530745, Tokens per sec: 171552.89324740416, Loss: 2.483196973800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14441 , TFLOPS: 68.73325559647833, Tokens per sec: 170328.57896122918, Loss: 2.506574869155884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14442 , TFLOPS: 71.53219758250181, Tokens per sec: 177264.66553150836, Loss: 2.4818053245544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14443 , TFLOPS: 69.62190078610591, Tokens per sec: 172530.7396334752, Loss: 2.4911818504333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14444 , TFLOPS: 67.57901325525384, Tokens per sec: 167468.23928938445, Loss: 2.5053446292877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14445 , TFLOPS: 71.48316959777382, Tokens per sec: 177143.16878447402, Loss: 2.4856393337249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14446 , TFLOPS: 70.49909827280493, Tokens per sec: 174704.53163679564, Loss: 2.5014398097991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14447 , TFLOPS: 68.0756419281205, Tokens per sec: 168698.93984891265, Loss: 2.482665538787842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14448 , TFLOPS: 70.53564050602083, Tokens per sec: 174795.08731616393, Loss: 2.507119655609131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14449 , TFLOPS: 71.55591195189484, Tokens per sec: 177323.43235121775, Loss: 2.4809744358062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14450 , TFLOPS: 69.00545867391462, Tokens per sec: 171003.12817276068, Loss: 2.5056135654449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14451 , TFLOPS: 68.67798197356692, Tokens per sec: 170191.60483476246, Loss: 2.5148072242736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14452 , TFLOPS: 71.54144796771925, Tokens per sec: 177287.58900509123, Loss: 2.478869676589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14453 , TFLOPS: 70.58221946743826, Tokens per sec: 174910.5151136525, Loss: 2.4960622787475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14454 , TFLOPS: 68.81341567434427, Tokens per sec: 170527.22446454252, Loss: 2.4877941608428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14455 , TFLOPS: 70.48794780108346, Tokens per sec: 174676.89953954399, Loss: 2.489980936050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14456 , TFLOPS: 70.57859809087378, Tokens per sec: 174901.5409435982, Loss: 2.4979248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14457 , TFLOPS: 68.52884409167645, Tokens per sec: 169822.024734544, Loss: 2.491594076156616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14458 , TFLOPS: 70.6933688288696, Tokens per sec: 175185.95547539231, Loss: 2.4871010780334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14459 , TFLOPS: 70.3492646869432, Tokens per sec: 174333.22750549825, Loss: 2.4926223754882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14460 , TFLOPS: 69.67484498312513, Tokens per sec: 172661.94118597358, Loss: 2.4816274642944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14461 , TFLOPS: 68.93524643805736, Tokens per sec: 170829.13451779008, Loss: 2.5013084411621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14462 , TFLOPS: 71.54888439011614, Tokens per sec: 177306.01727898032, Loss: 2.511955738067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14463 , TFLOPS: 68.81326875113928, Tokens per sec: 170526.86037265562, Loss: 2.4961533546447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14464 , TFLOPS: 69.1363196294334, Tokens per sec: 171327.4160360605, Loss: 2.4886317253112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14465 , TFLOPS: 71.50467423853583, Tokens per sec: 177196.45965321406, Loss: 2.4853246212005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14466 , TFLOPS: 69.14295483579325, Tokens per sec: 171343.85880834854, Loss: 2.473417282104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14467 , TFLOPS: 68.83928001041856, Tokens per sec: 170591.31913852642, Loss: 2.4792988300323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14468 , TFLOPS: 70.96675865281604, Tokens per sec: 175863.4455755069, Loss: 2.4880125522613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14469 , TFLOPS: 71.53898737386267, Tokens per sec: 177281.49138245784, Loss: 2.501953363418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14470 , TFLOPS: 67.80037415394996, Tokens per sec: 168016.79598126948, Loss: 2.502197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14471 , TFLOPS: 69.7982979747863, Tokens per sec: 172967.87129877965, Loss: 2.4800937175750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14472 , TFLOPS: 71.5124724299719, Tokens per sec: 177215.7844306343, Loss: 2.4825241565704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14473 , TFLOPS: 70.61306385267252, Tokens per sec: 174986.95089805257, Loss: 2.47741436958313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14474 , TFLOPS: 67.51781305719233, Tokens per sec: 167316.57845682968, Loss: 2.4831321239471436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14475 , TFLOPS: 71.51639047978432, Tokens per sec: 177225.49378967832, Loss: 2.5030863285064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14476 , TFLOPS: 70.53131004526958, Tokens per sec: 174784.3559573827, Loss: 2.493030071258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14477 , TFLOPS: 68.7671651902162, Tokens per sec: 170412.61066996175, Loss: 2.485191822052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14478 , TFLOPS: 69.73395383891821, Tokens per sec: 172808.41944200685, Loss: 2.5010125637054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14479 , TFLOPS: 71.52815479173466, Tokens per sec: 177254.64705063723, Loss: 2.4993653297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14480 , TFLOPS: 69.7125866578584, Tokens per sec: 172755.46921355132, Loss: 2.4962611198425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14481 , TFLOPS: 67.88795985809094, Tokens per sec: 168233.84300449264, Loss: 2.482996940612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14482 , TFLOPS: 71.50338947395834, Tokens per sec: 177193.2758650625, Loss: 2.503117322921753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14483 , TFLOPS: 70.59455262836188, Tokens per sec: 174941.0780450311, Loss: 2.489771842956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14484 , TFLOPS: 68.36269946532545, Tokens per sec: 169410.3000481047, Loss: 2.487959384918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14485 , TFLOPS: 70.54032775884433, Tokens per sec: 174806.7028450041, Loss: 2.4764561653137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14486 , TFLOPS: 71.50098743625054, Tokens per sec: 177187.32335101627, Loss: 2.5204644203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14487 , TFLOPS: 69.0705587582825, Tokens per sec: 171164.4533531906, Loss: 2.4954733848571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14488 , TFLOPS: 69.06088195497068, Tokens per sec: 171140.473168596, Loss: 2.4876511096954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14489 , TFLOPS: 71.48852537939757, Tokens per sec: 177156.44100132573, Loss: 2.472898244857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14490 , TFLOPS: 69.5354105479428, Tokens per sec: 172316.40729562074, Loss: 2.4878647327423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14491 , TFLOPS: 69.31848482259441, Tokens per sec: 171778.84145186606, Loss: 2.4620842933654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14492 , TFLOPS: 70.52840186385414, Tokens per sec: 174777.1491634726, Loss: 2.50311279296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14493 , TFLOPS: 70.65417290072548, Tokens per sec: 175088.8236476633, Loss: 2.50459361076355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14494 , TFLOPS: 68.69991484063391, Tokens per sec: 170245.95689545927, Loss: 2.50053071975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14495 , TFLOPS: 71.52389872293632, Tokens per sec: 177244.10004331023, Loss: 2.5120503902435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14496 , TFLOPS: 69.46544905975078, Tokens per sec: 172143.03502098678, Loss: 2.495328426361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14497 , TFLOPS: 68.75577766960666, Tokens per sec: 170384.39113945235, Loss: 2.486607551574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14498 , TFLOPS: 70.68701104357531, Tokens per sec: 175170.20018306523, Loss: 2.502044916152954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14499 , TFLOPS: 71.49874793949509, Tokens per sec: 177181.77363135535, Loss: 2.4897282123565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14500 , TFLOPS: 69.666687779672, Tokens per sec: 172641.72673722662, Loss: 2.4874775409698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14501 , TFLOPS: 68.34610034046504, Tokens per sec: 169369.1655881561, Loss: 2.4712746143341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14502 , TFLOPS: 71.55771043305447, Tokens per sec: 177327.88918565036, Loss: 2.499847888946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14503 , TFLOPS: 69.54745776989529, Tokens per sec: 172346.26163873007, Loss: 2.47534441947937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14504 , TFLOPS: 68.81924817392449, Tokens per sec: 170541.6780410052, Loss: 2.5066261291503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14505 , TFLOPS: 70.90018342235199, Tokens per sec: 175698.46482054988, Loss: 2.515460729598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14506 , TFLOPS: 71.50441866860415, Tokens per sec: 177195.82632276998, Loss: 2.491502523422241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14507 , TFLOPS: 69.50800463728515, Tokens per sec: 172248.4924300008, Loss: 2.511837959289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14508 , TFLOPS: 69.1964939476657, Tokens per sec: 171476.5346253885, Loss: 2.495389461517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14509 , TFLOPS: 71.55906643095078, Tokens per sec: 177331.2494978129, Loss: 2.4785327911376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14510 , TFLOPS: 69.56219257924866, Tokens per sec: 172382.776119481, Loss: 2.4869484901428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14511 , TFLOPS: 68.82378871126879, Tokens per sec: 170552.92999272584, Loss: 2.4841325283050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14512 , TFLOPS: 71.50255391599156, Tokens per sec: 177191.20526037653, Loss: 2.5040154457092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14513 , TFLOPS: 70.63268737876764, Tokens per sec: 175035.5801574261, Loss: 2.4855165481567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14514 , TFLOPS: 68.5877795172567, Tokens per sec: 169968.0732114049, Loss: 2.4846434593200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14515 , TFLOPS: 70.54572469835573, Tokens per sec: 174820.0770556353, Loss: 2.491671562194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14516 , TFLOPS: 70.55501347589932, Tokens per sec: 174843.09566963167, Loss: 2.50317645072937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14517 , TFLOPS: 69.72252344561659, Tokens per sec: 172780.09366824367, Loss: 2.4804511070251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14518 , TFLOPS: 68.92120633980946, Tokens per sec: 170794.3416076873, Loss: 2.4817678928375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14519 , TFLOPS: 71.54290910318547, Tokens per sec: 177291.20985973388, Loss: 2.4890058040618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14520 , TFLOPS: 70.63173177592674, Tokens per sec: 175033.212068318, Loss: 2.5031819343566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14521 , TFLOPS: 67.85455243934615, Tokens per sec: 168151.05574071204, Loss: 2.4704275131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14522 , TFLOPS: 70.44604390515259, Tokens per sec: 174573.05706933208, Loss: 2.486713409423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14523 , TFLOPS: 70.62320851049667, Tokens per sec: 175012.0904776671, Loss: 2.5054073333740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14524 , TFLOPS: 68.89033732106562, Tokens per sec: 170717.84477873694, Loss: 2.461167812347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14525 , TFLOPS: 69.58932258604924, Tokens per sec: 172450.00726494976, Loss: 2.502882719039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14526 , TFLOPS: 71.46091053363075, Tokens per sec: 177088.00837148915, Loss: 2.4999938011169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14527 , TFLOPS: 69.67139162673907, Tokens per sec: 172653.3833884303, Loss: 2.472618579864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14528 , TFLOPS: 68.97155595353237, Tokens per sec: 170919.11349696052, Loss: 2.498253345489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14529 , TFLOPS: 70.48905783111923, Tokens per sec: 174679.65031625252, Loss: 2.49782395362854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14530 , TFLOPS: 70.60292814431305, Tokens per sec: 174961.83349619733, Loss: 2.5007333755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14531 , TFLOPS: 68.33780084994704, Tokens per sec: 169348.59853638808, Loss: 2.494638681411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14532 , TFLOPS: 71.49116878888536, Tokens per sec: 177162.99166123208, Loss: 2.4898464679718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14533 , TFLOPS: 69.54496789044437, Tokens per sec: 172340.09144316782, Loss: 2.488548755645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14534 , TFLOPS: 69.69701338461465, Tokens per sec: 172716.87692691563, Loss: 2.5087084770202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14535 , TFLOPS: 70.80210625869373, Tokens per sec: 175455.41880491326, Loss: 2.50270938873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14536 , TFLOPS: 71.52627861601219, Tokens per sec: 177249.99768051913, Loss: 2.5135507583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14537 , TFLOPS: 67.7765526417404, Tokens per sec: 167957.76364985798, Loss: 2.4951720237731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14538 , TFLOPS: 69.26085252391287, Tokens per sec: 171636.02226701146, Loss: 2.48288631439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14539 , TFLOPS: 71.52857750119594, Tokens per sec: 177255.69457124756, Loss: 2.4803924560546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14540 , TFLOPS: 68.60064398085952, Tokens per sec: 169999.95276935032, Loss: 2.49153733253479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14541 , TFLOPS: 68.10337753057739, Tokens per sec: 168767.67172712967, Loss: 2.500309944152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14542 , TFLOPS: 71.50500533080145, Tokens per sec: 177197.28013632106, Loss: 2.466261863708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14543 , TFLOPS: 70.58189835226266, Tokens per sec: 174909.71935487364, Loss: 2.5050737857818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14544 , TFLOPS: 68.43652451188014, Tokens per sec: 169593.24664596748, Loss: 2.5015530586242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14545 , TFLOPS: 70.55956130785638, Tokens per sec: 174854.3656982063, Loss: 2.4873292446136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14546 , TFLOPS: 71.5254774543221, Tokens per sec: 177248.01231359583, Loss: 2.486156463623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14547 , TFLOPS: 69.64443683354595, Tokens per sec: 172586.58644158219, Loss: 2.5041675567626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14548 , TFLOPS: 68.10439235073495, Tokens per sec: 168770.18656327177, Loss: 2.464852809906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14549 , TFLOPS: 71.49543056008268, Tokens per sec: 177173.55279973187, Loss: 2.500619649887085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14550 , TFLOPS: 70.59051399440767, Tokens per sec: 174931.06986520032, Loss: 2.4912309646606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14551 , TFLOPS: 68.17521980589706, Tokens per sec: 168945.70479945175, Loss: 2.496650218963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14552 , TFLOPS: 70.32656309270855, Tokens per sec: 174276.97045419773, Loss: 2.511375904083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14553 , TFLOPS: 71.49076244021948, Tokens per sec: 177161.98468447503, Loss: 2.499591827392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14554 , TFLOPS: 68.91525571753198, Tokens per sec: 170779.59531016854, Loss: 2.521005868911743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14555 , TFLOPS: 69.52656351061457, Tokens per sec: 172294.48336253976, Loss: 2.492405652999878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14556 , TFLOPS: 71.5016421879454, Tokens per sec: 177188.945897843, Loss: 2.477832794189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14557 , TFLOPS: 69.60906602864337, Tokens per sec: 172498.93369061776, Loss: 2.4945755004882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14558 , TFLOPS: 69.02522909208042, Tokens per sec: 171052.1214468669, Loss: 2.4965460300445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14559 , TFLOPS: 70.43115757077874, Tokens per sec: 174536.16709288856, Loss: 2.5014331340789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14560 , TFLOPS: 70.61484082194691, Tokens per sec: 174991.3544236638, Loss: 2.485372543334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14561 , TFLOPS: 68.07993234168319, Tokens per sec: 168709.57196634906, Loss: 2.5020627975463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14562 , TFLOPS: 70.49592090606346, Tokens per sec: 174696.65777199442, Loss: 2.468651533126831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14563 , TFLOPS: 70.66137241498679, Tokens per sec: 175106.66483709475, Loss: 2.489461660385132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14564 , TFLOPS: 68.90102850746878, Tokens per sec: 170744.33871637538, Loss: 2.5077433586120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14565 , TFLOPS: 70.60713242953946, Tokens per sec: 174972.25217812866, Loss: 2.4725804328918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14566 , TFLOPS: 70.48996421867777, Tokens per sec: 174681.89644446943, Loss: 2.499113082885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14567 , TFLOPS: 70.58313304223034, Tokens per sec: 174912.7790526259, Loss: 2.506221294403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14568 , TFLOPS: 68.31199410776505, Tokens per sec: 169284.64658641376, Loss: 2.4896678924560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14569 , TFLOPS: 71.55925482258804, Tokens per sec: 177331.7163530431, Loss: 2.4741148948669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14570 , TFLOPS: 69.57077630951713, Tokens per sec: 172404.04754866217, Loss: 2.483612298965454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14571 , TFLOPS: 68.37453033853043, Tokens per sec: 169439.61825518493, Loss: 2.497889995574951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14572 , TFLOPS: 70.8465100781953, Tokens per sec: 175565.45636112266, Loss: 2.4673752784729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14573 , TFLOPS: 71.55621345095855, Tokens per sec: 177324.17949911047, Loss: 2.484097957611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14574 , TFLOPS: 68.60433194687243, Tokens per sec: 170009.09195539335, Loss: 2.483642101287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14575 , TFLOPS: 69.90637814871943, Tokens per sec: 173235.70587579996, Loss: 2.5111494064331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14576 , TFLOPS: 71.49200682258004, Tokens per sec: 177165.06840104453, Loss: 2.4797937870025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14577 , TFLOPS: 68.72718363686292, Tokens per sec: 170313.5319764206, Loss: 2.480738401412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14578 , TFLOPS: 68.97774760438472, Tokens per sec: 170934.45708983927, Loss: 2.4923226833343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14579 , TFLOPS: 71.465737612311, Tokens per sec: 177099.97040420544, Loss: 2.5063588619232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14580 , TFLOPS: 70.58324553879388, Tokens per sec: 174913.0578315028, Loss: 2.4824025630950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14581 , TFLOPS: 67.96210512463742, Tokens per sec: 168417.5831427702, Loss: 2.490692138671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14582 , TFLOPS: 71.48495516803864, Tokens per sec: 177147.59362428705, Loss: 2.510486125946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14583 , TFLOPS: 70.71209664810837, Tokens per sec: 175232.3650745056, Loss: 2.4969844818115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14584 , TFLOPS: 69.56250603676867, Tokens per sec: 172383.55290174572, Loss: 2.482382297515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14585 , TFLOPS: 68.92954227267649, Tokens per sec: 170814.9989676106, Loss: 2.487008810043335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14586 , TFLOPS: 71.4933241664354, Tokens per sec: 177168.3329242355, Loss: 2.496910333633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14587 , TFLOPS: 70.60130859715719, Tokens per sec: 174957.82007993726, Loss: 2.4936130046844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14588 , TFLOPS: 68.21394581585506, Tokens per sec: 169041.6721181498, Loss: 2.4861531257629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14589 , TFLOPS: 70.52721133761197, Tokens per sec: 174774.19890829726, Loss: 2.512357234954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14590 , TFLOPS: 70.6967076342673, Tokens per sec: 175194.22940296758, Loss: 2.4866158962249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14591 , TFLOPS: 68.68867903004382, Tokens per sec: 170218.1133190903, Loss: 2.482311487197876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14592 , TFLOPS: 69.60442550733302, Tokens per sec: 172487.43396761507, Loss: 2.490825653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14593 , TFLOPS: 71.51645336948357, Tokens per sec: 177225.64963728093, Loss: 2.4755098819732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14594 , TFLOPS: 69.49659384792085, Tokens per sec: 172220.21523695384, Loss: 2.4762673377990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14595 , TFLOPS: 69.83918416304493, Tokens per sec: 173069.19177727003, Loss: 2.473228931427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14596 , TFLOPS: 70.48591146412933, Tokens per sec: 174671.8532722506, Loss: 2.4858720302581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14597 , TFLOPS: 69.6964367561557, Tokens per sec: 172715.4479780735, Loss: 2.4833898544311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14598 , TFLOPS: 68.86714755970765, Tokens per sec: 170660.37799552878, Loss: 2.4900600910186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14599 , TFLOPS: 70.38564686022974, Tokens per sec: 174423.3865387295, Loss: 2.4952797889709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14600 , TFLOPS: 70.60089009183982, Tokens per sec: 174956.78297765861, Loss: 2.4969730377197266 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/optimizer.pt +[rank0]:[2024-08-27 08:51:58,646] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006352793003316037, 'preprocessing_with_comm': 0.0012274070031708106, 'state_converting': 0.9909422550117597, : 0.9998875669989502}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012600 + eval ppl=10.071361541748047, eval loss=2.3096959590911865 +------------------------------------------------------------------ +iteration: 14601 , TFLOPS: 65.3327602722169, Tokens per sec: 161901.7769522257, Loss: 2.4878125190734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14602 , TFLOPS: 71.4796250416853, Tokens per sec: 177134.38498402157, Loss: 2.476611375808716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14603 , TFLOPS: 70.11646015168562, Tokens per sec: 173756.31222159328, Loss: 2.4865775108337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14604 , TFLOPS: 69.39527041133137, Tokens per sec: 171969.12459938714, Loss: 2.4929380416870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14605 , TFLOPS: 70.08400037791586, Tokens per sec: 173675.87332645254, Loss: 2.5202431678771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14606 , TFLOPS: 70.4332448387261, Tokens per sec: 174541.3395727937, Loss: 2.4879472255706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14607 , TFLOPS: 70.56860858680844, Tokens per sec: 174876.78585208958, Loss: 2.515421152114868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14608 , TFLOPS: 68.10823666819715, Tokens per sec: 168779.7132054296, Loss: 2.4827845096588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14609 , TFLOPS: 71.40165900900132, Tokens per sec: 176941.17656636317, Loss: 2.4937171936035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14610 , TFLOPS: 69.2869623940871, Tokens per sec: 171700.7254015428, Loss: 2.5050673484802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14611 , TFLOPS: 69.68380876835879, Tokens per sec: 172684.15443322505, Loss: 2.4668331146240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14612 , TFLOPS: 70.56615229589487, Tokens per sec: 174870.69889262295, Loss: 2.485938787460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14613 , TFLOPS: 71.52272215047948, Tokens per sec: 177241.18436715202, Loss: 2.494576930999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14614 , TFLOPS: 68.51222899643683, Tokens per sec: 169780.85069823638, Loss: 2.5065994262695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14615 , TFLOPS: 69.71756381414542, Tokens per sec: 172767.8031551096, Loss: 2.491333484649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14616 , TFLOPS: 71.52533714913753, Tokens per sec: 177247.66462189256, Loss: 2.486523389816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14617 , TFLOPS: 69.65525128234034, Tokens per sec: 172613.38583700455, Loss: 2.496623992919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14618 , TFLOPS: 69.55712879787252, Tokens per sec: 172370.22751140932, Loss: 2.4835383892059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14619 , TFLOPS: 71.53564493686754, Tokens per sec: 177273.20845538407, Loss: 2.465200424194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14620 , TFLOPS: 70.60233050642306, Tokens per sec: 174960.3524836707, Loss: 2.4837841987609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14621 , TFLOPS: 67.7911946343544, Tokens per sec: 167994.0481205041, Loss: 2.4788978099823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14622 , TFLOPS: 71.51450676804689, Tokens per sec: 177220.825744485, Loss: 2.4951202869415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14623 , TFLOPS: 71.48777236166364, Tokens per sec: 177154.57494042933, Loss: 2.4777982234954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14624 , TFLOPS: 68.851760861978, Tokens per sec: 170622.24806357, Loss: 2.4902706146240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14625 , TFLOPS: 69.60261168638849, Tokens per sec: 172482.93911950567, Loss: 2.5093350410461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14626 , TFLOPS: 71.50887143076717, Tokens per sec: 177206.8607579222, Loss: 2.5156519412994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14627 , TFLOPS: 70.58537160602762, Tokens per sec: 174918.3264603137, Loss: 2.4915056228637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14628 , TFLOPS: 68.69994241328766, Tokens per sec: 170246.0252235326, Loss: 2.503887414932251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14629 , TFLOPS: 70.40362320799879, Tokens per sec: 174467.9339087594, Loss: 2.4921910762786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14630 , TFLOPS: 70.63840244418869, Tokens per sec: 175049.74271910588, Loss: 2.4754889011383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14631 , TFLOPS: 69.51351337763553, Tokens per sec: 172262.1436954269, Loss: 2.4924702644348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14632 , TFLOPS: 70.66119548310047, Tokens per sec: 175106.2263803898, Loss: 2.4859068393707275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14633 , TFLOPS: 70.53084106689256, Tokens per sec: 174783.19377730225, Loss: 2.495042324066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14634 , TFLOPS: 69.73617840551826, Tokens per sec: 172813.932163098, Loss: 2.490032196044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14635 , TFLOPS: 69.78549970998867, Tokens per sec: 172936.1557887659, Loss: 2.483166217803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14636 , TFLOPS: 70.31633507239454, Tokens per sec: 174251.62429315012, Loss: 2.4906234741210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14637 , TFLOPS: 70.54214745324575, Tokens per sec: 174811.2122481867, Loss: 2.495899200439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14638 , TFLOPS: 68.6248366868843, Tokens per sec: 170059.90495992632, Loss: 2.507737159729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14639 , TFLOPS: 71.50764649212375, Tokens per sec: 177203.82522502568, Loss: 2.510467767715454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14640 , TFLOPS: 70.40143676777545, Tokens per sec: 174462.5156690297, Loss: 2.4846556186676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14641 , TFLOPS: 69.15808577384462, Tokens per sec: 171381.35493965022, Loss: 2.4699950218200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14642 , TFLOPS: 70.91218596384306, Tokens per sec: 175728.20844055386, Loss: 2.4962966442108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14643 , TFLOPS: 70.40179221043478, Tokens per sec: 174463.39649509478, Loss: 2.4861936569213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14644 , TFLOPS: 69.5551302237771, Tokens per sec: 172365.27482463975, Loss: 2.4769179821014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14645 , TFLOPS: 70.13698118733832, Tokens per sec: 173807.1656085194, Loss: 2.489250898361206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14646 , TFLOPS: 71.51681646053767, Tokens per sec: 177226.54941691077, Loss: 2.483703136444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14647 , TFLOPS: 69.71112322505577, Tokens per sec: 172751.842665856, Loss: 2.4812703132629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14648 , TFLOPS: 68.7009908251584, Tokens per sec: 170248.62330364043, Loss: 2.4878549575805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14649 , TFLOPS: 71.50146524301651, Tokens per sec: 177188.50740881712, Loss: 2.495180606842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14650 , TFLOPS: 70.60248027274416, Tokens per sec: 174960.7236211125, Loss: 2.4919674396514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14651 , TFLOPS: 68.61101468075508, Tokens per sec: 170025.65250611847, Loss: 2.5157463550567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14652 , TFLOPS: 70.68419648528409, Tokens per sec: 175163.22539757038, Loss: 2.4617464542388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14653 , TFLOPS: 71.52041750640447, Tokens per sec: 177235.47320525665, Loss: 2.46026873588562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14654 , TFLOPS: 70.66574483772825, Tokens per sec: 175117.50018231024, Loss: 2.4827890396118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14655 , TFLOPS: 68.7593423572274, Tokens per sec: 170393.2248280593, Loss: 2.484205722808838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14656 , TFLOPS: 71.456365843306, Tokens per sec: 177076.74612822547, Loss: 2.492733955383301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14657 , TFLOPS: 70.48930865319436, Tokens per sec: 174680.27188098503, Loss: 2.5028138160705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14658 , TFLOPS: 68.58441100673222, Tokens per sec: 169959.7256712532, Loss: 2.488595724105835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14659 , TFLOPS: 70.98575720735897, Tokens per sec: 175910.52608652783, Loss: 2.485440492630005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14660 , TFLOPS: 71.4760242521718, Tokens per sec: 177125.4618309474, Loss: 2.488719940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14661 , TFLOPS: 69.60535717498726, Tokens per sec: 172489.74274269395, Loss: 2.49322509765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14662 , TFLOPS: 68.99868697921845, Tokens per sec: 170986.34716733982, Loss: 2.4849607944488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14663 , TFLOPS: 71.55326817180118, Tokens per sec: 177316.8807729092, Loss: 2.4886293411254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14664 , TFLOPS: 69.6132682265095, Tokens per sec: 172509.34719984035, Loss: 2.476217746734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14665 , TFLOPS: 69.80352274094709, Tokens per sec: 172980.8188448816, Loss: 2.4804251194000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14666 , TFLOPS: 70.44476225773836, Tokens per sec: 174569.8810058545, Loss: 2.490891218185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14667 , TFLOPS: 70.72200584111289, Tokens per sec: 175256.9211463585, Loss: 2.483689069747925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14668 , TFLOPS: 68.85789708067544, Tokens per sec: 170637.45428946256, Loss: 2.465407133102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14669 , TFLOPS: 71.44803702408645, Tokens per sec: 177056.10639670424, Loss: 2.4976396560668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14670 , TFLOPS: 69.49859555514374, Tokens per sec: 172225.17568795843, Loss: 2.4960684776306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14671 , TFLOPS: 69.58214499029695, Tokens per sec: 172432.22039199815, Loss: 2.482834577560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14672 , TFLOPS: 70.70821124810409, Tokens per sec: 175222.73662528285, Loss: 2.474252462387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14673 , TFLOPS: 70.48138253186536, Tokens per sec: 174660.63007919744, Loss: 2.506362199783325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14674 , TFLOPS: 70.69039786663372, Tokens per sec: 175178.59310369333, Loss: 2.4829277992248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14675 , TFLOPS: 68.75394843301912, Tokens per sec: 170379.8580896814, Loss: 2.467376232147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14676 , TFLOPS: 71.50795003780706, Tokens per sec: 177204.57744466726, Loss: 2.495638608932495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14677 , TFLOPS: 69.52657580920501, Tokens per sec: 172294.5138398017, Loss: 2.494804859161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14678 , TFLOPS: 69.64660227841944, Tokens per sec: 172591.9526525221, Loss: 2.4983577728271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14679 , TFLOPS: 70.66642211965451, Tokens per sec: 175119.1785615323, Loss: 2.493663787841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14680 , TFLOPS: 70.44957901181104, Tokens per sec: 174581.81745305588, Loss: 2.5174832344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14681 , TFLOPS: 69.68374204422453, Tokens per sec: 172683.98908347008, Loss: 2.472088575363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14682 , TFLOPS: 69.81870364488421, Tokens per sec: 173018.4387971524, Loss: 2.502476453781128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14683 , TFLOPS: 71.56428691372332, Tokens per sec: 177344.18642920692, Loss: 2.486477851867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14684 , TFLOPS: 69.53024518874601, Tokens per sec: 172303.6069665198, Loss: 2.4913864135742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14685 , TFLOPS: 69.11602602106376, Tokens per sec: 171277.126238995, Loss: 2.5011298656463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14686 , TFLOPS: 71.5490580017792, Tokens per sec: 177306.44750780624, Loss: 2.48042368888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14687 , TFLOPS: 70.7175460787063, Tokens per sec: 175245.86936383162, Loss: 2.4793152809143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14688 , TFLOPS: 68.68416429452893, Tokens per sec: 170206.9253071452, Loss: 2.485529899597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14689 , TFLOPS: 70.88817986529611, Tokens per sec: 175668.71868386446, Loss: 2.4804604053497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14690 , TFLOPS: 71.51279441801091, Tokens per sec: 177216.5823524649, Loss: 2.473893404006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14691 , TFLOPS: 69.83929049571256, Tokens per sec: 173069.4552813339, Loss: 2.4923572540283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14692 , TFLOPS: 69.12086606949707, Tokens per sec: 171289.1204121881, Loss: 2.5042409896850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14693 , TFLOPS: 71.5256241377698, Tokens per sec: 177248.3758113377, Loss: 2.491610527038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14694 , TFLOPS: 70.66082024285498, Tokens per sec: 175105.29649372536, Loss: 2.477919340133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14695 , TFLOPS: 67.80936545587045, Tokens per sec: 168039.07741790212, Loss: 2.4779109954833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14696 , TFLOPS: 71.54506623857989, Tokens per sec: 177296.55547886933, Loss: 2.4960780143737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14697 , TFLOPS: 70.62230613505312, Tokens per sec: 175009.85429191316, Loss: 2.486457109451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14698 , TFLOPS: 69.6946001911348, Tokens per sec: 172710.89676763737, Loss: 2.4945249557495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14699 , TFLOPS: 69.61026525160571, Tokens per sec: 172501.90549722375, Loss: 2.5008163452148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14700 , TFLOPS: 71.52382836077355, Tokens per sec: 177243.92567811985, Loss: 2.4938204288482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14701 , TFLOPS: 69.65433279443602, Tokens per sec: 172611.10972279785, Loss: 2.474351406097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14702 , TFLOPS: 69.64744375616678, Tokens per sec: 172594.03792707645, Loss: 2.4769740104675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14703 , TFLOPS: 70.50071448581917, Tokens per sec: 174708.5367906855, Loss: 2.4550983905792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14704 , TFLOPS: 70.56081194608845, Tokens per sec: 174857.46491751453, Loss: 2.4742918014526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14705 , TFLOPS: 68.90023049703133, Tokens per sec: 170742.36115860325, Loss: 2.5013580322265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14706 , TFLOPS: 71.56247254776972, Tokens per sec: 177339.690230505, Loss: 2.525207042694092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14707 , TFLOPS: 70.44708117833883, Tokens per sec: 174575.62754655266, Loss: 2.481523036956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14708 , TFLOPS: 69.06119859500136, Tokens per sec: 171141.2578374728, Loss: 2.465195655822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14709 , TFLOPS: 70.6189332748442, Tokens per sec: 175001.49597276418, Loss: 2.4744675159454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14710 , TFLOPS: 70.50069313749012, Tokens per sec: 174708.48388717437, Loss: 2.510254144668579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14711 , TFLOPS: 69.62121514313668, Tokens per sec: 172529.04053466764, Loss: 2.467099666595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14712 , TFLOPS: 69.79646419925223, Tokens per sec: 172963.32700099918, Loss: 2.4930877685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14713 , TFLOPS: 71.4576993502224, Tokens per sec: 177080.05070526223, Loss: 2.4804394245147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14714 , TFLOPS: 69.57410507452069, Tokens per sec: 172412.2965950347, Loss: 2.4829087257385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14715 , TFLOPS: 68.87303299256803, Tokens per sec: 170674.96274649072, Loss: 2.4915943145751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14716 , TFLOPS: 71.53304562457677, Tokens per sec: 177266.76707318967, Loss: 2.491922616958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14717 , TFLOPS: 69.34790885141733, Tokens per sec: 171851.75743661364, Loss: 2.48664927482605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14718 , TFLOPS: 69.59199458693416, Tokens per sec: 172456.6287774878, Loss: 2.477024793624878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14719 , TFLOPS: 70.56182768037223, Tokens per sec: 174859.98201896212, Loss: 2.4929378032684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14720 , TFLOPS: 71.52946170723844, Tokens per sec: 177257.88573122417, Loss: 2.4643983840942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14721 , TFLOPS: 69.67957252834273, Tokens per sec: 172673.65656380492, Loss: 2.487994909286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14722 , TFLOPS: 69.22016015060032, Tokens per sec: 171535.18208330797, Loss: 2.525787830352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14723 , TFLOPS: 71.47593576575262, Tokens per sec: 177125.24255185312, Loss: 2.485152244567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14724 , TFLOPS: 70.54160010356968, Tokens per sec: 174809.85585539386, Loss: 2.490675687789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14725 , TFLOPS: 68.75113123554048, Tokens per sec: 170372.87676398942, Loss: 2.479491710662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14726 , TFLOPS: 70.91165992546136, Tokens per sec: 175726.9048594961, Loss: 2.4995882511138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14727 , TFLOPS: 71.53176810364359, Tokens per sec: 177263.60123558668, Loss: 2.499452590942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14728 , TFLOPS: 69.54430186740235, Tokens per sec: 172338.44096470118, Loss: 2.4969944953918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14729 , TFLOPS: 69.16223469932675, Tokens per sec: 171391.6364340368, Loss: 2.48447322845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14730 , TFLOPS: 71.56218486670458, Tokens per sec: 177338.97732513488, Loss: 2.470275640487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14731 , TFLOPS: 69.46131688915546, Tokens per sec: 172132.79504705538, Loss: 2.4726815223693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14732 , TFLOPS: 69.5706917280607, Tokens per sec: 172403.8379464957, Loss: 2.473599910736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14733 , TFLOPS: 71.54004236981449, Tokens per sec: 177284.10577864383, Loss: 2.52592134475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14734 , TFLOPS: 70.55235002225287, Tokens per sec: 174836.49533809212, Loss: 2.488804817199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14735 , TFLOPS: 68.75223523448298, Tokens per sec: 170375.61259498444, Loss: 2.5116617679595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14736 , TFLOPS: 70.58264426320225, Tokens per sec: 174911.56780435084, Loss: 2.4783663749694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14737 , TFLOPS: 70.7143019361641, Tokens per sec: 175237.83002124025, Loss: 2.4942874908447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14738 , TFLOPS: 69.68712002477399, Tokens per sec: 172692.36009138694, Loss: 2.498819351196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14739 , TFLOPS: 70.67338856126179, Tokens per sec: 175136.442171252, Loss: 2.5203821659088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14740 , TFLOPS: 70.4684991882395, Tokens per sec: 174628.70373447513, Loss: 2.4876859188079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14741 , TFLOPS: 70.5915189504103, Tokens per sec: 174933.56025687983, Loss: 2.499643325805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14742 , TFLOPS: 68.77921957147268, Tokens per sec: 170442.48275461502, Loss: 2.505826234817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14743 , TFLOPS: 71.53245084777018, Tokens per sec: 177265.29315074286, Loss: 2.504225730895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14744 , TFLOPS: 69.58749674711208, Tokens per sec: 172445.48263493073, Loss: 2.5113041400909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14745 , TFLOPS: 69.63089649446073, Tokens per sec: 172553.03198973753, Loss: 2.500664234161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14746 , TFLOPS: 69.96905925815909, Tokens per sec: 173391.03656988568, Loss: 2.4777588844299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14747 , TFLOPS: 70.49739955494566, Tokens per sec: 174700.32202681116, Loss: 2.480872392654419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14748 , TFLOPS: 69.57608693702535, Tokens per sec: 172417.20786864145, Loss: 2.4829869270324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14749 , TFLOPS: 69.79753152204749, Tokens per sec: 172965.97194446117, Loss: 2.496509313583374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14750 , TFLOPS: 71.52769344287059, Tokens per sec: 177253.50377733022, Loss: 2.490842580795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14751 , TFLOPS: 68.7525604876621, Tokens per sec: 170376.41860819186, Loss: 2.4904518127441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14752 , TFLOPS: 69.78918862435911, Tokens per sec: 172945.29732494333, Loss: 2.484711170196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14753 , TFLOPS: 71.51917366134657, Tokens per sec: 177232.3908201823, Loss: 2.48081636428833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14754 , TFLOPS: 70.3904456505771, Tokens per sec: 174435.27846984068, Loss: 2.48079514503479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14755 , TFLOPS: 68.83810773270396, Tokens per sec: 170588.41410521264, Loss: 2.481985569000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14756 , TFLOPS: 71.52881390649881, Tokens per sec: 177256.28040963024, Loss: 2.4663288593292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14757 , TFLOPS: 71.559916832018, Tokens per sec: 177333.3568853369, Loss: 2.50700306892395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14758 , TFLOPS: 68.8417150762812, Tokens per sec: 170597.3534999787, Loss: 2.5022389888763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14759 , TFLOPS: 70.64210769442394, Tokens per sec: 175058.92473735588, Loss: 2.4949839115142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14760 , TFLOPS: 71.54385208658935, Tokens per sec: 177293.5466764869, Loss: 2.4828672409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14761 , TFLOPS: 70.60232045115475, Tokens per sec: 174960.32756560814, Loss: 2.4750823974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14762 , TFLOPS: 67.24359771648334, Tokens per sec: 166637.04263523934, Loss: 2.4871809482574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14763 , TFLOPS: 71.50540763096434, Tokens per sec: 177198.2770804417, Loss: 2.4881198406219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14764 , TFLOPS: 70.22901160418677, Tokens per sec: 174035.22711945715, Loss: 2.5057506561279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14765 , TFLOPS: 69.4627658059757, Tokens per sec: 172136.38562254733, Loss: 2.4867727756500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14766 , TFLOPS: 70.55268171846645, Tokens per sec: 174837.3173178487, Loss: 2.488269805908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14767 , TFLOPS: 71.50661732036234, Tokens per sec: 177201.27482402825, Loss: 2.506521224975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14768 , TFLOPS: 69.5465031225475, Tokens per sec: 172343.8959174392, Loss: 2.492568016052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14769 , TFLOPS: 68.9354332413419, Tokens per sec: 170829.59743690726, Loss: 2.4886960983276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14770 , TFLOPS: 71.52197897278097, Tokens per sec: 177239.34269094735, Loss: 2.497319459915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14771 , TFLOPS: 70.66566137384493, Tokens per sec: 175117.29334961306, Loss: 2.5005526542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14772 , TFLOPS: 68.89609183167005, Tokens per sec: 170732.10508992636, Loss: 2.4733686447143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14773 , TFLOPS: 70.39536625982858, Tokens per sec: 174447.4722816157, Loss: 2.474705457687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14774 , TFLOPS: 71.52688410339714, Tokens per sec: 177251.49814495892, Loss: 2.4802017211914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14775 , TFLOPS: 68.89686845245933, Tokens per sec: 170734.02964179407, Loss: 2.482428550720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14776 , TFLOPS: 71.01028340101794, Tokens per sec: 175971.30469619803, Loss: 2.460787534713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14777 , TFLOPS: 70.4587174735767, Tokens per sec: 174604.46356799584, Loss: 2.4751596450805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14778 , TFLOPS: 69.68883719352178, Tokens per sec: 172696.6154247057, Loss: 2.474360942840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14779 , TFLOPS: 69.98559029563548, Tokens per sec: 173432.00230179628, Loss: 2.492138624191284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14780 , TFLOPS: 71.37352279697062, Tokens per sec: 176871.45193349148, Loss: 2.4817588329315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14781 , TFLOPS: 69.60409590513154, Tokens per sec: 172486.61717705117, Loss: 2.4930965900421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14782 , TFLOPS: 68.89940320293175, Tokens per sec: 170740.31103269054, Loss: 2.4959192276000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14783 , TFLOPS: 71.5325625728996, Tokens per sec: 177265.5700179211, Loss: 2.497709274291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14784 , TFLOPS: 69.74417611351137, Tokens per sec: 172833.7513645252, Loss: 2.4867165088653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14785 , TFLOPS: 69.5270110293143, Tokens per sec: 172295.59236317597, Loss: 2.4780426025390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14786 , TFLOPS: 70.6794180432139, Tokens per sec: 175151.3838917316, Loss: 2.486844301223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14787 , TFLOPS: 71.51064399976855, Tokens per sec: 177211.25337917218, Loss: 2.4831643104553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14788 , TFLOPS: 69.50497437523772, Tokens per sec: 172240.9831068364, Loss: 2.5139124393463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14789 , TFLOPS: 69.15316910023772, Tokens per sec: 171369.17088083606, Loss: 2.4997854232788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14790 , TFLOPS: 71.53312434576938, Tokens per sec: 177266.96215297678, Loss: 2.4891905784606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14791 , TFLOPS: 69.73137919900519, Tokens per sec: 172802.03920068237, Loss: 2.482962131500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14792 , TFLOPS: 69.66193570390281, Tokens per sec: 172629.95056998616, Loss: 2.4951155185699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14793 , TFLOPS: 70.86497560537818, Tokens per sec: 175611.2159716276, Loss: 2.5035250186920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14794 , TFLOPS: 71.50083649906865, Tokens per sec: 177186.9493120525, Loss: 2.530027151107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14795 , TFLOPS: 68.51450818553162, Tokens per sec: 169786.49877988666, Loss: 2.5034070014953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14796 , TFLOPS: 69.98285058524034, Tokens per sec: 173425.21299763315, Loss: 2.5016376972198486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14797 , TFLOPS: 71.52392566198714, Tokens per sec: 177244.1668012457, Loss: 2.52268385887146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14798 , TFLOPS: 69.70026793094951, Tokens per sec: 172724.94205125127, Loss: 2.4949352741241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14799 , TFLOPS: 68.88388134890836, Tokens per sec: 170701.8461685472, Loss: 2.4937901496887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14800 , TFLOPS: 71.49662442511182, Tokens per sec: 177176.5113287892, Loss: 2.494046211242676 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/optimizer.pt +[rank0]:[2024-08-27 09:17:14,352] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006448349988204427, 'preprocessing_with_comm': 0.0012639680062420666, 'state_converting': 0.958334063005168, : 0.967411400997662}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800, took 4.26s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0012800 + eval ppl=10.024728775024414, eval loss=2.3050549030303955 +------------------------------------------------------------------ +iteration: 14801 , TFLOPS: 66.13945285834636, Tokens per sec: 163900.8500451788, Loss: 2.4989571571350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14802 , TFLOPS: 68.03667177211348, Tokens per sec: 168602.3675094102, Loss: 2.4848339557647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14803 , TFLOPS: 70.05523077038822, Tokens per sec: 173604.57906976517, Loss: 2.502763032913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14804 , TFLOPS: 71.56992789308633, Tokens per sec: 177358.16539747364, Loss: 2.503291130065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14805 , TFLOPS: 68.76244090336377, Tokens per sec: 170400.90336671827, Loss: 2.4860551357269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14806 , TFLOPS: 70.53194903731541, Tokens per sec: 174785.9394500631, Loss: 2.5058329105377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14807 , TFLOPS: 70.38470890624319, Tokens per sec: 174421.06218542668, Loss: 2.4860548973083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14808 , TFLOPS: 70.60448775966653, Tokens per sec: 174965.69839484905, Loss: 2.487147808074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14809 , TFLOPS: 69.47104739193689, Tokens per sec: 172156.90830485118, Loss: 2.494368076324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14810 , TFLOPS: 70.42990409452739, Tokens per sec: 174533.0608406559, Loss: 2.4969851970672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14811 , TFLOPS: 70.63276911570154, Tokens per sec: 175035.7827105524, Loss: 2.492004156112671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14812 , TFLOPS: 69.69224396050755, Tokens per sec: 172705.0577685828, Loss: 2.4899988174438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14813 , TFLOPS: 70.86447874146269, Tokens per sec: 175609.98468811094, Loss: 2.4875588417053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14814 , TFLOPS: 70.49261443212616, Tokens per sec: 174688.46396533964, Loss: 2.479241132736206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14815 , TFLOPS: 70.57377027362638, Tokens per sec: 174889.57708063116, Loss: 2.4953293800354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14816 , TFLOPS: 69.79751084584446, Tokens per sec: 172965.92070655237, Loss: 2.497347831726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14817 , TFLOPS: 70.43361139721715, Tokens per sec: 174542.24794511424, Loss: 2.496858835220337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14818 , TFLOPS: 70.68641063191197, Tokens per sec: 175168.71229682266, Loss: 2.483907699584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14819 , TFLOPS: 68.92151942162596, Tokens per sec: 170795.11745891738, Loss: 2.4911890029907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14820 , TFLOPS: 71.53753380285232, Tokens per sec: 177277.88927337012, Loss: 2.470381021499634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14821 , TFLOPS: 70.42817609712554, Tokens per sec: 174528.7786727395, Loss: 2.4843454360961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14822 , TFLOPS: 67.99726823170812, Tokens per sec: 168504.72119562738, Loss: 2.488373279571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14823 , TFLOPS: 71.48099756699933, Tokens per sec: 177137.7862529459, Loss: 2.4965999126434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14824 , TFLOPS: 71.52212444367983, Tokens per sec: 177239.70318385964, Loss: 2.484811782836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14825 , TFLOPS: 68.69918338330889, Tokens per sec: 170244.14426363117, Loss: 2.472660541534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14826 , TFLOPS: 69.81823859857084, Tokens per sec: 173017.2863611588, Loss: 2.5035502910614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14827 , TFLOPS: 71.50520248703583, Tokens per sec: 177197.7687111884, Loss: 2.5015487670898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14828 , TFLOPS: 70.5114265248064, Tokens per sec: 174735.08240332437, Loss: 2.4922823905944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14829 , TFLOPS: 67.84798217033992, Tokens per sec: 168134.7738903397, Loss: 2.491058826446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14830 , TFLOPS: 71.46270942235634, Tokens per sec: 177092.46621591513, Loss: 2.486980676651001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14831 , TFLOPS: 71.51864318728964, Tokens per sec: 177231.07624703273, Loss: 2.481079578399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14832 , TFLOPS: 68.93397034723297, Tokens per sec: 170825.97222415428, Loss: 2.5156922340393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14833 , TFLOPS: 70.66164203447725, Tokens per sec: 175107.33298389317, Loss: 2.503370761871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14834 , TFLOPS: 71.51367476594389, Tokens per sec: 177218.76395162116, Loss: 2.458425760269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14835 , TFLOPS: 70.68258650370983, Tokens per sec: 175159.23568587526, Loss: 2.492187023162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14836 , TFLOPS: 68.36879282005174, Tokens per sec: 169425.40005235487, Loss: 2.474663257598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14837 , TFLOPS: 71.50680517509444, Tokens per sec: 177201.74034874828, Loss: 2.484226942062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14838 , TFLOPS: 70.60385047498856, Tokens per sec: 174964.11913321435, Loss: 2.483856678009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14839 , TFLOPS: 68.79503691017752, Tokens per sec: 170481.6798623493, Loss: 2.5069503784179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14840 , TFLOPS: 70.44274884720038, Tokens per sec: 174564.89155274592, Loss: 2.4869844913482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14841 , TFLOPS: 71.5054295340638, Tokens per sec: 177198.331358735, Loss: 2.4887125492095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14842 , TFLOPS: 68.74862053250364, Tokens per sec: 170366.6549652873, Loss: 2.4923369884490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14843 , TFLOPS: 69.99450034670045, Tokens per sec: 173454.0823898584, Loss: 2.4999589920043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14844 , TFLOPS: 71.48945176431057, Tokens per sec: 177158.73668518986, Loss: 2.505653142929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14845 , TFLOPS: 70.621199412475, Tokens per sec: 175007.11171144896, Loss: 2.493975877761841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14846 , TFLOPS: 68.84189583198, Tokens per sec: 170597.8014325122, Loss: 2.5075435638427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14847 , TFLOPS: 70.53235615475153, Tokens per sec: 174786.94833191746, Loss: 2.509322166442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14848 , TFLOPS: 71.4906285168024, Tokens per sec: 177161.65280749463, Loss: 2.484933614730835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14849 , TFLOPS: 68.97145147401825, Tokens per sec: 170918.85458521516, Loss: 2.474310874938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14850 , TFLOPS: 71.51656124817197, Tokens per sec: 177225.91697255455, Loss: 2.4773178100585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14851 , TFLOPS: 70.44126275218095, Tokens per sec: 174561.2088455809, Loss: 2.508298397064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14852 , TFLOPS: 68.6667004254773, Tokens per sec: 170163.6479158313, Loss: 2.5041627883911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14853 , TFLOPS: 70.72659304665679, Tokens per sec: 175268.28874701873, Loss: 2.495283842086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14854 , TFLOPS: 70.48488518978606, Tokens per sec: 174669.31005137454, Loss: 2.4931981563568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14855 , TFLOPS: 70.66990887602883, Tokens per sec: 175127.81912792576, Loss: 2.482991933822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14856 , TFLOPS: 69.62137626665698, Tokens per sec: 172529.43981650015, Loss: 2.4718034267425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14857 , TFLOPS: 71.46792888852482, Tokens per sec: 177105.4006280523, Loss: 2.5038466453552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14858 , TFLOPS: 69.54761685303663, Tokens per sec: 172346.65586427876, Loss: 2.513930320739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14859 , TFLOPS: 68.73996393621479, Tokens per sec: 170345.20296608077, Loss: 2.49859619140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14860 , TFLOPS: 71.46560474745776, Tokens per sec: 177099.64115046285, Loss: 2.4817347526550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14861 , TFLOPS: 71.49817675167496, Tokens per sec: 177180.35816501555, Loss: 2.4966814517974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14862 , TFLOPS: 68.83126111760474, Tokens per sec: 170571.44743877984, Loss: 2.4738810062408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14863 , TFLOPS: 69.95311874162589, Tokens per sec: 173351.53421392356, Loss: 2.5165982246398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14864 , TFLOPS: 71.50589415950672, Tokens per sec: 177199.4827517653, Loss: 2.4796648025512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14865 , TFLOPS: 70.55113936747274, Tokens per sec: 174833.49520218233, Loss: 2.4774317741394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14866 , TFLOPS: 68.69134953033526, Tokens per sec: 170224.7311129924, Loss: 2.4885153770446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14867 , TFLOPS: 71.52167338515063, Tokens per sec: 177238.58541113613, Loss: 2.483522891998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14868 , TFLOPS: 71.50144983139546, Tokens per sec: 177188.46921712242, Loss: 2.496365547180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14869 , TFLOPS: 68.04764124802965, Tokens per sec: 168629.55107911798, Loss: 2.496814727783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14870 , TFLOPS: 71.50227958877453, Tokens per sec: 177190.52544731306, Loss: 2.5109760761260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14871 , TFLOPS: 71.55535462745388, Tokens per sec: 177322.0512398587, Loss: 2.491316795349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14872 , TFLOPS: 68.8376709000368, Tokens per sec: 170587.3315857423, Loss: 2.478602647781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14873 , TFLOPS: 69.74762753734518, Tokens per sec: 172842.3043729911, Loss: 2.4921865463256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14874 , TFLOPS: 71.45311301348141, Tokens per sec: 177068.68525759137, Loss: 2.5032970905303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14875 , TFLOPS: 70.64215196695875, Tokens per sec: 175059.03444957416, Loss: 2.4994590282440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14876 , TFLOPS: 69.56180534935677, Tokens per sec: 172381.81652115198, Loss: 2.494776487350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14877 , TFLOPS: 70.38112722904317, Tokens per sec: 174412.18639477072, Loss: 2.5023326873779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14878 , TFLOPS: 70.58682776267631, Tokens per sec: 174921.93497689802, Loss: 2.47841477394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14879 , TFLOPS: 69.65859393765982, Tokens per sec: 172621.6693051101, Loss: 2.4615511894226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14880 , TFLOPS: 70.99137697063739, Tokens per sec: 175924.45247899977, Loss: 2.4896042346954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14881 , TFLOPS: 69.75145963722812, Tokens per sec: 172851.80073864167, Loss: 2.4954493045806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14882 , TFLOPS: 70.71144838132848, Tokens per sec: 175230.75859801253, Loss: 2.489769458770752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14883 , TFLOPS: 68.98430021988095, Tokens per sec: 170950.6951928691, Loss: 2.50295352935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14884 , TFLOPS: 70.4473574327403, Tokens per sec: 174576.31213539143, Loss: 2.497274875640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14885 , TFLOPS: 70.55788827027433, Tokens per sec: 174850.21972677766, Loss: 2.487833261489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14886 , TFLOPS: 68.81631161203367, Tokens per sec: 170534.40091715075, Loss: 2.4775023460388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14887 , TFLOPS: 71.51432889985408, Tokens per sec: 177220.3849675094, Loss: 2.4729604721069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14888 , TFLOPS: 70.12824160128399, Tokens per sec: 173785.50795153185, Loss: 2.5114951133728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14889 , TFLOPS: 68.66821474263153, Tokens per sec: 170167.4005605553, Loss: 2.5049655437469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14890 , TFLOPS: 71.46964320067812, Tokens per sec: 177109.64888241523, Loss: 2.502821683883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14891 , TFLOPS: 70.33074333757118, Tokens per sec: 174287.3295614593, Loss: 2.469130754470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14892 , TFLOPS: 68.11128795416498, Tokens per sec: 168787.2746281848, Loss: 2.4651994705200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14893 , TFLOPS: 70.67702539894012, Tokens per sec: 175145.45465564964, Loss: 2.500148296356201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14894 , TFLOPS: 71.48184219505691, Tokens per sec: 177139.87933431604, Loss: 2.4993345737457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14895 , TFLOPS: 69.66515212275634, Tokens per sec: 172637.921210223, Loss: 2.4899203777313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14896 , TFLOPS: 69.55987384346652, Tokens per sec: 172377.030036783, Loss: 2.4825761318206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14897 , TFLOPS: 71.49470159144235, Tokens per sec: 177171.74633513862, Loss: 2.488565683364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14898 , TFLOPS: 71.53359069405035, Tokens per sec: 177268.11781538953, Loss: 2.4816763401031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14899 , TFLOPS: 67.37892061246608, Tokens per sec: 166972.38767853216, Loss: 2.487276077270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14900 , TFLOPS: 70.76515203440871, Tokens per sec: 175363.84216629114, Loss: 2.491570234298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14901 , TFLOPS: 71.51130557364344, Tokens per sec: 177212.89283211256, Loss: 2.4848713874816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14902 , TFLOPS: 70.6229824810706, Tokens per sec: 175011.5303518501, Loss: 2.4980568885803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14903 , TFLOPS: 68.54178818088141, Tokens per sec: 169854.101613503, Loss: 2.4982218742370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14904 , TFLOPS: 71.5084749059212, Tokens per sec: 177205.87812567307, Loss: 2.4905149936676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14905 , TFLOPS: 70.51283739027315, Tokens per sec: 174738.57868337064, Loss: 2.458329677581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14906 , TFLOPS: 67.96980336206241, Tokens per sec: 168436.66022314015, Loss: 2.480487823486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14907 , TFLOPS: 71.55110120073005, Tokens per sec: 177311.5107798834, Loss: 2.4974253177642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14908 , TFLOPS: 71.50216514791565, Tokens per sec: 177190.24185025805, Loss: 2.4765148162841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14909 , TFLOPS: 68.7657099559305, Tokens per sec: 170409.00443909448, Loss: 2.4655957221984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14910 , TFLOPS: 70.4537665059637, Tokens per sec: 174592.19452485672, Loss: 2.4587438106536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14911 , TFLOPS: 71.46366496447826, Tokens per sec: 177094.8341545549, Loss: 2.4824178218841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14912 , TFLOPS: 69.78596889190712, Tokens per sec: 172937.31847324444, Loss: 2.4894399642944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14913 , TFLOPS: 69.96609813797012, Tokens per sec: 173383.69858786312, Loss: 2.4972946643829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14914 , TFLOPS: 70.5570213955058, Tokens per sec: 174848.0715156071, Loss: 2.497976541519165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14915 , TFLOPS: 71.49809821360499, Tokens per sec: 177180.1635390265, Loss: 2.510918140411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14916 , TFLOPS: 69.10804381295287, Tokens per sec: 171257.34544798342, Loss: 2.4741554260253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14917 , TFLOPS: 70.49123460653419, Tokens per sec: 174685.04460552242, Loss: 2.5020501613616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14918 , TFLOPS: 71.49202111020448, Tokens per sec: 177165.1038073518, Loss: 2.4859848022460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14919 , TFLOPS: 68.72190485035901, Tokens per sec: 170300.4505619576, Loss: 2.50095534324646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14920 , TFLOPS: 70.72696254196374, Tokens per sec: 175269.2043970926, Loss: 2.4929633140563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14921 , TFLOPS: 70.53789915832452, Tokens per sec: 174800.6845053836, Loss: 2.4855992794036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14922 , TFLOPS: 70.69508143128378, Tokens per sec: 175190.1994928334, Loss: 2.462230920791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14923 , TFLOPS: 68.87682521447279, Tokens per sec: 170684.36029011846, Loss: 2.4839999675750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14924 , TFLOPS: 71.4481370310498, Tokens per sec: 177056.3542249761, Loss: 2.5118820667266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14925 , TFLOPS: 69.59934435848282, Tokens per sec: 172474.84232102957, Loss: 2.490203857421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14926 , TFLOPS: 68.7233528251594, Tokens per sec: 170304.03880302055, Loss: 2.4888839721679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14927 , TFLOPS: 71.51149285422805, Tokens per sec: 177213.3569340319, Loss: 2.5120794773101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14928 , TFLOPS: 70.53141728395721, Tokens per sec: 174784.62170666398, Loss: 2.4774909019470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14929 , TFLOPS: 69.76018757693618, Tokens per sec: 172873.42953470018, Loss: 2.5064151287078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14930 , TFLOPS: 70.7383137522003, Tokens per sec: 175297.33394649785, Loss: 2.4677560329437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14931 , TFLOPS: 71.488500822244, Tokens per sec: 177156.380145994, Loss: 2.5028767585754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14932 , TFLOPS: 69.69208037123973, Tokens per sec: 172704.6523763563, Loss: 2.4877097606658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14933 , TFLOPS: 69.10690398429739, Tokens per sec: 171254.52082701263, Loss: 2.5009803771972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14934 , TFLOPS: 71.60146338872002, Tokens per sec: 177436.31382957552, Loss: 2.4825754165649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14935 , TFLOPS: 71.50885371714008, Tokens per sec: 177206.81686160294, Loss: 2.4848692417144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14936 , TFLOPS: 67.86130800920718, Tokens per sec: 168167.79678701513, Loss: 2.4891719818115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14937 , TFLOPS: 71.5248084968028, Tokens per sec: 177246.35456317125, Loss: 2.4861812591552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14938 , TFLOPS: 71.549295780144, Tokens per sec: 177307.0367487877, Loss: 2.5046348571777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14939 , TFLOPS: 68.76367697209275, Tokens per sec: 170403.96648119288, Loss: 2.4910008907318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14940 , TFLOPS: 69.68399411083618, Tokens per sec: 172684.6137323012, Loss: 2.500124216079712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14941 , TFLOPS: 71.53758460822965, Tokens per sec: 177278.01517469183, Loss: 2.5129342079162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14942 , TFLOPS: 70.55435093618541, Tokens per sec: 174841.4538232358, Loss: 2.4865260124206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14943 , TFLOPS: 67.78543650061881, Tokens per sec: 167979.77883079657, Loss: 2.502943277359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14944 , TFLOPS: 71.45604800870788, Tokens per sec: 177075.95849907896, Loss: 2.460479974746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14945 , TFLOPS: 71.54817882268328, Tokens per sec: 177304.268805157, Loss: 2.490680694580078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14946 , TFLOPS: 68.99482380977413, Tokens per sec: 170976.77380789354, Loss: 2.486323595046997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14947 , TFLOPS: 70.01082296632842, Tokens per sec: 173494.53163938152, Loss: 2.4818625450134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14948 , TFLOPS: 71.45612685944906, Tokens per sec: 177076.15389990172, Loss: 2.483083724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14949 , TFLOPS: 70.52022716774437, Tokens per sec: 174756.89136599549, Loss: 2.4935150146484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14950 , TFLOPS: 69.74295149970186, Tokens per sec: 172830.71663660288, Loss: 2.5008466243743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14951 , TFLOPS: 70.45968843934105, Tokens per sec: 174606.8697281206, Loss: 2.4746150970458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14952 , TFLOPS: 70.60555788600401, Tokens per sec: 174968.3502857975, Loss: 2.483851909637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14953 , TFLOPS: 68.76170101404371, Tokens per sec: 170399.0698394776, Loss: 2.4815540313720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14954 , TFLOPS: 71.49732942993145, Tokens per sec: 177178.2584083949, Loss: 2.5130152702331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14955 , TFLOPS: 70.45974337971357, Tokens per sec: 174607.0058764159, Loss: 2.508817195892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14956 , TFLOPS: 68.6157966963893, Tokens per sec: 170037.50286764256, Loss: 2.4811017513275146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14957 , TFLOPS: 71.46124050685754, Tokens per sec: 177088.82608149468, Loss: 2.509430170059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14958 , TFLOPS: 70.50352445915446, Tokens per sec: 174715.50021415445, Loss: 2.5021026134490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14959 , TFLOPS: 68.35025520088308, Tokens per sec: 169379.4617899676, Loss: 2.4789767265319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14960 , TFLOPS: 70.6334350889876, Tokens per sec: 175037.43306571824, Loss: 2.4834072589874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14961 , TFLOPS: 71.53399502662633, Tokens per sec: 177269.11979605374, Loss: 2.4891436100006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14962 , TFLOPS: 68.92516629944119, Tokens per sec: 170804.1548238803, Loss: 2.489291191101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14963 , TFLOPS: 68.69358644444173, Tokens per sec: 170230.27443256465, Loss: 2.4921562671661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14964 , TFLOPS: 71.54235553922099, Tokens per sec: 177289.8380672498, Loss: 2.4929895401000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14965 , TFLOPS: 70.3169871352283, Tokens per sec: 174253.24017668288, Loss: 2.516425132751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14966 , TFLOPS: 68.9763557667677, Tokens per sec: 170931.0079629007, Loss: 2.5194172859191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14967 , TFLOPS: 70.62487033641627, Tokens per sec: 175016.2086653611, Loss: 2.4944748878479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14968 , TFLOPS: 71.52212827955061, Tokens per sec: 177239.71268957, Loss: 2.4734339714050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14969 , TFLOPS: 69.5630424928397, Tokens per sec: 172384.88229898526, Loss: 2.4883482456207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14970 , TFLOPS: 69.47578339680679, Tokens per sec: 172168.64464663278, Loss: 2.498084545135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14971 , TFLOPS: 71.53572566458813, Tokens per sec: 177273.40850756856, Loss: 2.456718683242798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14972 , TFLOPS: 70.66014973559182, Tokens per sec: 175103.63490286478, Loss: 2.494374990463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14973 , TFLOPS: 68.51078351226566, Tokens per sec: 169777.26862922698, Loss: 2.4952480792999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14974 , TFLOPS: 71.45217257501558, Tokens per sec: 177066.35474747518, Loss: 2.5185928344726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14975 , TFLOPS: 71.46503060646192, Tokens per sec: 177098.2183658283, Loss: 2.4589717388153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14976 , TFLOPS: 68.29624793564322, Tokens per sec: 169245.62583731185, Loss: 2.4725873470306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14977 , TFLOPS: 70.53114453227613, Tokens per sec: 174783.94579795218, Loss: 2.484419584274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14978 , TFLOPS: 71.48913812407011, Tokens per sec: 177157.9594501237, Loss: 2.5050735473632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14979 , TFLOPS: 69.8591177392532, Tokens per sec: 173118.58937497876, Loss: 2.475855827331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14980 , TFLOPS: 68.45030027084434, Tokens per sec: 169627.38449420588, Loss: 2.484633445739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14981 , TFLOPS: 71.47089141147214, Tokens per sec: 177112.7420862646, Loss: 2.4794669151306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14982 , TFLOPS: 71.49604171425294, Tokens per sec: 177175.06730708995, Loss: 2.491668701171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14983 , TFLOPS: 68.22528209191842, Tokens per sec: 169069.76465902873, Loss: 2.467857599258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14984 , TFLOPS: 70.12568673310376, Tokens per sec: 173779.17670673926, Loss: 2.4925906658172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14985 , TFLOPS: 71.48827212573838, Tokens per sec: 177155.81341086005, Loss: 2.498126745223999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14986 , TFLOPS: 68.85432951894853, Tokens per sec: 170628.61347850395, Loss: 2.503541946411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14987 , TFLOPS: 70.54750712561508, Tokens per sec: 174824.49410673458, Loss: 2.50495982170105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14988 , TFLOPS: 70.43678060057874, Tokens per sec: 174550.10158016084, Loss: 2.4954371452331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14989 , TFLOPS: 70.5458996141143, Tokens per sec: 174820.51051615362, Loss: 2.487699508666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14990 , TFLOPS: 69.53189225589307, Tokens per sec: 172307.68858034967, Loss: 2.4854531288146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14991 , TFLOPS: 70.55392109349376, Tokens per sec: 174840.38862569482, Loss: 2.4813051223754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14992 , TFLOPS: 70.64526317655253, Tokens per sec: 175066.74436967555, Loss: 2.4979844093322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14993 , TFLOPS: 68.7781808862399, Tokens per sec: 170439.90877818756, Loss: 2.4824764728546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14994 , TFLOPS: 71.47198856094178, Tokens per sec: 177115.46094351134, Loss: 2.495781898498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14995 , TFLOPS: 70.62115705023713, Tokens per sec: 175007.00673315697, Loss: 2.491020679473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14996 , TFLOPS: 69.6913930517548, Tokens per sec: 172702.94912295815, Loss: 2.4991791248321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14997 , TFLOPS: 70.70362059144615, Tokens per sec: 175211.3604723819, Loss: 2.491511106491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14998 , TFLOPS: 71.53919014042226, Tokens per sec: 177281.9938603291, Loss: 2.465575695037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14999 , TFLOPS: 69.63111686831853, Tokens per sec: 172553.5781004333, Loss: 2.482987642288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15000 , TFLOPS: 68.97972643632527, Tokens per sec: 170939.36085337424, Loss: 2.4833903312683105 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/optimizer.pt +[rank0]:[2024-08-27 09:42:29,340] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063620360015193, 'preprocessing_with_comm': 0.0013872679992346093, 'state_converting': 1.0023833900049794, : 1.0115052839973941}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013000 + eval ppl=9.467419624328613, eval loss=2.247856378555298 +------------------------------------------------------------------ +iteration: 15001 , TFLOPS: 66.31450871101242, Tokens per sec: 164334.65773207945, Loss: 2.498852491378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15002 , TFLOPS: 70.15319756492106, Tokens per sec: 173847.35157855088, Loss: 2.4834792613983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15003 , TFLOPS: 68.88544818291928, Tokens per sec: 170705.72895582728, Loss: 2.478926658630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15004 , TFLOPS: 70.98641726163746, Tokens per sec: 175912.16177374087, Loss: 2.4793918132781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15005 , TFLOPS: 71.53819834892073, Tokens per sec: 177279.53609173343, Loss: 2.49251389503479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15006 , TFLOPS: 69.58611722618144, Tokens per sec: 172442.06403009783, Loss: 2.506704568862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15007 , TFLOPS: 68.99726304967623, Tokens per sec: 170982.8185130754, Loss: 2.48270320892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15008 , TFLOPS: 71.4692049212969, Tokens per sec: 177108.56277782813, Loss: 2.4640953540802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15009 , TFLOPS: 69.1944508228346, Tokens per sec: 171471.47153698825, Loss: 2.49235200881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15010 , TFLOPS: 68.91177302702773, Tokens per sec: 170770.964819449, Loss: 2.4657022953033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15011 , TFLOPS: 71.47039285505008, Tokens per sec: 177111.50660853018, Loss: 2.5056495666503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15012 , TFLOPS: 71.51179423027065, Tokens per sec: 177214.10377706474, Loss: 2.4888789653778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15013 , TFLOPS: 67.64414329358486, Tokens per sec: 167629.63869903472, Loss: 2.490271806716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15014 , TFLOPS: 70.59090107222477, Tokens per sec: 174932.02908667122, Loss: 2.4962258338928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15015 , TFLOPS: 71.5266968581985, Tokens per sec: 177251.03413072997, Loss: 2.483712673187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15016 , TFLOPS: 70.54309820849605, Tokens per sec: 174813.56832443128, Loss: 2.5061917304992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15017 , TFLOPS: 69.29832456969183, Tokens per sec: 171728.8821243383, Loss: 2.5099594593048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15018 , TFLOPS: 71.41745448425667, Tokens per sec: 176980.31949406117, Loss: 2.4779348373413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15019 , TFLOPS: 70.66874812940823, Tokens per sec: 175124.94266993308, Loss: 2.482382297515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15020 , TFLOPS: 68.8353453424862, Tokens per sec: 170581.56859795062, Loss: 2.512256383895874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15021 , TFLOPS: 70.45036698590793, Tokens per sec: 174583.77013967047, Loss: 2.5108962059020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15022 , TFLOPS: 71.51336478657043, Tokens per sec: 177217.99578858668, Loss: 2.5063178539276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15023 , TFLOPS: 68.77015409431834, Tokens per sec: 170420.0175035818, Loss: 2.4764461517333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15024 , TFLOPS: 70.418249880477, Tokens per sec: 174504.18041442143, Loss: 2.5088913440704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15025 , TFLOPS: 71.48168583744696, Tokens per sec: 177139.49186293437, Loss: 2.5028738975524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15026 , TFLOPS: 69.80876036572117, Tokens per sec: 172993.79825604378, Loss: 2.4684691429138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15027 , TFLOPS: 69.87900705428709, Tokens per sec: 173167.87728862168, Loss: 2.4948272705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15028 , TFLOPS: 70.43817003741218, Tokens per sec: 174553.54475769322, Loss: 2.5084946155548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15029 , TFLOPS: 71.54228063546084, Tokens per sec: 177289.65244748082, Loss: 2.4867939949035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15030 , TFLOPS: 68.1854681956396, Tokens per sec: 168971.1014381871, Loss: 2.4821252822875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15031 , TFLOPS: 71.4795633941355, Tokens per sec: 177134.23221460215, Loss: 2.471940279006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15032 , TFLOPS: 70.4426156910909, Tokens per sec: 174564.56157723826, Loss: 2.491893768310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15033 , TFLOPS: 68.70741762378103, Tokens per sec: 170264.54961859758, Loss: 2.4782776832580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15034 , TFLOPS: 70.57040141742527, Tokens per sec: 174881.2286838547, Loss: 2.478750705718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15035 , TFLOPS: 70.44618805065656, Tokens per sec: 174573.41427776986, Loss: 2.4541916847229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15036 , TFLOPS: 70.55969489687371, Tokens per sec: 174854.69674650728, Loss: 2.4954023361206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15037 , TFLOPS: 69.49016994028096, Tokens per sec: 172204.29608617033, Loss: 2.510725498199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15038 , TFLOPS: 71.49217130564784, Tokens per sec: 177165.47600820585, Loss: 2.5032429695129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15039 , TFLOPS: 69.65627049367052, Tokens per sec: 172615.91155495614, Loss: 2.485579013824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15040 , TFLOPS: 68.89638805410428, Tokens per sec: 170732.83916175, Loss: 2.4902591705322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15041 , TFLOPS: 71.51463935118997, Tokens per sec: 177221.15430011877, Loss: 2.500486135482788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15042 , TFLOPS: 71.53514968129325, Tokens per sec: 177271.98115751406, Loss: 2.493690013885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15043 , TFLOPS: 69.38922275347821, Tokens per sec: 171954.1378370225, Loss: 2.4702579975128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15044 , TFLOPS: 70.7188526058072, Tokens per sec: 175249.10708191348, Loss: 2.4988856315612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15045 , TFLOPS: 71.53572278427022, Tokens per sec: 177273.4013698235, Loss: 2.475257158279419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15046 , TFLOPS: 69.62955855772474, Tokens per sec: 172549.71643511983, Loss: 2.486196517944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15047 , TFLOPS: 68.97722614066014, Tokens per sec: 170933.1648452858, Loss: 2.5140011310577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15048 , TFLOPS: 71.44863486239703, Tokens per sec: 177057.58790589517, Loss: 2.5055670738220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15049 , TFLOPS: 71.50742190330838, Tokens per sec: 177203.26866920065, Loss: 2.484553337097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15050 , TFLOPS: 67.82452993968634, Tokens per sec: 168076.6566793037, Loss: 2.4914441108703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15051 , TFLOPS: 71.48931499829969, Tokens per sec: 177158.3977639489, Loss: 2.4827349185943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15052 , TFLOPS: 71.55744554745361, Tokens per sec: 177327.23276995186, Loss: 2.471306800842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15053 , TFLOPS: 68.73912949108802, Tokens per sec: 170343.1351191349, Loss: 2.465461015701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15054 , TFLOPS: 69.55286503699992, Tokens per sec: 172359.66144227498, Loss: 2.4913787841796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15055 , TFLOPS: 71.51605709877605, Tokens per sec: 177224.66763481486, Loss: 2.4941465854644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15056 , TFLOPS: 70.65477276499284, Tokens per sec: 175090.3101773983, Loss: 2.509036064147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15057 , TFLOPS: 69.4640189787795, Tokens per sec: 172139.49112280353, Loss: 2.495379686355591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15058 , TFLOPS: 70.46595354144347, Tokens per sec: 174622.3953412884, Loss: 2.512326717376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15059 , TFLOPS: 70.56149379163492, Tokens per sec: 174859.1546058902, Loss: 2.491351842880249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15060 , TFLOPS: 69.6745632100496, Tokens per sec: 172661.24292125256, Loss: 2.4751927852630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15061 , TFLOPS: 69.84589544811261, Tokens per sec: 173085.8230809768, Loss: 2.49090313911438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15062 , TFLOPS: 71.52525811647057, Tokens per sec: 177247.46877023764, Loss: 2.508268117904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15063 , TFLOPS: 70.68074176034935, Tokens per sec: 175154.6642086127, Loss: 2.496746301651001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15064 , TFLOPS: 69.91728781716864, Tokens per sec: 173262.74123601633, Loss: 2.470613956451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15065 , TFLOPS: 70.50891004020966, Tokens per sec: 174728.84627727998, Loss: 2.477381706237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15066 , TFLOPS: 70.57035219280733, Tokens per sec: 174881.10669982884, Loss: 2.4842629432678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15067 , TFLOPS: 68.82202340728715, Tokens per sec: 170548.55537499505, Loss: 2.494123935699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15068 , TFLOPS: 71.52800829126505, Tokens per sec: 177254.28400633516, Loss: 2.4665873050689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15069 , TFLOPS: 70.45331659671932, Tokens per sec: 174591.07960018754, Loss: 2.4820027351379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15070 , TFLOPS: 67.9986059807309, Tokens per sec: 168508.03628507105, Loss: 2.5008673667907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15071 , TFLOPS: 71.56239167254053, Tokens per sec: 177339.4898127779, Loss: 2.509401559829712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15072 , TFLOPS: 71.54101033467899, Tokens per sec: 177286.50450220818, Loss: 2.4811792373657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15073 , TFLOPS: 68.77661765042811, Tokens per sec: 170436.0349076413, Loss: 2.477773904800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15074 , TFLOPS: 70.03398361335735, Tokens per sec: 173551.92627407511, Loss: 2.4882030487060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15075 , TFLOPS: 71.51065935744437, Tokens per sec: 177211.29143718467, Loss: 2.4970662593841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15076 , TFLOPS: 69.53480448045924, Tokens per sec: 172314.90539363265, Loss: 2.49413800239563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15077 , TFLOPS: 69.50178188090302, Tokens per sec: 172233.07175419267, Loss: 2.4762959480285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15078 , TFLOPS: 71.50184196166852, Tokens per sec: 177189.44095913577, Loss: 2.513995885848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15079 , TFLOPS: 71.48246942764436, Tokens per sec: 177141.43368576313, Loss: 2.4966773986816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15080 , TFLOPS: 68.09830141320269, Tokens per sec: 168755.09254909467, Loss: 2.4912569522857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15081 , TFLOPS: 70.66729043033442, Tokens per sec: 175121.3303310494, Loss: 2.4965007305145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15082 , TFLOPS: 71.48680151249327, Tokens per sec: 177152.16906923725, Loss: 2.48887300491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15083 , TFLOPS: 69.65199319561891, Tokens per sec: 172605.31193920097, Loss: 2.4851410388946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15084 , TFLOPS: 69.58090681968591, Tokens per sec: 172429.15206882823, Loss: 2.482416868209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15085 , TFLOPS: 71.47837766455396, Tokens per sec: 177131.29384608017, Loss: 2.4883289337158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15086 , TFLOPS: 70.58152859269272, Tokens per sec: 174908.80304992694, Loss: 2.5016746520996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15087 , TFLOPS: 68.5586401511817, Tokens per sec: 169895.8626523049, Loss: 2.463438034057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15088 , TFLOPS: 71.52720853078341, Tokens per sec: 177252.30211176077, Loss: 2.4865317344665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15089 , TFLOPS: 71.48725858896165, Tokens per sec: 177153.30175507706, Loss: 2.487473487854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15090 , TFLOPS: 68.49737908009293, Tokens per sec: 169744.05096968284, Loss: 2.477839469909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15091 , TFLOPS: 69.80117795913904, Tokens per sec: 172975.0082172622, Loss: 2.501399517059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15092 , TFLOPS: 71.52074757557048, Tokens per sec: 177236.29115301016, Loss: 2.465975046157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15093 , TFLOPS: 70.02488960930671, Tokens per sec: 173529.39033025037, Loss: 2.5095643997192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15094 , TFLOPS: 68.98199738862213, Tokens per sec: 170944.98852333194, Loss: 2.4967339038848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15095 , TFLOPS: 70.44576614263974, Tokens per sec: 174572.3687432272, Loss: 2.4573872089385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15096 , TFLOPS: 71.53782965672322, Tokens per sec: 177278.6224318532, Loss: 2.5110819339752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15097 , TFLOPS: 69.01450255999163, Tokens per sec: 171025.5399187258, Loss: 2.487027645111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15098 , TFLOPS: 70.55662484420282, Tokens per sec: 174847.0888177946, Loss: 2.4947447776794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15099 , TFLOPS: 71.54244495378535, Tokens per sec: 177290.0596463901, Loss: 2.4933395385742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15100 , TFLOPS: 68.6538191453806, Tokens per sec: 170131.72668476152, Loss: 2.4585647583007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15101 , TFLOPS: 70.72121972351421, Tokens per sec: 175254.97306035098, Loss: 2.466130256652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15102 , TFLOPS: 70.41263346091877, Tokens per sec: 174490.26230805644, Loss: 2.4969472885131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15103 , TFLOPS: 70.68380593954791, Tokens per sec: 175162.25758221393, Loss: 2.491508722305298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15104 , TFLOPS: 69.63387493713533, Tokens per sec: 172560.41289878773, Loss: 2.486201286315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15105 , TFLOPS: 71.5178970800904, Tokens per sec: 177229.22731120253, Loss: 2.508758306503296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15106 , TFLOPS: 69.64383863894832, Tokens per sec: 172585.10404947266, Loss: 2.4695990085601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15107 , TFLOPS: 68.84280200619645, Tokens per sec: 170600.04703204392, Loss: 2.483283758163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15108 , TFLOPS: 71.50708306224465, Tokens per sec: 177202.42898371833, Loss: 2.502190351486206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15109 , TFLOPS: 70.41983035531106, Tokens per sec: 174508.09700516387, Loss: 2.4849002361297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15110 , TFLOPS: 69.86185676739248, Tokens per sec: 173125.37698843676, Loss: 2.4784321784973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15111 , TFLOPS: 69.93026001562309, Tokens per sec: 173294.8877727921, Loss: 2.4944939613342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15112 , TFLOPS: 71.5235155782647, Tokens per sec: 177243.15056860712, Loss: 2.479548215866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15113 , TFLOPS: 69.62032715967797, Tokens per sec: 172526.8400138375, Loss: 2.5158917903900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15114 , TFLOPS: 68.89031561913197, Tokens per sec: 170717.79099895465, Loss: 2.4914193153381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15115 , TFLOPS: 71.50421272876488, Tokens per sec: 177195.31598116213, Loss: 2.4679384231567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15116 , TFLOPS: 71.4875899351798, Tokens per sec: 177154.1228675064, Loss: 2.480290174484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15117 , TFLOPS: 67.1767109645611, Tokens per sec: 166471.28989579203, Loss: 2.4669668674468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15118 , TFLOPS: 71.52906005452765, Tokens per sec: 177256.89039156114, Loss: 2.505258798599243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15119 , TFLOPS: 71.52829534000506, Tokens per sec: 177254.9953447339, Loss: 2.5095129013061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15120 , TFLOPS: 68.63490233107721, Tokens per sec: 170084.8487350586, Loss: 2.495478630065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15121 , TFLOPS: 70.70470917274801, Tokens per sec: 175214.05809676502, Loss: 2.5118966102600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15122 , TFLOPS: 71.54685340386628, Tokens per sec: 177300.9842713215, Loss: 2.495210647583008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15123 , TFLOPS: 70.62577680013996, Tokens per sec: 175018.45498232366, Loss: 2.479058265686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15124 , TFLOPS: 68.68029769437446, Tokens per sec: 170197.34344602126, Loss: 2.5102856159210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15125 , TFLOPS: 71.53029683101553, Tokens per sec: 177259.95525994033, Loss: 2.503297805786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15126 , TFLOPS: 70.58748206605102, Tokens per sec: 174923.55641273805, Loss: 2.507899761199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15127 , TFLOPS: 69.57933110466873, Tokens per sec: 172425.2472734363, Loss: 2.4988961219787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15128 , TFLOPS: 69.9963724538131, Tokens per sec: 173458.7216775132, Loss: 2.4687387943267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15129 , TFLOPS: 71.5187213067041, Tokens per sec: 177231.26983554728, Loss: 2.498542547225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15130 , TFLOPS: 70.60403935998735, Tokens per sec: 174964.58721104878, Loss: 2.480480194091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15131 , TFLOPS: 69.04283514952445, Tokens per sec: 171095.7511966805, Loss: 2.4897618293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15132 , TFLOPS: 70.56780165560907, Tokens per sec: 174874.78618768672, Loss: 2.4828720092773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15133 , TFLOPS: 70.51609823760337, Tokens per sec: 174746.6594222669, Loss: 2.479624032974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15134 , TFLOPS: 68.82374704635959, Tokens per sec: 170552.82674249104, Loss: 2.5203380584716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15135 , TFLOPS: 70.34952724671494, Tokens per sec: 174333.87815753598, Loss: 2.475511074066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15136 , TFLOPS: 71.48127211700836, Tokens per sec: 177138.46661811243, Loss: 2.4887776374816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15137 , TFLOPS: 68.73637087111267, Tokens per sec: 170336.2989549489, Loss: 2.499065637588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15138 , TFLOPS: 71.5449675165465, Tokens per sec: 177296.31083479547, Loss: 2.471853256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15139 , TFLOPS: 70.55434656405279, Tokens per sec: 174841.44298860955, Loss: 2.498037815093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15140 , TFLOPS: 68.7037189856317, Tokens per sec: 170255.38398582494, Loss: 2.486445903778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15141 , TFLOPS: 70.67834997000068, Tokens per sec: 175148.73708865128, Loss: 2.5040283203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15142 , TFLOPS: 71.42484487712203, Tokens per sec: 176998.6337017005, Loss: 2.50122332572937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15143 , TFLOPS: 69.48275279360513, Tokens per sec: 172185.9155796415, Loss: 2.4977502822875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15144 , TFLOPS: 69.71422625568282, Tokens per sec: 172759.53231757684, Loss: 2.493403911590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15145 , TFLOPS: 71.54858016812138, Tokens per sec: 177305.26338336442, Loss: 2.49885630607605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15146 , TFLOPS: 71.52409885305143, Tokens per sec: 177244.59598778162, Loss: 2.4855215549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15147 , TFLOPS: 68.48180854255261, Tokens per sec: 169705.46546242165, Loss: 2.5108463764190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15148 , TFLOPS: 70.6998552884912, Tokens per sec: 175202.02963687712, Loss: 2.4780049324035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15149 , TFLOPS: 71.55051848141956, Tokens per sec: 177310.06673724015, Loss: 2.4973435401916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15150 , TFLOPS: 69.02320988125634, Tokens per sec: 171047.11762001034, Loss: 2.469205379486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15151 , TFLOPS: 68.80416864312122, Tokens per sec: 170504.30930252722, Loss: 2.495697498321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15152 , TFLOPS: 71.49999514906906, Tokens per sec: 177184.86435407115, Loss: 2.496184825897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15153 , TFLOPS: 70.60521401676507, Tokens per sec: 174967.4981399431, Loss: 2.464794635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15154 , TFLOPS: 68.42971202830617, Tokens per sec: 169576.36456121414, Loss: 2.4847095012664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15155 , TFLOPS: 71.49252803910788, Tokens per sec: 177166.36003301723, Loss: 2.4938852787017822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15156 , TFLOPS: 71.48746054112797, Tokens per sec: 177153.80221479206, Loss: 2.487253427505493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15157 , TFLOPS: 68.45867631097904, Tokens per sec: 169648.14124435664, Loss: 2.4919965267181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15158 , TFLOPS: 71.47000127237821, Tokens per sec: 177110.5362235329, Loss: 2.486121892929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15159 , TFLOPS: 71.52746326833072, Tokens per sec: 177252.93337946464, Loss: 2.479153871536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15160 , TFLOPS: 69.35575574476715, Tokens per sec: 171871.20290274115, Loss: 2.4887290000915527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15161 , TFLOPS: 68.95138362788448, Tokens per sec: 170869.1242518399, Loss: 2.498332977294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15162 , TFLOPS: 71.51925865112301, Tokens per sec: 177232.60143421072, Loss: 2.485940456390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15163 , TFLOPS: 71.52147368407634, Tokens per sec: 177238.0905298751, Loss: 2.4832944869995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15164 , TFLOPS: 68.95101604570353, Tokens per sec: 170868.21334270298, Loss: 2.485947608947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15165 , TFLOPS: 70.45242493276437, Tokens per sec: 174588.8699586823, Loss: 2.4760658740997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15166 , TFLOPS: 71.47989643513033, Tokens per sec: 177135.05752687482, Loss: 2.4792795181274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15167 , TFLOPS: 68.83738768592288, Tokens per sec: 170586.6297499694, Loss: 2.4869861602783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15168 , TFLOPS: 70.65017132939208, Tokens per sec: 175078.90731308964, Loss: 2.5059804916381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15169 , TFLOPS: 70.42908099463259, Tokens per sec: 174531.02110844466, Loss: 2.5017852783203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15170 , TFLOPS: 70.54019358516719, Tokens per sec: 174806.37034785168, Loss: 2.4948976039886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15171 , TFLOPS: 68.8630081921227, Tokens per sec: 170650.1201866643, Loss: 2.4882631301879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15172 , TFLOPS: 70.59181250936632, Tokens per sec: 174934.2877283111, Loss: 2.4919018745422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15173 , TFLOPS: 70.64331282023733, Tokens per sec: 175061.911171877, Loss: 2.4736573696136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15174 , TFLOPS: 68.7597947884705, Tokens per sec: 170394.34600251913, Loss: 2.4838345050811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15175 , TFLOPS: 71.52756748862357, Tokens per sec: 177253.19164883113, Loss: 2.4953980445861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15176 , TFLOPS: 70.41706350086328, Tokens per sec: 174501.2404350481, Loss: 2.474595069885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15177 , TFLOPS: 69.57117665755497, Tokens per sec: 172405.03965520195, Loss: 2.4887874126434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15178 , TFLOPS: 70.67538546837129, Tokens per sec: 175141.3907270468, Loss: 2.4775493144989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15179 , TFLOPS: 71.45458375504992, Tokens per sec: 177072.32991721368, Loss: 2.4927430152893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15180 , TFLOPS: 69.67401613834433, Tokens per sec: 172659.887217302, Loss: 2.4900779724121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15181 , TFLOPS: 69.12095275544588, Tokens per sec: 171289.3352295185, Loss: 2.4822449684143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15182 , TFLOPS: 71.56349319033691, Tokens per sec: 177342.21949521953, Loss: 2.4635658264160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15183 , TFLOPS: 70.49109550435676, Tokens per sec: 174684.69989500337, Loss: 2.4915311336517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15184 , TFLOPS: 68.6831460492108, Tokens per sec: 170204.40198307793, Loss: 2.4820327758789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15185 , TFLOPS: 71.54463457489423, Tokens per sec: 177295.4857687046, Loss: 2.501189708709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15186 , TFLOPS: 71.4713491624375, Tokens per sec: 177113.8764435822, Loss: 2.4937996864318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15187 , TFLOPS: 67.89796540837229, Tokens per sec: 168258.63786029216, Loss: 2.518342971801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15188 , TFLOPS: 70.59369803963696, Tokens per sec: 174938.96028003, Loss: 2.4710628986358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15189 , TFLOPS: 71.49188528036787, Tokens per sec: 177164.76720605377, Loss: 2.4906582832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15190 , TFLOPS: 70.61177444448816, Tokens per sec: 174983.75560253166, Loss: 2.4993395805358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15191 , TFLOPS: 68.68572694813706, Tokens per sec: 170210.7977349267, Loss: 2.505465507507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15192 , TFLOPS: 71.45359529162832, Tokens per sec: 177069.8803959667, Loss: 2.490879535675049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15193 , TFLOPS: 71.55115485867543, Tokens per sec: 177311.643750183, Loss: 2.4800634384155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15194 , TFLOPS: 67.94239230294448, Tokens per sec: 168368.7325990685, Loss: 2.5030417442321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15195 , TFLOPS: 70.89340054098895, Tokens per sec: 175681.65609333623, Loss: 2.4833827018737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15196 , TFLOPS: 71.40607121637966, Tokens per sec: 176952.11050229147, Loss: 2.4863572120666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15197 , TFLOPS: 70.4942530722322, Tokens per sec: 174692.5246960335, Loss: 2.4649195671081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15198 , TFLOPS: 68.70177974273527, Tokens per sec: 170250.57832830233, Loss: 2.5076944828033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15199 , TFLOPS: 71.52332395438967, Tokens per sec: 177242.67570353564, Loss: 2.504173755645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15200 , TFLOPS: 70.61893595940242, Tokens per sec: 175001.5026253952, Loss: 2.494131088256836 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/optimizer.pt +[rank0]:[2024-08-27 10:07:44,016] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006421483994927257, 'preprocessing_with_comm': 0.0012767440057359636, 'state_converting': 0.9726930659962818, : 0.9817595089989481}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200, took 4.26s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013200 + eval ppl=9.838297843933105, eval loss=2.286282777786255 +------------------------------------------------------------------ +iteration: 15201 , TFLOPS: 64.91056844636675, Tokens per sec: 160855.53909950002, Loss: 2.489760160446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15202 , TFLOPS: 69.9838217547661, Tokens per sec: 173427.6196627011, Loss: 2.463442087173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15203 , TFLOPS: 70.52694409582008, Tokens per sec: 174773.5366536978, Loss: 2.50470232963562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15204 , TFLOPS: 69.62794100685439, Tokens per sec: 172545.70796587516, Loss: 2.4907140731811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15205 , TFLOPS: 69.60871277046725, Tokens per sec: 172498.05827794274, Loss: 2.511643171310425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15206 , TFLOPS: 71.50648819541853, Tokens per sec: 177200.9548381932, Loss: 2.5049426555633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15207 , TFLOPS: 69.1282042113968, Tokens per sec: 171307.3051361787, Loss: 2.472003698348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15208 , TFLOPS: 70.5358721883044, Tokens per sec: 174795.66145038453, Loss: 2.492910385131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15209 , TFLOPS: 70.35054422659377, Tokens per sec: 174336.39834570527, Loss: 2.4898691177368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15210 , TFLOPS: 69.74907124957193, Tokens per sec: 172845.882050927, Loss: 2.5115835666656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15211 , TFLOPS: 69.60602460583523, Tokens per sec: 172491.3967098588, Loss: 2.490468740463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15212 , TFLOPS: 71.45059827375417, Tokens per sec: 177062.4534555258, Loss: 2.478817939758301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15213 , TFLOPS: 69.5004125168103, Tokens per sec: 172229.678319123, Loss: 2.471696615219116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15214 , TFLOPS: 69.54933803495842, Tokens per sec: 172350.92114268482, Loss: 2.4692418575286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15215 , TFLOPS: 71.50115906778198, Tokens per sec: 177187.74867285797, Loss: 2.4873125553131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15216 , TFLOPS: 70.61803629305686, Tokens per sec: 174999.27315308445, Loss: 2.4960432052612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15217 , TFLOPS: 68.63278768448197, Tokens per sec: 170079.60840784805, Loss: 2.4760353565216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15218 , TFLOPS: 70.69349308924866, Tokens per sec: 175186.26340630004, Loss: 2.5019853115081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15219 , TFLOPS: 71.52413501041681, Tokens per sec: 177244.6855897161, Loss: 2.4839470386505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15220 , TFLOPS: 70.32392081043156, Tokens per sec: 174270.42258764393, Loss: 2.5005908012390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15221 , TFLOPS: 68.85394004699823, Tokens per sec: 170627.6483241072, Loss: 2.511626958847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15222 , TFLOPS: 71.50989218688652, Tokens per sec: 177209.3903040315, Loss: 2.482285499572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15223 , TFLOPS: 70.59273249768715, Tokens per sec: 174936.56756071516, Loss: 2.468710422515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15224 , TFLOPS: 69.41793777216617, Tokens per sec: 172025.2968165517, Loss: 2.490065813064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15225 , TFLOPS: 70.6946886035202, Tokens per sec: 175189.2260223617, Loss: 2.5007100105285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15226 , TFLOPS: 71.52668942720084, Tokens per sec: 177251.01571589915, Loss: 2.485898733139038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15227 , TFLOPS: 68.66449997500726, Tokens per sec: 170158.1949571669, Loss: 2.5190505981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15228 , TFLOPS: 70.65171354430427, Tokens per sec: 175082.72909153104, Loss: 2.4835703372955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15229 , TFLOPS: 71.54276929022201, Tokens per sec: 177290.86338780876, Loss: 2.4960641860961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15230 , TFLOPS: 69.64162692181065, Tokens per sec: 172579.62317076477, Loss: 2.477444887161255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15231 , TFLOPS: 68.73709332177465, Tokens per sec: 170338.0892672738, Loss: 2.4782652854919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15232 , TFLOPS: 71.5298634707772, Tokens per sec: 177258.88134553115, Loss: 2.483802318572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15233 , TFLOPS: 70.62793183075912, Tokens per sec: 175023.79538559416, Loss: 2.46565580368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15234 , TFLOPS: 70.41153819757723, Tokens per sec: 174487.54812484287, Loss: 2.478187084197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15235 , TFLOPS: 69.87403504024313, Tokens per sec: 173155.55609010818, Loss: 2.503221035003662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15236 , TFLOPS: 70.64737219500606, Tokens per sec: 175071.97074973097, Loss: 2.493502140045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15237 , TFLOPS: 70.53307543175922, Tokens per sec: 174788.7307795774, Loss: 2.4697248935699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15238 , TFLOPS: 69.95023978213518, Tokens per sec: 173344.39983516204, Loss: 2.4768168926239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15239 , TFLOPS: 70.43456705335852, Tokens per sec: 174544.61616630678, Loss: 2.49025559425354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15240 , TFLOPS: 70.68911058750832, Tokens per sec: 175175.40308421507, Loss: 2.474883556365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15241 , TFLOPS: 68.79317682797401, Tokens per sec: 170477.07037374403, Loss: 2.4882712364196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15242 , TFLOPS: 70.45660595826318, Tokens per sec: 174599.23100044634, Loss: 2.4723973274230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15243 , TFLOPS: 70.62914596122044, Tokens per sec: 175026.8041346248, Loss: 2.4967286586761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15244 , TFLOPS: 69.61149638550907, Tokens per sec: 172504.9563826569, Loss: 2.4769318103790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15245 , TFLOPS: 70.76899536570834, Tokens per sec: 175373.3663646295, Loss: 2.475992441177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15246 , TFLOPS: 70.46258111002003, Tokens per sec: 174614.03808471712, Loss: 2.4950952529907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15247 , TFLOPS: 68.69824857441694, Tokens per sec: 170241.8277042192, Loss: 2.4978017807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15248 , TFLOPS: 70.7546025491983, Tokens per sec: 175337.699379818, Loss: 2.493028402328491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15249 , TFLOPS: 71.52253512025845, Tokens per sec: 177240.72088566126, Loss: 2.506331205368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15250 , TFLOPS: 69.60569926208326, Tokens per sec: 172490.59047220185, Loss: 2.4923152923583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15251 , TFLOPS: 69.5141007560726, Tokens per sec: 172263.59928389904, Loss: 2.492663860321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15252 , TFLOPS: 70.85760448053126, Tokens per sec: 175592.9495122605, Loss: 2.5192582607269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15253 , TFLOPS: 70.4482233868579, Tokens per sec: 174578.4580650876, Loss: 2.4624223709106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15254 , TFLOPS: 70.43375895865368, Tokens per sec: 174542.61361860923, Loss: 2.4809272289276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15255 , TFLOPS: 70.72928890104697, Tokens per sec: 175274.96937117047, Loss: 2.479522466659546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15256 , TFLOPS: 71.49783794988866, Tokens per sec: 177179.51857686698, Loss: 2.4880638122558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15257 , TFLOPS: 69.20205423026431, Tokens per sec: 171490.31361818354, Loss: 2.500427722930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15258 , TFLOPS: 69.59122210214606, Tokens per sec: 172454.7144750867, Loss: 2.502195358276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15259 , TFLOPS: 71.53184269511351, Tokens per sec: 177263.78608146607, Loss: 2.4818787574768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15260 , TFLOPS: 70.51056152376722, Tokens per sec: 174732.93883546154, Loss: 2.4947218894958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15261 , TFLOPS: 67.81068868223619, Tokens per sec: 168042.35651860182, Loss: 2.5070557594299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15262 , TFLOPS: 71.53755416935921, Tokens per sec: 177277.93974381775, Loss: 2.4811644554138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15263 , TFLOPS: 70.69291924281033, Tokens per sec: 175184.84135161145, Loss: 2.5009102821350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15264 , TFLOPS: 69.49683314093042, Tokens per sec: 172220.80823139186, Loss: 2.4948790073394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15265 , TFLOPS: 70.65207647819192, Tokens per sec: 175083.6284816851, Loss: 2.4709150791168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15266 , TFLOPS: 71.53292450353071, Tokens per sec: 177266.46692189496, Loss: 2.4945785999298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15267 , TFLOPS: 69.20518831913384, Tokens per sec: 171498.08023565062, Loss: 2.499476909637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15268 , TFLOPS: 68.98450313574581, Tokens per sec: 170951.19804073527, Loss: 2.499675750732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15269 , TFLOPS: 71.50304558182705, Tokens per sec: 177192.42366247834, Loss: 2.4777321815490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15270 , TFLOPS: 70.65164336027658, Tokens per sec: 175082.55516777904, Loss: 2.481210708618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15271 , TFLOPS: 69.64509215142898, Tokens per sec: 172588.21039148557, Loss: 2.4880073070526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15272 , TFLOPS: 69.71993658474592, Tokens per sec: 172773.68314203987, Loss: 2.4915339946746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15273 , TFLOPS: 71.54469508752366, Tokens per sec: 177295.63572566636, Loss: 2.479802131652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15274 , TFLOPS: 69.72204162822925, Tokens per sec: 172778.89967168128, Loss: 2.489474296569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15275 , TFLOPS: 70.74265695640497, Tokens per sec: 175308.09688496043, Loss: 2.4914650917053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15276 , TFLOPS: 70.47690825749314, Tokens per sec: 174649.54233442104, Loss: 2.4799954891204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15277 , TFLOPS: 69.8428058846934, Tokens per sec: 173078.1668024805, Loss: 2.5005478858947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15278 , TFLOPS: 69.56267627517416, Tokens per sec: 172383.97477126782, Loss: 2.503648281097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15279 , TFLOPS: 70.56641850878894, Tokens per sec: 174871.35859750025, Loss: 2.479905843734741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15280 , TFLOPS: 70.57646717249847, Tokens per sec: 174896.26029312413, Loss: 2.5058584213256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15281 , TFLOPS: 68.85418182104867, Tokens per sec: 170628.2474668378, Loss: 2.4976017475128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15282 , TFLOPS: 71.52887959150203, Tokens per sec: 177256.44318430402, Loss: 2.495417356491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15283 , TFLOPS: 69.60947821195232, Tokens per sec: 172499.95512626308, Loss: 2.4881110191345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15284 , TFLOPS: 69.70314348339936, Tokens per sec: 172732.06798699027, Loss: 2.473296880722046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15285 , TFLOPS: 70.54165666640439, Tokens per sec: 174809.9960243292, Loss: 2.4913761615753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15286 , TFLOPS: 71.48714226579364, Tokens per sec: 177153.01349345263, Loss: 2.48402738571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15287 , TFLOPS: 69.67500471375386, Tokens per sec: 172662.33701606738, Loss: 2.5142173767089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15288 , TFLOPS: 69.05409462308032, Tokens per sec: 171123.65341248535, Loss: 2.484049081802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15289 , TFLOPS: 71.55815830973734, Tokens per sec: 177328.9990734082, Loss: 2.4819533824920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15290 , TFLOPS: 70.70632799049763, Tokens per sec: 175218.06970547608, Loss: 2.4782235622406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15291 , TFLOPS: 68.67179236598152, Tokens per sec: 170176.2663053243, Loss: 2.4991607666015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15292 , TFLOPS: 70.59345552928012, Tokens per sec: 174938.35931265107, Loss: 2.4814951419830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15293 , TFLOPS: 71.48863455423687, Tokens per sec: 177156.71154860413, Loss: 2.4827938079833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15294 , TFLOPS: 68.54166573479647, Tokens per sec: 169853.79817861598, Loss: 2.4748566150665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15295 , TFLOPS: 70.67957235443927, Tokens per sec: 175151.76629194684, Loss: 2.504314422607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15296 , TFLOPS: 71.5152602814878, Tokens per sec: 177222.6930337979, Loss: 2.4700474739074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15297 , TFLOPS: 69.78164417724582, Tokens per sec: 172926.60135390714, Loss: 2.4866936206817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15298 , TFLOPS: 69.50009782129348, Tokens per sec: 172228.89846896575, Loss: 2.4918792247772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15299 , TFLOPS: 71.52928151275619, Tokens per sec: 177257.4391894471, Loss: 2.4798038005828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15300 , TFLOPS: 70.56722575628368, Tokens per sec: 174873.35904571784, Loss: 2.4886600971221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15301 , TFLOPS: 68.83814375827713, Tokens per sec: 170588.50338055144, Loss: 2.5010790824890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15302 , TFLOPS: 71.02104249072323, Tokens per sec: 175997.9669056993, Loss: 2.4896039962768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15303 , TFLOPS: 70.72331183393743, Tokens per sec: 175260.15754044472, Loss: 2.4963996410369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15304 , TFLOPS: 70.53570500921542, Tokens per sec: 174795.24716218575, Loss: 2.4794116020202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15305 , TFLOPS: 68.86188320337746, Tokens per sec: 170647.33234062584, Loss: 2.512704610824585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15306 , TFLOPS: 71.48210559472189, Tokens per sec: 177140.53206770174, Loss: 2.4840316772460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15307 , TFLOPS: 70.56554695685676, Tokens per sec: 174869.19879580292, Loss: 2.4796173572540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15308 , TFLOPS: 68.94326577917904, Tokens per sec: 170849.00732849297, Loss: 2.485773801803589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15309 , TFLOPS: 70.51614373300951, Tokens per sec: 174746.77216489505, Loss: 2.5049946308135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15310 , TFLOPS: 70.68510251917876, Tokens per sec: 175165.4706493692, Loss: 2.4875755310058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15311 , TFLOPS: 69.58942085098373, Tokens per sec: 172450.2507762822, Loss: 2.493650197982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15312 , TFLOPS: 70.63005358994221, Tokens per sec: 175029.05333858088, Loss: 2.4932351112365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15313 , TFLOPS: 70.43366937034719, Tokens per sec: 174542.39160891675, Loss: 2.4758331775665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15314 , TFLOPS: 69.56641457661186, Tokens per sec: 172393.2386940384, Loss: 2.5010180473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15315 , TFLOPS: 70.73108008746478, Tokens per sec: 175279.40812842923, Loss: 2.5029940605163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15316 , TFLOPS: 70.57482717297403, Tokens per sec: 174892.1961936417, Loss: 2.476921796798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15317 , TFLOPS: 70.69478648435519, Tokens per sec: 175189.4685818533, Loss: 2.4923672676086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15318 , TFLOPS: 70.00886060317553, Tokens per sec: 173489.66868731653, Loss: 2.4846043586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15319 , TFLOPS: 70.79133710325921, Tokens per sec: 175428.7316514261, Loss: 2.49967098236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15320 , TFLOPS: 69.51535997116156, Tokens per sec: 172266.71975760345, Loss: 2.510911464691162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15321 , TFLOPS: 69.57137015963417, Tokens per sec: 172405.5191746702, Loss: 2.501816749572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15322 , TFLOPS: 70.6356546794615, Tokens per sec: 175042.93345542066, Loss: 2.498704195022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15323 , TFLOPS: 71.55205908885942, Tokens per sec: 177313.88453218807, Loss: 2.4651577472686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15324 , TFLOPS: 69.64929932317696, Tokens per sec: 172598.63622653528, Loss: 2.4873194694519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15325 , TFLOPS: 69.6458104132615, Tokens per sec: 172589.9903234296, Loss: 2.4747164249420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15326 , TFLOPS: 71.50102914192587, Tokens per sec: 177187.426702274, Loss: 2.4998955726623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15327 , TFLOPS: 69.67485770103451, Tokens per sec: 172661.97270235405, Loss: 2.5088846683502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15328 , TFLOPS: 68.75733260044102, Tokens per sec: 170388.24442934964, Loss: 2.480447292327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15329 , TFLOPS: 71.46295943700851, Tokens per sec: 177093.08577976457, Loss: 2.49438738822937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15330 , TFLOPS: 70.01188733759606, Tokens per sec: 173497.16926863295, Loss: 2.5023880004882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15331 , TFLOPS: 69.48235576034384, Tokens per sec: 172184.93168748316, Loss: 2.501309633255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15332 , TFLOPS: 70.65236285662095, Tokens per sec: 175084.3381589794, Loss: 2.4747300148010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15333 , TFLOPS: 71.48499248265405, Tokens per sec: 177147.68609401459, Loss: 2.493138074874878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15334 , TFLOPS: 70.4539993101397, Tokens per sec: 174592.77143925035, Loss: 2.4800608158111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15335 , TFLOPS: 68.94593530000431, Tokens per sec: 170855.62269517, Loss: 2.494910955429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15336 , TFLOPS: 71.50275566496484, Tokens per sec: 177191.7052165568, Loss: 2.485225200653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15337 , TFLOPS: 70.69832581183604, Tokens per sec: 175198.2394252399, Loss: 2.4837985038757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15338 , TFLOPS: 68.6469256128418, Tokens per sec: 170114.64375174508, Loss: 2.5163726806640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15339 , TFLOPS: 70.56838073397124, Tokens per sec: 174876.22120765893, Loss: 2.495800733566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15340 , TFLOPS: 71.52782239297555, Tokens per sec: 177253.8233298954, Loss: 2.4913527965545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15341 , TFLOPS: 68.97262128006624, Tokens per sec: 170921.75349346682, Loss: 2.505343198776245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15342 , TFLOPS: 69.79769065609699, Tokens per sec: 172966.36629616586, Loss: 2.4835457801818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15343 , TFLOPS: 71.55769956224208, Tokens per sec: 177327.8622465797, Loss: 2.506150960922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15344 , TFLOPS: 69.8541745642577, Tokens per sec: 173106.3396428043, Loss: 2.4966444969177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15345 , TFLOPS: 69.59070277396692, Tokens per sec: 172453.42752265005, Loss: 2.4979145526885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15346 , TFLOPS: 70.52775814464371, Tokens per sec: 174775.55395635767, Loss: 2.4968249797821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15347 , TFLOPS: 70.56345312856513, Tokens per sec: 174864.01005864202, Loss: 2.47625994682312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15348 , TFLOPS: 69.60160144699368, Tokens per sec: 172480.4356349983, Loss: 2.4829490184783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15349 , TFLOPS: 70.98691893032303, Tokens per sec: 175913.4049640071, Loss: 2.478813409805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15350 , TFLOPS: 69.03263283796797, Tokens per sec: 171070.46874476614, Loss: 2.498342990875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15351 , TFLOPS: 69.64435289744033, Tokens per sec: 172586.37843866614, Loss: 2.470475673675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15352 , TFLOPS: 70.68618968092066, Tokens per sec: 175168.1647559264, Loss: 2.477383852005005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15353 , TFLOPS: 70.3579176963453, Tokens per sec: 174354.6706160043, Loss: 2.4836323261260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15354 , TFLOPS: 70.61715293489993, Tokens per sec: 174997.08409426166, Loss: 2.4896669387817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15355 , TFLOPS: 69.54204481228464, Tokens per sec: 172332.84773348438, Loss: 2.4865355491638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15356 , TFLOPS: 71.54791404633713, Tokens per sec: 177303.6126602037, Loss: 2.4661593437194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15357 , TFLOPS: 70.42764476434145, Tokens per sec: 174527.4619715694, Loss: 2.5217957496643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15358 , TFLOPS: 69.569947306052, Tokens per sec: 172401.9931867533, Loss: 2.510910987854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15359 , TFLOPS: 70.68373155843605, Tokens per sec: 175162.073257625, Loss: 2.490351676940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15360 , TFLOPS: 71.50339126047983, Tokens per sec: 177193.28029225956, Loss: 2.499497175216675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15361 , TFLOPS: 68.04872011018645, Tokens per sec: 168632.224618389, Loss: 2.479090929031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15362 , TFLOPS: 70.66917912709049, Tokens per sec: 175126.01072966805, Loss: 2.4759469032287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15363 , TFLOPS: 71.55241866358726, Tokens per sec: 177314.7755979741, Loss: 2.484804391860962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15364 , TFLOPS: 70.59732474080775, Tokens per sec: 174947.94764504535, Loss: 2.4822418689727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15365 , TFLOPS: 68.57947671580075, Tokens per sec: 169947.49795476798, Loss: 2.4741616249084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15366 , TFLOPS: 71.53590090391472, Tokens per sec: 177273.8427699241, Loss: 2.5065677165985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15367 , TFLOPS: 70.65318600730276, Tokens per sec: 175086.3780170465, Loss: 2.4693095684051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15368 , TFLOPS: 69.49764472990502, Tokens per sec: 172222.81943827483, Loss: 2.4826462268829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15369 , TFLOPS: 70.60962943082863, Tokens per sec: 174978.4400223906, Loss: 2.489636182785034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15370 , TFLOPS: 71.50138136726454, Tokens per sec: 177188.29955546407, Loss: 2.4770419597625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15371 , TFLOPS: 69.89175447232746, Tokens per sec: 173199.46679476977, Loss: 2.508939027786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15372 , TFLOPS: 69.01700914877256, Tokens per sec: 171031.75152184846, Loss: 2.492236614227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15373 , TFLOPS: 71.47231410182205, Tokens per sec: 177116.26766967395, Loss: 2.4798946380615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15374 , TFLOPS: 70.63247815623649, Tokens per sec: 175035.06168094615, Loss: 2.483262538909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15375 , TFLOPS: 68.60624515579195, Tokens per sec: 170013.8330978533, Loss: 2.4904093742370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15376 , TFLOPS: 71.52350049529451, Tokens per sec: 177243.11319134542, Loss: 2.4769644737243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15377 , TFLOPS: 70.61639727132984, Tokens per sec: 174995.2114766918, Loss: 2.492865800857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15378 , TFLOPS: 69.61769133028669, Tokens per sec: 172520.3081382453, Loss: 2.515061616897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15379 , TFLOPS: 69.6232902961041, Tokens per sec: 172534.18299231786, Loss: 2.494382619857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15380 , TFLOPS: 71.47124521191024, Tokens per sec: 177113.61884272454, Loss: 2.494655132293701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15381 , TFLOPS: 68.88118250515818, Tokens per sec: 170695.15813643206, Loss: 2.4738528728485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15382 , TFLOPS: 70.57875987525465, Tokens per sec: 174901.941863116, Loss: 2.5044515132904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15383 , TFLOPS: 70.51354670881348, Tokens per sec: 174740.33645285145, Loss: 2.4770023822784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15384 , TFLOPS: 70.56543682095423, Tokens per sec: 174868.92586690397, Loss: 2.4908344745635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15385 , TFLOPS: 69.5036421673098, Tokens per sec: 172237.68174883616, Loss: 2.4762916564941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15386 , TFLOPS: 71.01768855637907, Tokens per sec: 175989.6554869281, Loss: 2.476602077484131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15387 , TFLOPS: 69.29228848211983, Tokens per sec: 171713.92403440495, Loss: 2.502197742462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15388 , TFLOPS: 69.62091919191865, Tokens per sec: 172528.30713494762, Loss: 2.5076818466186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15389 , TFLOPS: 70.53671660947774, Tokens per sec: 174797.75401907274, Loss: 2.488524913787842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15390 , TFLOPS: 70.57974038114723, Tokens per sec: 174904.37166472946, Loss: 2.4847683906555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15391 , TFLOPS: 69.62640160039679, Tokens per sec: 172541.89314709362, Loss: 2.470898151397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15392 , TFLOPS: 69.72742871117951, Tokens per sec: 172792.2494566682, Loss: 2.505692720413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15393 , TFLOPS: 71.52275994898046, Tokens per sec: 177241.2780360013, Loss: 2.502051830291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15394 , TFLOPS: 70.52209520803025, Tokens per sec: 174761.52057560554, Loss: 2.4721875190734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15395 , TFLOPS: 69.51566626161748, Tokens per sec: 172267.47877909392, Loss: 2.4869308471679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15396 , TFLOPS: 71.51149223604963, Tokens per sec: 177213.35540211768, Loss: 2.506932020187378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15397 , TFLOPS: 70.60695817041372, Tokens per sec: 174971.8203448188, Loss: 2.4926440715789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15398 , TFLOPS: 67.70570941527009, Tokens per sec: 167782.20633063896, Loss: 2.4652700424194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15399 , TFLOPS: 70.74546116043776, Tokens per sec: 175315.04601146225, Loss: 2.4991488456726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15400 , TFLOPS: 71.56282481779779, Tokens per sec: 177340.5631944403, Loss: 2.4980380535125732 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/model.pt +[rank0]:[2024-08-27 10:32:58,443] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006356117999530397, 'preprocessing_with_comm': 0.0012882660084869713, 'state_converting': 0.9732688069925644, : 0.9822757860092679}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400, took 4.61s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013400 + eval ppl=9.84315299987793, eval loss=2.286776065826416 +------------------------------------------------------------------ +iteration: 15401 , TFLOPS: 64.03681306344112, Tokens per sec: 158690.27700850906, Loss: 2.4865403175354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15402 , TFLOPS: 69.60129345649618, Tokens per sec: 172479.67240061754, Loss: 2.5053725242614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15403 , TFLOPS: 71.48590496374457, Tokens per sec: 177149.947322675, Loss: 2.4772984981536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15404 , TFLOPS: 70.57707713026416, Tokens per sec: 174897.7718356599, Loss: 2.504347085952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15405 , TFLOPS: 67.41002440870533, Tokens per sec: 167049.46631197905, Loss: 2.485020399093628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15406 , TFLOPS: 71.49148376956654, Tokens per sec: 177163.772218058, Loss: 2.4807238578796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15407 , TFLOPS: 70.61371774785499, Tokens per sec: 174988.5713223472, Loss: 2.4888339042663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15408 , TFLOPS: 69.62267401621884, Tokens per sec: 172532.65578287336, Loss: 2.470659017562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15409 , TFLOPS: 69.78461491725753, Tokens per sec: 172933.9631749101, Loss: 2.4946117401123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15410 , TFLOPS: 71.50433696850763, Tokens per sec: 177195.62386093076, Loss: 2.4668469429016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15411 , TFLOPS: 70.54613980508948, Tokens per sec: 174821.10573584933, Loss: 2.497455358505249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15412 , TFLOPS: 69.05605245585954, Tokens per sec: 171128.50513778534, Loss: 2.5146734714508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15413 , TFLOPS: 70.42640161505877, Tokens per sec: 174524.3813107027, Loss: 2.463989496231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15414 , TFLOPS: 70.58577240478108, Tokens per sec: 174919.3196837764, Loss: 2.526991844177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15415 , TFLOPS: 68.71465441343025, Tokens per sec: 170282.48318054658, Loss: 2.4861412048339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15416 , TFLOPS: 70.41880459012174, Tokens per sec: 174505.55504602738, Loss: 2.490933895111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15417 , TFLOPS: 71.4892325217721, Tokens per sec: 177158.19337802797, Loss: 2.506545305252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15418 , TFLOPS: 68.69482246350778, Tokens per sec: 170233.33742396903, Loss: 2.4716713428497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15419 , TFLOPS: 70.67082383532096, Tokens per sec: 175130.08649784274, Loss: 2.4761154651641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15420 , TFLOPS: 70.23671545581216, Tokens per sec: 174054.31811243418, Loss: 2.484081268310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15421 , TFLOPS: 69.69256029757838, Tokens per sec: 172705.8416866917, Loss: 2.5034921169281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15422 , TFLOPS: 69.73587546890896, Tokens per sec: 172813.18145280893, Loss: 2.4996445178985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15423 , TFLOPS: 70.45268979126304, Tokens per sec: 174589.52630721856, Loss: 2.4651644229888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15424 , TFLOPS: 70.57967874239701, Tokens per sec: 174904.21891711632, Loss: 2.4725570678710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15425 , TFLOPS: 68.68180890473315, Tokens per sec: 170201.08839176368, Loss: 2.503352165222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15426 , TFLOPS: 71.53631259897216, Tokens per sec: 177274.86299562838, Loss: 2.4974563121795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15427 , TFLOPS: 70.39771683527775, Tokens per sec: 174453.29726651602, Loss: 2.498014211654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15428 , TFLOPS: 68.95119496920496, Tokens per sec: 170868.6567348496, Loss: 2.4816713333129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15429 , TFLOPS: 70.61800826115403, Tokens per sec: 174999.20368694127, Loss: 2.4880571365356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15430 , TFLOPS: 71.49551931676115, Tokens per sec: 177173.77274855846, Loss: 2.471906900405884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15431 , TFLOPS: 69.51364243277017, Tokens per sec: 172262.46350826713, Loss: 2.4927468299865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15432 , TFLOPS: 68.84748559458362, Tokens per sec: 170611.65348000714, Loss: 2.485373020172119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15433 , TFLOPS: 71.50657765569005, Tokens per sec: 177201.1765306009, Loss: 2.482701539993286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15434 , TFLOPS: 70.60915374124582, Tokens per sec: 174977.26121120286, Loss: 2.5012753009796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15435 , TFLOPS: 67.98186213186952, Tokens per sec: 168466.54318310533, Loss: 2.4985427856445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15436 , TFLOPS: 71.40443553070979, Tokens per sec: 176948.05709301587, Loss: 2.475614070892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15437 , TFLOPS: 71.5112216361498, Tokens per sec: 177212.68482575685, Loss: 2.486966609954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15438 , TFLOPS: 68.55523928685197, Tokens per sec: 169887.43493585967, Loss: 2.4959940910339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15439 , TFLOPS: 70.61702858487105, Tokens per sec: 174996.77594119182, Loss: 2.492459535598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15440 , TFLOPS: 71.48457823150946, Tokens per sec: 177146.65953404477, Loss: 2.5116126537323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15441 , TFLOPS: 70.62671404863875, Tokens per sec: 175020.7775873501, Loss: 2.5050811767578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15442 , TFLOPS: 68.08519586853691, Tokens per sec: 168722.61556571795, Loss: 2.5149359703063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15443 , TFLOPS: 71.52274129121099, Tokens per sec: 177241.23179999323, Loss: 2.494145393371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15444 , TFLOPS: 70.60402215770404, Tokens per sec: 174964.5445818957, Loss: 2.517042875289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15445 , TFLOPS: 69.72577952993764, Tokens per sec: 172788.1626038785, Loss: 2.5009710788726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15446 , TFLOPS: 69.81773956621784, Tokens per sec: 173016.04970401508, Loss: 2.4728527069091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15447 , TFLOPS: 71.50755191999977, Tokens per sec: 177203.5908648845, Loss: 2.5042028427124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15448 , TFLOPS: 69.54341330548935, Tokens per sec: 172336.23901039764, Loss: 2.4879777431488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15449 , TFLOPS: 69.98547552383752, Tokens per sec: 173431.71788463776, Loss: 2.4905683994293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15450 , TFLOPS: 70.52009996510212, Tokens per sec: 174756.57614383547, Loss: 2.489410400390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15451 , TFLOPS: 70.64325076876959, Tokens per sec: 175061.7574015044, Loss: 2.500640392303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15452 , TFLOPS: 68.78118473256414, Tokens per sec: 170447.35264028, Loss: 2.47711181640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15453 , TFLOPS: 70.35653054467815, Tokens per sec: 174351.23310136577, Loss: 2.4845328330993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15454 , TFLOPS: 70.5796041474687, Tokens per sec: 174904.0340626667, Loss: 2.4934885501861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15455 , TFLOPS: 68.7641286921498, Tokens per sec: 170405.0858932553, Loss: 2.4806089401245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15456 , TFLOPS: 71.53007660335888, Tokens per sec: 177259.40951154704, Loss: 2.4923322200775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15457 , TFLOPS: 70.52430532616393, Tokens per sec: 174766.9974918054, Loss: 2.4981846809387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15458 , TFLOPS: 69.87858071581995, Tokens per sec: 173166.82077493507, Loss: 2.486067771911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15459 , TFLOPS: 69.86218181830873, Tokens per sec: 173126.18250041455, Loss: 2.4931068420410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15460 , TFLOPS: 70.46934301965847, Tokens per sec: 174630.79484168696, Loss: 2.481618881225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15461 , TFLOPS: 70.56696591436811, Tokens per sec: 174872.71512882714, Loss: 2.4720778465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15462 , TFLOPS: 68.82493059134002, Tokens per sec: 170555.75969733082, Loss: 2.488111734390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15463 , TFLOPS: 71.53142904137339, Tokens per sec: 177262.76100193034, Loss: 2.4960684776306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15464 , TFLOPS: 69.92783367280383, Tokens per sec: 173288.87502800222, Loss: 2.5065248012542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15465 , TFLOPS: 69.13466499674445, Tokens per sec: 171323.3156739844, Loss: 2.4823241233825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15466 , TFLOPS: 70.62230131184612, Tokens per sec: 175009.84233947485, Loss: 2.5060856342315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15467 , TFLOPS: 71.53849499137122, Tokens per sec: 177280.27120440363, Loss: 2.484409809112549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15468 , TFLOPS: 68.55195339566424, Tokens per sec: 169879.2921355257, Loss: 2.5070080757141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15469 , TFLOPS: 69.930048320187, Tokens per sec: 173294.36316818124, Loss: 2.478459119796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15470 , TFLOPS: 71.50718969571184, Tokens per sec: 177202.69323319668, Loss: 2.5049097537994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15471 , TFLOPS: 70.22182400293441, Tokens per sec: 174017.4154517742, Loss: 2.5051844120025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15472 , TFLOPS: 68.58123073588052, Tokens per sec: 169951.8446097469, Loss: 2.50594162940979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15473 , TFLOPS: 71.50654973496671, Tokens per sec: 177201.10733997275, Loss: 2.4858107566833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15474 , TFLOPS: 70.5983472973843, Tokens per sec: 174950.48165288594, Loss: 2.504277229309082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15475 , TFLOPS: 67.93258216294353, Tokens per sec: 168344.42199147, Loss: 2.4858012199401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15476 , TFLOPS: 71.49538346393732, Tokens per sec: 177173.43609029558, Loss: 2.478365421295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15477 , TFLOPS: 71.52780482464388, Tokens per sec: 177253.7797936342, Loss: 2.506502389907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15478 , TFLOPS: 69.75660931620308, Tokens per sec: 172864.5622104256, Loss: 2.5022525787353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15479 , TFLOPS: 68.64442356504037, Tokens per sec: 170108.44340166982, Loss: 2.465205430984497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15480 , TFLOPS: 71.53203461596105, Tokens per sec: 177264.26168246812, Loss: 2.4961395263671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15481 , TFLOPS: 70.55679380563075, Tokens per sec: 174847.50752282565, Loss: 2.481607437133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15482 , TFLOPS: 68.81756293776789, Tokens per sec: 170537.50184016503, Loss: 2.486027956008911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15483 , TFLOPS: 70.47279012984306, Tokens per sec: 174639.33716045495, Loss: 2.4657018184661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15484 , TFLOPS: 71.55620222257164, Tokens per sec: 177324.1516739308, Loss: 2.5189287662506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15485 , TFLOPS: 68.86753014895437, Tokens per sec: 170661.32609382784, Loss: 2.4969000816345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15486 , TFLOPS: 70.67908428634736, Tokens per sec: 175150.55680544997, Loss: 2.496513843536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15487 , TFLOPS: 70.47616033942094, Tokens per sec: 174647.68891104814, Loss: 2.4760231971740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15488 , TFLOPS: 70.55352671140551, Tokens per sec: 174839.41130343536, Loss: 2.5078859329223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15489 , TFLOPS: 69.0674949939565, Tokens per sec: 171156.86100768924, Loss: 2.4963161945343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15490 , TFLOPS: 70.55862573706976, Tokens per sec: 174852.04725073543, Loss: 2.503918409347534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15491 , TFLOPS: 70.68913524982132, Tokens per sec: 175175.46420014347, Loss: 2.49379825592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15492 , TFLOPS: 68.70382898552505, Tokens per sec: 170255.65657767814, Loss: 2.504854679107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15493 , TFLOPS: 71.53580797780873, Tokens per sec: 177273.61248879676, Loss: 2.49308180809021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15494 , TFLOPS: 69.53508192782442, Tokens per sec: 172315.5929387669, Loss: 2.49099063873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15495 , TFLOPS: 69.49759870563838, Tokens per sec: 172222.7053850721, Loss: 2.49190092086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15496 , TFLOPS: 71.0146243375872, Tokens per sec: 175982.06201521025, Loss: 2.485403299331665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15497 , TFLOPS: 70.24073540134286, Tokens per sec: 174064.27998029283, Loss: 2.5057201385498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15498 , TFLOPS: 70.63991463548753, Tokens per sec: 175053.49009572607, Loss: 2.4838736057281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15499 , TFLOPS: 69.03850339634351, Tokens per sec: 171085.01663511613, Loss: 2.4921343326568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15500 , TFLOPS: 71.51415554356012, Tokens per sec: 177219.95537151617, Loss: 2.4784226417541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15501 , TFLOPS: 68.95327260714672, Tokens per sec: 170873.80535053997, Loss: 2.4979777336120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15502 , TFLOPS: 68.80021720100689, Tokens per sec: 170494.5171936806, Loss: 2.4872496128082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15503 , TFLOPS: 71.45823748514525, Tokens per sec: 177081.3842628815, Loss: 2.4939584732055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15504 , TFLOPS: 71.50879046413637, Tokens per sec: 177206.66011369185, Loss: 2.4786441326141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15505 , TFLOPS: 67.97690391538677, Tokens per sec: 168454.25617646845, Loss: 2.4693539142608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15506 , TFLOPS: 70.62722742612543, Tokens per sec: 175022.04979331527, Loss: 2.478976249694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15507 , TFLOPS: 71.48506988191737, Tokens per sec: 177147.8778979153, Loss: 2.4783689975738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15508 , TFLOPS: 70.57093284642534, Tokens per sec: 174882.54562345857, Loss: 2.5054781436920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15509 , TFLOPS: 67.86471288721717, Tokens per sec: 168176.23444980214, Loss: 2.4941446781158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15510 , TFLOPS: 71.5357320246932, Tokens per sec: 177273.42426860958, Loss: 2.477245569229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15511 , TFLOPS: 70.63777047523496, Tokens per sec: 175048.17663042145, Loss: 2.485417604446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15512 , TFLOPS: 68.68164127532044, Tokens per sec: 170200.67298761307, Loss: 2.4885802268981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15513 , TFLOPS: 70.96019751567077, Tokens per sec: 175847.18635489198, Loss: 2.4634878635406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15514 , TFLOPS: 71.4993880407798, Tokens per sec: 177183.35987285187, Loss: 2.4750490188598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15515 , TFLOPS: 69.70966675566878, Tokens per sec: 172748.23337427166, Loss: 2.499359130859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15516 , TFLOPS: 69.05937372686071, Tokens per sec: 171136.73561319444, Loss: 2.4810631275177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15517 , TFLOPS: 71.55034996811393, Tokens per sec: 177309.64914270557, Loss: 2.504783868789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15518 , TFLOPS: 70.68561925599629, Tokens per sec: 175166.75118012633, Loss: 2.483161211013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15519 , TFLOPS: 68.8355537643256, Tokens per sec: 170582.08509022818, Loss: 2.502042770385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15520 , TFLOPS: 70.3832491468866, Tokens per sec: 174417.4447409353, Loss: 2.4884231090545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15521 , TFLOPS: 70.60773758774064, Tokens per sec: 174973.75182681484, Loss: 2.5140182971954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15522 , TFLOPS: 68.75629873712376, Tokens per sec: 170385.68240216005, Loss: 2.4939093589782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15523 , TFLOPS: 71.55415919682608, Tokens per sec: 177319.08883107544, Loss: 2.487034797668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15524 , TFLOPS: 70.55342547937171, Tokens per sec: 174839.160439304, Loss: 2.5075747966766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15525 , TFLOPS: 69.86082258911186, Tokens per sec: 173122.81418073326, Loss: 2.4969840049743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15526 , TFLOPS: 69.8495558333987, Tokens per sec: 173094.89391894042, Loss: 2.4824538230895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15527 , TFLOPS: 70.50877877494568, Tokens per sec: 174728.52098749578, Loss: 2.504396438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15528 , TFLOPS: 70.5569715595786, Tokens per sec: 174847.94801668965, Loss: 2.4910943508148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15529 , TFLOPS: 68.84366502915115, Tokens per sec: 170602.18569799548, Loss: 2.4725191593170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15530 , TFLOPS: 71.52566142721943, Tokens per sec: 177248.4682187017, Loss: 2.5119807720184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15531 , TFLOPS: 70.5039521762222, Tokens per sec: 174716.56014416527, Loss: 2.4550888538360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15532 , TFLOPS: 68.90333915360122, Tokens per sec: 170750.06475203024, Loss: 2.493267059326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15533 , TFLOPS: 70.6066435627964, Tokens per sec: 174971.04071248622, Loss: 2.4769630432128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15534 , TFLOPS: 70.45603928905572, Tokens per sec: 174597.8267317265, Loss: 2.4763176441192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15535 , TFLOPS: 69.61922785242058, Tokens per sec: 172524.115809355, Loss: 2.4881906509399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15536 , TFLOPS: 69.73465425125, Tokens per sec: 172810.15514092182, Loss: 2.4986143112182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15537 , TFLOPS: 71.48756122793012, Tokens per sec: 177154.05172777933, Loss: 2.4819905757904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15538 , TFLOPS: 69.68326955822391, Tokens per sec: 172682.81821111194, Loss: 2.490887403488159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15539 , TFLOPS: 68.38354281268283, Tokens per sec: 169461.95215894058, Loss: 2.4883759021759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15540 , TFLOPS: 71.48063302524376, Tokens per sec: 177136.88287831802, Loss: 2.461699962615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15541 , TFLOPS: 70.59777955085929, Tokens per sec: 174949.07471445453, Loss: 2.4983808994293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15542 , TFLOPS: 67.81633713720679, Tokens per sec: 168056.35401224767, Loss: 2.503776788711548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15543 , TFLOPS: 71.53703103959029, Tokens per sec: 177276.64337062286, Loss: 2.4697623252868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15544 , TFLOPS: 71.52338439230873, Tokens per sec: 177242.82547535686, Loss: 2.4930858612060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15545 , TFLOPS: 69.25914872573392, Tokens per sec: 171631.80006743516, Loss: 2.4920437335968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15546 , TFLOPS: 68.7344750089687, Tokens per sec: 170331.60079970967, Loss: 2.484426975250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15547 , TFLOPS: 71.50337073944792, Tokens per sec: 177193.2294388819, Loss: 2.4906296730041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15548 , TFLOPS: 70.67886428695532, Tokens per sec: 175150.01162272153, Loss: 2.4699387550354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15549 , TFLOPS: 68.5624210124114, Tokens per sec: 169905.23204292834, Loss: 2.4724280834198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15550 , TFLOPS: 71.53676056539145, Tokens per sec: 177275.97310576294, Loss: 2.4912147521972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15551 , TFLOPS: 71.47115645736493, Tokens per sec: 177113.3988991841, Loss: 2.481801986694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15552 , TFLOPS: 68.72838182279936, Tokens per sec: 170316.50121316218, Loss: 2.4853079319000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15553 , TFLOPS: 69.68080648190515, Tokens per sec: 172676.7144366638, Loss: 2.4850621223449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15554 , TFLOPS: 71.50369988822054, Tokens per sec: 177194.0451057992, Loss: 2.4715890884399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15555 , TFLOPS: 69.72842249459902, Tokens per sec: 172794.71216145597, Loss: 2.496750831604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15556 , TFLOPS: 69.7183085341368, Tokens per sec: 172769.64865328587, Loss: 2.491575002670288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15557 , TFLOPS: 70.596050535663, Tokens per sec: 174944.7900243314, Loss: 2.507688045501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15558 , TFLOPS: 70.57579804883134, Tokens per sec: 174894.60213096693, Loss: 2.4794464111328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15559 , TFLOPS: 68.85961675822921, Tokens per sec: 170641.71583987895, Loss: 2.523369550704956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15560 , TFLOPS: 71.55581499878079, Tokens per sec: 177323.19209072084, Loss: 2.5075600147247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15561 , TFLOPS: 69.61839567991287, Tokens per sec: 172522.0535942095, Loss: 2.48459792137146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15562 , TFLOPS: 69.66057695966177, Tokens per sec: 172626.58345207883, Loss: 2.4972646236419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15563 , TFLOPS: 70.89542097971496, Tokens per sec: 175686.66296306896, Loss: 2.516430616378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15564 , TFLOPS: 69.8499510011083, Tokens per sec: 173095.87318805614, Loss: 2.4759128093719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15565 , TFLOPS: 70.57260266449832, Tokens per sec: 174886.68361658906, Loss: 2.477633476257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15566 , TFLOPS: 69.19113838444103, Tokens per sec: 171463.26294975376, Loss: 2.5031180381774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15567 , TFLOPS: 71.51620848681392, Tokens per sec: 177225.0427910495, Loss: 2.4821670055389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15568 , TFLOPS: 69.66123405689974, Tokens per sec: 172628.21181141978, Loss: 2.504800796508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15569 , TFLOPS: 68.58215507793874, Tokens per sec: 169954.13523119185, Loss: 2.486704111099243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15570 , TFLOPS: 71.53154708413715, Tokens per sec: 177263.0535249024, Loss: 2.4915943145751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15571 , TFLOPS: 70.56535782637853, Tokens per sec: 174868.7301096434, Loss: 2.520693302154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15572 , TFLOPS: 68.73040245041814, Tokens per sec: 170321.50855099206, Loss: 2.499825954437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15573 , TFLOPS: 70.67729477007335, Tokens per sec: 175146.12218699144, Loss: 2.4789514541625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15574 , TFLOPS: 71.5488874455474, Tokens per sec: 177306.02485067552, Loss: 2.4923722743988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15575 , TFLOPS: 69.61158504777879, Tokens per sec: 172505.17609752822, Loss: 2.4872231483459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15576 , TFLOPS: 69.02318077072127, Tokens per sec: 171047.04548089765, Loss: 2.4933407306671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15577 , TFLOPS: 71.53591030866875, Tokens per sec: 177273.8660759406, Loss: 2.482374668121338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15578 , TFLOPS: 70.60530970102917, Tokens per sec: 174967.73525609, Loss: 2.4899728298187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15579 , TFLOPS: 68.64919963730414, Tokens per sec: 170120.27903486724, Loss: 2.4777169227600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15580 , TFLOPS: 71.01906060391141, Tokens per sec: 175993.05557185676, Loss: 2.4604153633117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15581 , TFLOPS: 71.47322609527171, Tokens per sec: 177118.5276899066, Loss: 2.509045124053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15582 , TFLOPS: 69.99880205586193, Tokens per sec: 173464.74249903296, Loss: 2.4876396656036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15583 , TFLOPS: 68.99898634153563, Tokens per sec: 170987.08902013925, Loss: 2.4937424659729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15584 , TFLOPS: 71.54513791874534, Tokens per sec: 177296.73311021557, Loss: 2.488091230392456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15585 , TFLOPS: 70.62414174352968, Tokens per sec: 175014.403131927, Loss: 2.50030779838562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15586 , TFLOPS: 68.74227913499101, Tokens per sec: 170350.94028368843, Loss: 2.5022592544555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15587 , TFLOPS: 71.57096657002542, Tokens per sec: 177360.7393533484, Loss: 2.494764804840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15588 , TFLOPS: 70.63187161264659, Tokens per sec: 175033.5585991141, Loss: 2.4936633110046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15589 , TFLOPS: 68.81849944914956, Tokens per sec: 170539.82261853403, Loss: 2.4704954624176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15590 , TFLOPS: 70.49835600411554, Tokens per sec: 174702.69221321566, Loss: 2.504507064819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15591 , TFLOPS: 71.50334310129911, Tokens per sec: 177193.16094850458, Loss: 2.507228374481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15592 , TFLOPS: 69.57197039200739, Tokens per sec: 172407.0066166121, Loss: 2.4942548274993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15593 , TFLOPS: 69.93370354550187, Tokens per sec: 173303.42121916782, Loss: 2.483109951019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15594 , TFLOPS: 70.45680173528021, Tokens per sec: 174599.71615746114, Loss: 2.4937782287597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15595 , TFLOPS: 70.70696642774145, Tokens per sec: 175219.65182329592, Loss: 2.493225574493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15596 , TFLOPS: 68.89815491560456, Tokens per sec: 170737.21763918357, Loss: 2.4870433807373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15597 , TFLOPS: 71.52964744008932, Tokens per sec: 177258.34599768886, Loss: 2.5086677074432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15598 , TFLOPS: 70.58191128118344, Tokens per sec: 174909.75139416353, Loss: 2.4780123233795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15599 , TFLOPS: 68.85556849954722, Tokens per sec: 170631.68380891244, Loss: 2.4799461364746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15600 , TFLOPS: 70.58966534816544, Tokens per sec: 174928.96682632595, Loss: 2.500561475753784 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/optimizer.pt +[rank0]:[2024-08-27 10:58:15,299] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006403400999261066, 'preprocessing_with_comm': 0.0012636980100069195, 'state_converting': 0.9833412500011036, : 0.9923651379940566}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600, took 4.29s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013600 + eval ppl=10.080399513244629, eval loss=2.3105928897857666 +------------------------------------------------------------------ +iteration: 15601 , TFLOPS: 64.48978366958549, Tokens per sec: 159812.78806936636, Loss: 2.512122631072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15602 , TFLOPS: 69.08481102701819, Tokens per sec: 171199.77204513462, Loss: 2.468635320663452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15603 , TFLOPS: 70.62344131469456, Tokens per sec: 175012.667392115, Loss: 2.501382827758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15604 , TFLOPS: 70.58065816350684, Tokens per sec: 174906.64603051866, Loss: 2.480245590209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15605 , TFLOPS: 70.6205781321166, Tokens per sec: 175005.5721102812, Loss: 2.4832143783569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15606 , TFLOPS: 68.82952050329276, Tokens per sec: 170567.13400477023, Loss: 2.489434242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15607 , TFLOPS: 71.43621339875278, Tokens per sec: 177026.80615064758, Loss: 2.4969561100006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15608 , TFLOPS: 69.61285184278476, Tokens per sec: 172508.31535510198, Loss: 2.472766876220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15609 , TFLOPS: 68.20962906568467, Tokens per sec: 169030.9747356983, Loss: 2.4510293006896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15610 , TFLOPS: 71.45726290511665, Tokens per sec: 177078.96914621178, Loss: 2.48129940032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15611 , TFLOPS: 71.53970337413017, Tokens per sec: 177283.26570999465, Loss: 2.4672322273254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15612 , TFLOPS: 68.69301340320366, Tokens per sec: 170228.854373251, Loss: 2.474252223968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15613 , TFLOPS: 69.89669575134513, Tokens per sec: 173211.71182850236, Loss: 2.4878008365631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15614 , TFLOPS: 71.49312970181782, Tokens per sec: 177167.8510194911, Loss: 2.517591953277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15615 , TFLOPS: 70.58878406453839, Tokens per sec: 174926.7829084167, Loss: 2.4836931228637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15616 , TFLOPS: 68.46504434967486, Tokens per sec: 169663.92194574224, Loss: 2.5094587802886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15617 , TFLOPS: 71.5074715165621, Tokens per sec: 177203.39161630877, Loss: 2.502056360244751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15618 , TFLOPS: 70.64086233511013, Tokens per sec: 175055.83859978858, Loss: 2.492351770401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15619 , TFLOPS: 68.9235727499703, Tokens per sec: 170800.20583274745, Loss: 2.4991352558135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15620 , TFLOPS: 71.44202341613287, Tokens per sec: 177041.2040137417, Loss: 2.48874831199646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15621 , TFLOPS: 70.57496409393931, Tokens per sec: 174892.53549887676, Loss: 2.479034185409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15622 , TFLOPS: 70.3872846485136, Tokens per sec: 174427.4451585142, Loss: 2.500756025314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15623 , TFLOPS: 69.87328282136492, Tokens per sec: 173153.69200886425, Loss: 2.4615941047668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15624 , TFLOPS: 71.4429127428474, Tokens per sec: 177043.4078633077, Loss: 2.490860939025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15625 , TFLOPS: 70.60230145148428, Tokens per sec: 174960.28048233173, Loss: 2.4853694438934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15626 , TFLOPS: 69.14105025798627, Tokens per sec: 171339.13905473627, Loss: 2.5078043937683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15627 , TFLOPS: 70.55204340361685, Tokens per sec: 174835.7355033352, Loss: 2.502180576324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15628 , TFLOPS: 70.53305477787814, Tokens per sec: 174788.67959698488, Loss: 2.4946787357330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15629 , TFLOPS: 68.75825597161065, Tokens per sec: 170390.53264482573, Loss: 2.4917781352996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15630 , TFLOPS: 71.447061295363, Tokens per sec: 177053.68843344215, Loss: 2.4880177974700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15631 , TFLOPS: 70.42757878163306, Tokens per sec: 174527.2984591494, Loss: 2.509474039077759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15632 , TFLOPS: 69.73738398715408, Tokens per sec: 172816.91972719732, Loss: 2.4970250129699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15633 , TFLOPS: 69.6596551324718, Tokens per sec: 172624.29906275446, Loss: 2.465947389602661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15634 , TFLOPS: 70.50999023009872, Tokens per sec: 174731.52310681783, Loss: 2.5062167644500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15635 , TFLOPS: 70.58367518360065, Tokens per sec: 174914.1225386633, Loss: 2.461378812789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15636 , TFLOPS: 68.79243767158405, Tokens per sec: 170475.23866278495, Loss: 2.501437187194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15637 , TFLOPS: 71.51793600378952, Tokens per sec: 177229.3237684167, Loss: 2.493173599243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15638 , TFLOPS: 69.63154735394586, Tokens per sec: 172554.64489123976, Loss: 2.4808971881866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15639 , TFLOPS: 69.67877955564666, Tokens per sec: 172671.69149011088, Loss: 2.475681781768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15640 , TFLOPS: 70.87307833432543, Tokens per sec: 175631.29542654814, Loss: 2.500290632247925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15641 , TFLOPS: 70.51992542151999, Tokens per sec: 174756.14360561137, Loss: 2.4733033180236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15642 , TFLOPS: 70.82518423779727, Tokens per sec: 175512.6085793811, Loss: 2.4716081619262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15643 , TFLOPS: 69.8811473288944, Tokens per sec: 173173.1811248693, Loss: 2.475548028945923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15644 , TFLOPS: 71.51462408837709, Tokens per sec: 177221.1164771871, Loss: 2.4960622787475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15645 , TFLOPS: 69.60993812018098, Tokens per sec: 172501.09482951678, Loss: 2.4817636013031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15646 , TFLOPS: 68.82494305450902, Tokens per sec: 170555.79058243652, Loss: 2.470919370651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15647 , TFLOPS: 71.49638874437562, Tokens per sec: 177175.92728596253, Loss: 2.482442617416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15648 , TFLOPS: 70.60267634366012, Tokens per sec: 174961.20950644126, Loss: 2.4843873977661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15649 , TFLOPS: 67.82026901454641, Tokens per sec: 168066.09763742727, Loss: 2.4781744480133057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15650 , TFLOPS: 71.53653376383052, Tokens per sec: 177275.4110665107, Loss: 2.4728612899780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15651 , TFLOPS: 71.51275577049383, Tokens per sec: 177216.4865796602, Loss: 2.487591028213501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15652 , TFLOPS: 69.68338728470272, Tokens per sec: 172683.10995029504, Loss: 2.478553533554077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15653 , TFLOPS: 69.48229585941812, Tokens per sec: 172184.7832463906, Loss: 2.4904351234436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15654 , TFLOPS: 71.49336122904323, Tokens per sec: 177168.4247694605, Loss: 2.468172073364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15655 , TFLOPS: 70.58656178168708, Tokens per sec: 174921.2758467065, Loss: 2.5036394596099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15656 , TFLOPS: 67.92132490276272, Tokens per sec: 168316.5252606524, Loss: 2.4833805561065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15657 , TFLOPS: 71.53183362966567, Tokens per sec: 177263.7636162877, Loss: 2.52443265914917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15658 , TFLOPS: 71.50937660888505, Tokens per sec: 177208.11264494827, Loss: 2.4974284172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15659 , TFLOPS: 69.1139831341921, Tokens per sec: 171272.06374028447, Loss: 2.491962432861328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15660 , TFLOPS: 69.62358585551722, Tokens per sec: 172534.9154211021, Loss: 2.4730677604675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15661 , TFLOPS: 71.52613756340101, Tokens per sec: 177249.64813661034, Loss: 2.5110416412353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15662 , TFLOPS: 70.58410275383919, Tokens per sec: 174915.18210481515, Loss: 2.5075020790100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15663 , TFLOPS: 69.58922852907436, Tokens per sec: 172449.7741814048, Loss: 2.503309488296509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15664 , TFLOPS: 70.46338482069243, Tokens per sec: 174616.02976829934, Loss: 2.505591630935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15665 , TFLOPS: 70.68541213568228, Tokens per sec: 175166.23791317205, Loss: 2.4766945838928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15666 , TFLOPS: 68.78611906568993, Tokens per sec: 170459.5804613274, Loss: 2.503823757171631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15667 , TFLOPS: 71.50329755921324, Tokens per sec: 177193.04809019892, Loss: 2.4772515296936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15668 , TFLOPS: 69.59437423247952, Tokens per sec: 172462.52580128878, Loss: 2.473423719406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15669 , TFLOPS: 69.6457491030855, Tokens per sec: 172589.83839005962, Loss: 2.489536762237549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15670 , TFLOPS: 70.9926774239665, Tokens per sec: 175927.67514560604, Loss: 2.504420518875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15671 , TFLOPS: 70.37425960161832, Tokens per sec: 174395.16765748034, Loss: 2.47802734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15672 , TFLOPS: 70.64542602994707, Tokens per sec: 175067.14793832702, Loss: 2.5047662258148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15673 , TFLOPS: 68.92156211997852, Tokens per sec: 170795.22327013864, Loss: 2.486433267593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15674 , TFLOPS: 71.54877966278276, Tokens per sec: 177305.7577531115, Loss: 2.4985594749450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15675 , TFLOPS: 70.41732388830572, Tokens per sec: 174501.8857038145, Loss: 2.471630334854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15676 , TFLOPS: 68.94695187050911, Tokens per sec: 170858.14186886535, Loss: 2.4891762733459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15677 , TFLOPS: 71.46634696631102, Tokens per sec: 177101.48045054337, Loss: 2.5001583099365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15678 , TFLOPS: 70.5477659781643, Tokens per sec: 174825.1355718665, Loss: 2.4925143718719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15679 , TFLOPS: 69.81214318738148, Tokens per sec: 173002.1812607666, Loss: 2.4829936027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15680 , TFLOPS: 69.92988650364612, Tokens per sec: 173293.9621689674, Loss: 2.4753425121307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15681 , TFLOPS: 71.55477440167998, Tokens per sec: 177320.6133764738, Loss: 2.521653413772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15682 , TFLOPS: 69.53774588310984, Tokens per sec: 172322.19451342305, Loss: 2.4868404865264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15683 , TFLOPS: 69.65635243328697, Tokens per sec: 172616.1146103521, Loss: 2.487245798110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15684 , TFLOPS: 71.52609284272181, Tokens per sec: 177249.5373138409, Loss: 2.5247180461883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15685 , TFLOPS: 70.68428194641885, Tokens per sec: 175163.4371796766, Loss: 2.506680727005005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15686 , TFLOPS: 67.84237767345051, Tokens per sec: 168120.88532966148, Loss: 2.4696779251098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15687 , TFLOPS: 71.51032750751469, Tokens per sec: 177210.46907650263, Loss: 2.5237374305725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15688 , TFLOPS: 71.50416440433835, Tokens per sec: 177195.19622790982, Loss: 2.512171506881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15689 , TFLOPS: 68.72112670983657, Tokens per sec: 170298.5222440237, Loss: 2.495232105255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15690 , TFLOPS: 69.53564940516841, Tokens per sec: 172316.9992101381, Loss: 2.501796007156372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15691 , TFLOPS: 71.54927984514887, Tokens per sec: 177306.99726011435, Loss: 2.4961166381835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15692 , TFLOPS: 70.66696759752567, Tokens per sec: 175120.53031578634, Loss: 2.501497268676758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15693 , TFLOPS: 67.78176764195747, Tokens per sec: 167970.6869948756, Loss: 2.4919333457946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15694 , TFLOPS: 71.51042592159615, Tokens per sec: 177210.7129574377, Loss: 2.5107882022857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15695 , TFLOPS: 70.60558310198383, Tokens per sec: 174968.41277377325, Loss: 2.482758045196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15696 , TFLOPS: 68.82645870434476, Tokens per sec: 170559.5465296921, Loss: 2.492093563079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15697 , TFLOPS: 70.47693829780326, Tokens per sec: 174649.61677761874, Loss: 2.490875005722046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15698 , TFLOPS: 71.45994686652506, Tokens per sec: 177085.62029824455, Loss: 2.518702745437622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15699 , TFLOPS: 69.61738305084315, Tokens per sec: 172519.54418782442, Loss: 2.496795177459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15700 , TFLOPS: 69.58246502924787, Tokens per sec: 172433.01348377374, Loss: 2.4949145317077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15701 , TFLOPS: 70.52031753995237, Tokens per sec: 174757.11531828225, Loss: 2.4986572265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15702 , TFLOPS: 70.6070221733807, Tokens per sec: 174971.97895122148, Loss: 2.489657163619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15703 , TFLOPS: 69.7467731531797, Tokens per sec: 172840.18711491086, Loss: 2.4833669662475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15704 , TFLOPS: 71.53650468896446, Tokens per sec: 177275.33901578977, Loss: 2.4792728424072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15705 , TFLOPS: 69.55018471078921, Tokens per sec: 172353.01929866255, Loss: 2.4943370819091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15706 , TFLOPS: 69.61013425402909, Tokens per sec: 172501.58087079832, Loss: 2.511566638946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15707 , TFLOPS: 70.97112896593859, Tokens per sec: 175874.27569285428, Loss: 2.4937918186187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15708 , TFLOPS: 70.57503742777737, Tokens per sec: 174892.71722820587, Loss: 2.5044515132904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15709 , TFLOPS: 69.6210094802305, Tokens per sec: 172528.53087933047, Loss: 2.4862208366394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15710 , TFLOPS: 70.09192057807186, Tokens per sec: 173695.50045492122, Loss: 2.4829466342926025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15711 , TFLOPS: 71.52497573989578, Tokens per sec: 177246.769009979, Loss: 2.49739933013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15712 , TFLOPS: 69.56103535348755, Tokens per sec: 172379.90838656598, Loss: 2.4784398078918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15713 , TFLOPS: 68.36237273533236, Tokens per sec: 169409.49037518955, Loss: 2.5065901279449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15714 , TFLOPS: 71.559565199776, Tokens per sec: 177332.48550190573, Loss: 2.4939868450164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15715 , TFLOPS: 69.65093906991733, Tokens per sec: 172602.69969959088, Loss: 2.4907524585723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15716 , TFLOPS: 68.98686241536838, Tokens per sec: 170957.04459553608, Loss: 2.4925947189331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15717 , TFLOPS: 71.52068442693451, Tokens per sec: 177236.1346637338, Loss: 2.4949162006378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15718 , TFLOPS: 71.55866010302805, Tokens per sec: 177330.24257245968, Loss: 2.4874792098999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15719 , TFLOPS: 68.67372954222088, Tokens per sec: 170181.066841443, Loss: 2.520446300506592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15720 , TFLOPS: 69.6740340731414, Tokens per sec: 172659.9316617048, Loss: 2.4920413494110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15721 , TFLOPS: 71.5039239515455, Tokens per sec: 177194.60035940105, Loss: 2.5064187049865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15722 , TFLOPS: 70.58244777676083, Tokens per sec: 174911.0808893042, Loss: 2.511392831802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15723 , TFLOPS: 68.65349324994804, Tokens per sec: 170130.9190799793, Loss: 2.496898889541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15724 , TFLOPS: 71.53031102777065, Tokens per sec: 177259.9904410634, Loss: 2.50447940826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15725 , TFLOPS: 71.52065613780121, Tokens per sec: 177236.06456014517, Loss: 2.4963104724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15726 , TFLOPS: 68.93722697038774, Tokens per sec: 170834.04249507844, Loss: 2.497593641281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15727 , TFLOPS: 69.62697530329804, Tokens per sec: 172543.31484608148, Loss: 2.510533094406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15728 , TFLOPS: 71.49669401931389, Tokens per sec: 177176.68379088838, Loss: 2.4871134757995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15729 , TFLOPS: 70.55886423758241, Tokens per sec: 174852.63828127878, Loss: 2.5017356872558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15730 , TFLOPS: 68.13142228464339, Tokens per sec: 168837.1697170899, Loss: 2.4923019409179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15731 , TFLOPS: 71.53342135190266, Tokens per sec: 177267.69816689298, Loss: 2.4810032844543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15732 , TFLOPS: 70.60813279263014, Tokens per sec: 174974.73118806645, Loss: 2.482830047607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15733 , TFLOPS: 68.79781202389717, Tokens per sec: 170488.5568998506, Loss: 2.494722366333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15734 , TFLOPS: 70.55673163322191, Tokens per sec: 174847.3534527477, Loss: 2.498300075531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15735 , TFLOPS: 70.67801872334631, Tokens per sec: 175147.9162229519, Loss: 2.502148151397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15736 , TFLOPS: 68.87006584970763, Tokens per sec: 170667.6098396249, Loss: 2.4988625049591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15737 , TFLOPS: 71.50022586151617, Tokens per sec: 177185.43608493012, Loss: 2.4972760677337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15738 , TFLOPS: 70.49668828973705, Tokens per sec: 174698.5594332719, Loss: 2.4893429279327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15739 , TFLOPS: 69.73089288750631, Tokens per sec: 172800.83406721652, Loss: 2.5004069805145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15740 , TFLOPS: 69.57230472755494, Tokens per sec: 172407.835136929, Loss: 2.4883766174316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15741 , TFLOPS: 71.50216981213283, Tokens per sec: 177190.25340870206, Loss: 2.4977383613586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15742 , TFLOPS: 69.71479625684677, Tokens per sec: 172760.94484325193, Loss: 2.475639820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15743 , TFLOPS: 69.6078917489195, Tokens per sec: 172496.0236961044, Loss: 2.4903485774993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15744 , TFLOPS: 71.52080342951506, Tokens per sec: 177236.42956523763, Loss: 2.4951930046081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15745 , TFLOPS: 70.43562634695694, Tokens per sec: 174547.24121253449, Loss: 2.4757957458496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15746 , TFLOPS: 69.70809604309805, Tokens per sec: 172744.34097549313, Loss: 2.481719493865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15747 , TFLOPS: 69.94262325803993, Tokens per sec: 173325.5252494246, Loss: 2.4717960357666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15748 , TFLOPS: 71.5617729994975, Tokens per sec: 177337.95667282483, Loss: 2.484703302383423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15749 , TFLOPS: 69.6847042602095, Tokens per sec: 172686.3735606777, Loss: 2.5087714195251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15750 , TFLOPS: 68.81821554287941, Tokens per sec: 170539.1190675218, Loss: 2.4706308841705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15751 , TFLOPS: 71.55020600688604, Tokens per sec: 177309.2923909242, Loss: 2.4630613327026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15752 , TFLOPS: 69.4780437385324, Tokens per sec: 172174.24602242606, Loss: 2.4766385555267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15753 , TFLOPS: 68.80750897494279, Tokens per sec: 170512.58701274902, Loss: 2.5005462169647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15754 , TFLOPS: 71.52113482568173, Tokens per sec: 177237.25080144493, Loss: 2.5184364318847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15755 , TFLOPS: 70.62431590830391, Tokens per sec: 175014.8347314234, Loss: 2.461719512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15756 , TFLOPS: 69.45340250482354, Tokens per sec: 172113.1823308395, Loss: 2.485599994659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15757 , TFLOPS: 70.92092184972556, Tokens per sec: 175749.85692810797, Loss: 2.4807512760162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15758 , TFLOPS: 71.57647398003006, Tokens per sec: 177374.3873220312, Loss: 2.494640827178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15759 , TFLOPS: 70.57095666416548, Tokens per sec: 174882.60464644237, Loss: 2.4848923683166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15760 , TFLOPS: 68.2033141865427, Tokens per sec: 169015.32576954318, Loss: 2.471721887588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15761 , TFLOPS: 71.5493208334282, Tokens per sec: 177307.09883358577, Loss: 2.510253667831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15762 , TFLOPS: 70.645169712497, Tokens per sec: 175066.51275545, Loss: 2.4885332584381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15763 , TFLOPS: 69.66036613321896, Tokens per sec: 172626.06100092913, Loss: 2.4796648025512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15764 , TFLOPS: 70.44665266993779, Tokens per sec: 174574.56565553078, Loss: 2.4941282272338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15765 , TFLOPS: 71.50049154994402, Tokens per sec: 177186.0944901223, Loss: 2.4696288108825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15766 , TFLOPS: 69.74993965743667, Tokens per sec: 172848.0340612789, Loss: 2.478634834289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15767 , TFLOPS: 68.7383834189865, Tokens per sec: 170341.2862702807, Loss: 2.4853994846343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15768 , TFLOPS: 71.51854750412078, Tokens per sec: 177230.8391335999, Loss: 2.4801225662231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15769 , TFLOPS: 70.6548854050995, Tokens per sec: 175090.58931199095, Loss: 2.493928909301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15770 , TFLOPS: 68.72202951998216, Tokens per sec: 170300.75950701724, Loss: 2.4941084384918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15771 , TFLOPS: 70.44891036078282, Tokens per sec: 174580.16046214994, Loss: 2.4969418048858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15772 , TFLOPS: 70.576014311634, Tokens per sec: 174895.1380540152, Loss: 2.4866995811462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15773 , TFLOPS: 69.67039046518742, Tokens per sec: 172650.9023998183, Loss: 2.5028419494628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15774 , TFLOPS: 70.99265729989587, Tokens per sec: 175927.6252759422, Loss: 2.481044292449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15775 , TFLOPS: 70.60079219779698, Tokens per sec: 174956.5403854366, Loss: 2.4722440242767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15776 , TFLOPS: 69.88001455267874, Tokens per sec: 173170.3739806213, Loss: 2.522850513458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15777 , TFLOPS: 69.93757030138747, Tokens per sec: 173313.00346621065, Loss: 2.4898805618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15778 , TFLOPS: 71.49732360789608, Tokens per sec: 177178.24398074992, Loss: 2.48199200630188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15779 , TFLOPS: 69.53504071239847, Tokens per sec: 172315.49080240112, Loss: 2.5103325843811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15780 , TFLOPS: 68.7558598527867, Tokens per sec: 170384.5947984257, Loss: 2.4710469245910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15781 , TFLOPS: 71.55630569530767, Tokens per sec: 177324.40809076914, Loss: 2.4896788597106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15782 , TFLOPS: 69.72371470850308, Tokens per sec: 172783.0457489048, Loss: 2.4751882553100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15783 , TFLOPS: 69.60943246962033, Tokens per sec: 172499.84177172548, Loss: 2.4977927207946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15784 , TFLOPS: 71.50547534939602, Tokens per sec: 177198.44489417522, Loss: 2.488337755203247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15785 , TFLOPS: 71.53607159163965, Tokens per sec: 177274.26575290927, Loss: 2.486069917678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15786 , TFLOPS: 68.80209451786914, Tokens per sec: 170499.16939166878, Loss: 2.4841320514678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15787 , TFLOPS: 69.71323097671684, Tokens per sec: 172757.06590666014, Loss: 2.466827154159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15788 , TFLOPS: 71.48319515490458, Tokens per sec: 177143.2321178594, Loss: 2.477048873901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15789 , TFLOPS: 69.63675294583791, Tokens per sec: 172567.54492139156, Loss: 2.4985451698303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15790 , TFLOPS: 68.6804140698907, Tokens per sec: 170197.63183737034, Loss: 2.4725403785705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15791 , TFLOPS: 71.54828231076917, Tokens per sec: 177304.52526003396, Loss: 2.495616912841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15792 , TFLOPS: 71.48324469606327, Tokens per sec: 177143.35488630805, Loss: 2.4953620433807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15793 , TFLOPS: 68.76714645080237, Tokens per sec: 170412.56423162998, Loss: 2.458758592605591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15794 , TFLOPS: 70.69644185347512, Tokens per sec: 175193.57076888642, Loss: 2.4811670780181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15795 , TFLOPS: 71.52814105975548, Tokens per sec: 177254.61302128006, Loss: 2.5149617195129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15796 , TFLOPS: 70.72618177536103, Tokens per sec: 175267.26957144254, Loss: 2.506993293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15797 , TFLOPS: 68.01259004855463, Tokens per sec: 168542.6903456105, Loss: 2.496751308441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15798 , TFLOPS: 71.55411222634486, Tokens per sec: 177318.97243304874, Loss: 2.5002589225769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15799 , TFLOPS: 70.61356150135151, Tokens per sec: 174988.18412629963, Loss: 2.4835734367370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15800 , TFLOPS: 68.80900458503042, Tokens per sec: 170516.2932993012, Loss: 2.481666088104248 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/model.pt +[rank0]:[2024-08-27 11:23:30,266] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006319671010714956, 'preprocessing_with_comm': 0.0012513199908426031, 'state_converting': 0.9736100709997118, : 0.9825489680079045}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0013800 + eval ppl=10.430150985717773, eval loss=2.344700813293457 +------------------------------------------------------------------ +iteration: 15801 , TFLOPS: 64.76590375544329, Tokens per sec: 160497.04405926043, Loss: 2.4630236625671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15802 , TFLOPS: 70.60465371119386, Tokens per sec: 174966.10964101486, Loss: 2.4982452392578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15803 , TFLOPS: 69.82754238747003, Tokens per sec: 173040.34217494793, Loss: 2.4803388118743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15804 , TFLOPS: 70.77053649142279, Tokens per sec: 175377.18544391892, Loss: 2.5149519443511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15805 , TFLOPS: 70.5488831386327, Tokens per sec: 174827.9040185726, Loss: 2.4733312129974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15806 , TFLOPS: 69.61164515358631, Tokens per sec: 172505.32504634044, Loss: 2.514383316040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15807 , TFLOPS: 69.60677131934251, Tokens per sec: 172493.24714818713, Loss: 2.515591859817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15808 , TFLOPS: 70.43657658503471, Tokens per sec: 174549.59600716858, Loss: 2.4807777404785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15809 , TFLOPS: 70.65919752943405, Tokens per sec: 175101.27523111165, Loss: 2.4773483276367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15810 , TFLOPS: 68.58447709034758, Tokens per sec: 169959.88943373173, Loss: 2.4787306785583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15811 , TFLOPS: 71.49811602812679, Tokens per sec: 177180.2076853739, Loss: 2.499176025390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15812 , TFLOPS: 70.50219769204516, Tokens per sec: 174712.21233910215, Loss: 2.498772621154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15813 , TFLOPS: 69.1548956091184, Tokens per sec: 171373.449360033, Loss: 2.475710868835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15814 , TFLOPS: 70.50524944263991, Tokens per sec: 174719.77491325402, Loss: 2.48099946975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15815 , TFLOPS: 71.51125013048448, Tokens per sec: 177212.75543785712, Loss: 2.4894297122955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15816 , TFLOPS: 69.53957345079753, Tokens per sec: 172326.72342747636, Loss: 2.4781246185302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15817 , TFLOPS: 69.97324481073129, Tokens per sec: 173401.40883023627, Loss: 2.477872133255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15818 , TFLOPS: 71.52839821960899, Tokens per sec: 177255.25029172556, Loss: 2.5095982551574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15819 , TFLOPS: 70.56561770353323, Tokens per sec: 174869.3741138606, Loss: 2.48138689994812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15820 , TFLOPS: 68.06888469024462, Tokens per sec: 168682.19466908503, Loss: 2.48347806930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15821 , TFLOPS: 71.52850905288626, Tokens per sec: 177255.52494879594, Loss: 2.4607183933258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15822 , TFLOPS: 70.54502575828859, Tokens per sec: 174818.34500515435, Loss: 2.4928808212280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15823 , TFLOPS: 67.89975557149111, Tokens per sec: 168263.07408170332, Loss: 2.4964258670806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15824 , TFLOPS: 71.45425421911004, Tokens per sec: 177071.51329085327, Loss: 2.486781120300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15825 , TFLOPS: 71.51001146249139, Tokens per sec: 177209.68588211967, Loss: 2.503540515899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15826 , TFLOPS: 69.68390384246956, Tokens per sec: 172684.39003734483, Loss: 2.491551637649536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15827 , TFLOPS: 68.46217905088781, Tokens per sec: 169656.82141970925, Loss: 2.5143795013427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15828 , TFLOPS: 71.50623120263963, Tokens per sec: 177200.317981777, Loss: 2.4963598251342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15829 , TFLOPS: 70.61816191822528, Tokens per sec: 174999.5844660906, Loss: 2.51739764213562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15830 , TFLOPS: 68.02011620858549, Tokens per sec: 168561.34099924195, Loss: 2.476465940475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15831 , TFLOPS: 71.53103638343315, Tokens per sec: 177261.78795229955, Loss: 2.4800682067871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15832 , TFLOPS: 71.52923142456542, Tokens per sec: 177257.31506539273, Loss: 2.507939577102661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15833 , TFLOPS: 69.77905977932699, Tokens per sec: 172920.1969311693, Loss: 2.501394510269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15834 , TFLOPS: 68.66934742247902, Tokens per sec: 170170.20746599237, Loss: 2.493978500366211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15835 , TFLOPS: 71.4718959260473, Tokens per sec: 177115.23138403826, Loss: 2.4727447032928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15836 , TFLOPS: 70.59954509382469, Tokens per sec: 174953.4499244134, Loss: 2.4973199367523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15837 , TFLOPS: 68.99742603697744, Tokens per sec: 170983.22241356233, Loss: 2.488551139831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15838 , TFLOPS: 70.44536265806175, Tokens per sec: 174571.36886399542, Loss: 2.4701051712036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15839 , TFLOPS: 70.71253130081944, Tokens per sec: 175233.44219180456, Loss: 2.4876840114593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15840 , TFLOPS: 68.84997880327951, Tokens per sec: 170617.8319258052, Loss: 2.463045597076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15841 , TFLOPS: 71.55510685748118, Tokens per sec: 177321.4372385722, Loss: 2.499375343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15842 , TFLOPS: 70.53861624588994, Tokens per sec: 174802.46152736442, Loss: 2.489485502243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15843 , TFLOPS: 69.17646376538386, Tokens per sec: 171426.89762718737, Loss: 2.474437713623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15844 , TFLOPS: 70.74658793742327, Tokens per sec: 175317.8382889571, Loss: 2.495204448699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15845 , TFLOPS: 70.52693634502145, Tokens per sec: 174773.517446365, Loss: 2.4869327545166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15846 , TFLOPS: 69.81606762340337, Tokens per sec: 173011.90644554116, Loss: 2.489504814147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15847 , TFLOPS: 69.56704244823962, Tokens per sec: 172394.79462909774, Loss: 2.484929323196411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15848 , TFLOPS: 71.46719205861748, Tokens per sec: 177103.57468237338, Loss: 2.4858880043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15849 , TFLOPS: 69.54592418569008, Tokens per sec: 172342.4612481313, Loss: 2.4850597381591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15850 , TFLOPS: 68.77137517377393, Tokens per sec: 170423.0434729858, Loss: 2.5076277256011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15851 , TFLOPS: 71.54744102358872, Tokens per sec: 177302.44045772537, Loss: 2.4845449924468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15852 , TFLOPS: 71.46076280368852, Tokens per sec: 177087.6422804185, Loss: 2.491248846054077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15853 , TFLOPS: 68.90304943552846, Tokens per sec: 170749.34679873087, Loss: 2.5157840251922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15854 , TFLOPS: 69.54265459605051, Tokens per sec: 172334.3588448294, Loss: 2.463460683822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15855 , TFLOPS: 71.51947471579, Tokens per sec: 177233.13686625703, Loss: 2.4889614582061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15856 , TFLOPS: 70.63363963381234, Tokens per sec: 175037.93995032643, Loss: 2.5152130126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15857 , TFLOPS: 67.78686019191731, Tokens per sec: 167983.30689466678, Loss: 2.505492925643921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15858 , TFLOPS: 71.53767516134276, Tokens per sec: 177278.23957528136, Loss: 2.4960148334503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15859 , TFLOPS: 70.72658065987298, Tokens per sec: 175268.2580512039, Loss: 2.4796013832092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15860 , TFLOPS: 68.80515991667589, Tokens per sec: 170506.7657875935, Loss: 2.4903597831726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15861 , TFLOPS: 70.69224276477884, Tokens per sec: 175183.16496452925, Loss: 2.5061373710632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15862 , TFLOPS: 71.54074332993558, Tokens per sec: 177285.84283504102, Loss: 2.4707536697387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15863 , TFLOPS: 68.92771629713903, Tokens per sec: 170810.47399908098, Loss: 2.504465341567993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15864 , TFLOPS: 69.46961385368805, Tokens per sec: 172153.35583915323, Loss: 2.4605259895324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15865 , TFLOPS: 71.51550941778223, Tokens per sec: 177223.31042098012, Loss: 2.4549875259399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15866 , TFLOPS: 69.74096997711952, Tokens per sec: 172825.80620536112, Loss: 2.47800350189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15867 , TFLOPS: 68.71003385777392, Tokens per sec: 170271.032934634, Loss: 2.489304304122925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15868 , TFLOPS: 71.51959753026154, Tokens per sec: 177233.44121404664, Loss: 2.4982688426971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15869 , TFLOPS: 70.61937371577814, Tokens per sec: 175002.58743391707, Loss: 2.462005615234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15870 , TFLOPS: 68.9273740664313, Tokens per sec: 170809.6259136874, Loss: 2.487338066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15871 , TFLOPS: 70.46472458871189, Tokens per sec: 174619.3498610392, Loss: 2.4880073070526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15872 , TFLOPS: 71.49600174266264, Tokens per sec: 177174.968253086, Loss: 2.5227205753326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15873 , TFLOPS: 69.69592693972174, Tokens per sec: 172714.18459678921, Loss: 2.498850107192993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15874 , TFLOPS: 69.53278235900127, Tokens per sec: 172309.89435390473, Loss: 2.4932703971862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15875 , TFLOPS: 70.59256027679253, Tokens per sec: 174936.14077836656, Loss: 2.5032901763916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15876 , TFLOPS: 70.56830186447867, Tokens per sec: 174876.02576036812, Loss: 2.5196373462677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15877 , TFLOPS: 68.77030144988353, Tokens per sec: 170420.38266690494, Loss: 2.4827511310577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15878 , TFLOPS: 71.51541633195718, Tokens per sec: 177223.07974405144, Loss: 2.516294479370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15879 , TFLOPS: 70.42188710857727, Tokens per sec: 174513.1938663276, Loss: 2.4793546199798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15880 , TFLOPS: 69.07905446729835, Tokens per sec: 171185.50665601145, Loss: 2.489834785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15881 , TFLOPS: 69.92787760884703, Tokens per sec: 173288.9839063572, Loss: 2.479201555252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15882 , TFLOPS: 70.51004422543733, Tokens per sec: 174731.65691321495, Loss: 2.4798359870910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15883 , TFLOPS: 69.57418753407056, Tokens per sec: 172412.500938883, Loss: 2.4961390495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15884 , TFLOPS: 69.87437118771105, Tokens per sec: 173156.3891005634, Loss: 2.5034663677215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15885 , TFLOPS: 71.52819021857786, Tokens per sec: 177254.73484225722, Loss: 2.4758973121643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15886 , TFLOPS: 69.60581949036208, Tokens per sec: 172490.888411121, Loss: 2.488253593444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15887 , TFLOPS: 69.10730026523193, Tokens per sec: 171255.50285482238, Loss: 2.4982566833496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15888 , TFLOPS: 71.51580962013293, Tokens per sec: 177224.05435547518, Loss: 2.4973459243774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15889 , TFLOPS: 70.65422610835316, Tokens per sec: 175088.95550202596, Loss: 2.503573179244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15890 , TFLOPS: 68.81622992858397, Tokens per sec: 170534.19849656418, Loss: 2.4945640563964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15891 , TFLOPS: 70.74775850307498, Tokens per sec: 175320.73907959048, Loss: 2.491783857345581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15892 , TFLOPS: 71.50590852042131, Tokens per sec: 177199.51833969363, Loss: 2.4791994094848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15893 , TFLOPS: 69.72121365256433, Tokens per sec: 172776.84785677455, Loss: 2.493443489074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15894 , TFLOPS: 68.66104894001589, Tokens per sec: 170149.6429122954, Loss: 2.4763426780700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15895 , TFLOPS: 71.50370165550515, Tokens per sec: 177194.04948532514, Loss: 2.4946045875549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15896 , TFLOPS: 70.5832250436669, Tokens per sec: 174913.00704232047, Loss: 2.4908852577209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15897 , TFLOPS: 67.89281439616131, Tokens per sec: 168245.8730846025, Loss: 2.509675979614258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15898 , TFLOPS: 71.50564780395067, Tokens per sec: 177198.87225555908, Loss: 2.4749011993408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15899 , TFLOPS: 71.49306041393133, Tokens per sec: 177167.67931647567, Loss: 2.4918179512023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15900 , TFLOPS: 69.71034020881615, Tokens per sec: 172749.90226535784, Loss: 2.4687299728393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15901 , TFLOPS: 68.90175150024696, Tokens per sec: 170746.1303721239, Loss: 2.5168075561523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15902 , TFLOPS: 71.55233591586945, Tokens per sec: 177314.57054001396, Loss: 2.49521541595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15903 , TFLOPS: 70.57927866210855, Tokens per sec: 174903.22747408907, Loss: 2.501199245452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15904 , TFLOPS: 68.0302288344963, Tokens per sec: 168586.40120024662, Loss: 2.4797494411468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15905 , TFLOPS: 71.45707870526654, Tokens per sec: 177078.5126786919, Loss: 2.468169689178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15906 , TFLOPS: 70.46317146082896, Tokens per sec: 174615.50103905416, Loss: 2.461900234222412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15907 , TFLOPS: 68.90464383867898, Tokens per sec: 170753.29790537583, Loss: 2.4876794815063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15908 , TFLOPS: 70.51461563915117, Tokens per sec: 174742.98537998067, Loss: 2.4940719604492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15909 , TFLOPS: 69.92618820513067, Tokens per sec: 173284.7973778449, Loss: 2.4823520183563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15910 , TFLOPS: 69.83982554514371, Tokens per sec: 173070.78119276487, Loss: 2.4830617904663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15911 , TFLOPS: 70.55271074201984, Tokens per sec: 174837.38924141123, Loss: 2.502079963684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15912 , TFLOPS: 70.44354508527171, Tokens per sec: 174566.86471839953, Loss: 2.482639789581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15913 , TFLOPS: 69.21091875549675, Tokens per sec: 171512.28088821762, Loss: 2.4618258476257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15914 , TFLOPS: 69.64582298312013, Tokens per sec: 172590.02147292395, Loss: 2.468778610229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15915 , TFLOPS: 71.50287561194176, Tokens per sec: 177192.00245837885, Loss: 2.5184366703033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15916 , TFLOPS: 69.0020785915308, Tokens per sec: 170994.75195626676, Loss: 2.493051767349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15917 , TFLOPS: 68.83627402332111, Tokens per sec: 170583.86997136215, Loss: 2.479454755783081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15918 , TFLOPS: 71.53362159339846, Tokens per sec: 177268.19438737805, Loss: 2.490062952041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15919 , TFLOPS: 70.45361971392259, Tokens per sec: 174591.83075800838, Loss: 2.4787933826446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15920 , TFLOPS: 69.6627410709081, Tokens per sec: 172631.94635814376, Loss: 2.477952480316162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15921 , TFLOPS: 69.65150741366742, Tokens per sec: 172604.10811801188, Loss: 2.481990337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15922 , TFLOPS: 71.556464671556, Tokens per sec: 177324.80205142518, Loss: 2.478315591812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15923 , TFLOPS: 69.21295422999745, Tokens per sec: 171517.32501825635, Loss: 2.472304344177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15924 , TFLOPS: 68.7881104590935, Tokens per sec: 170464.51535355154, Loss: 2.4973647594451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15925 , TFLOPS: 71.5207269865092, Tokens per sec: 177236.24013104822, Loss: 2.4785654544830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15926 , TFLOPS: 70.53591178890908, Tokens per sec: 174795.75958504528, Loss: 2.504317283630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15927 , TFLOPS: 68.52399324973311, Tokens per sec: 169810.003813844, Loss: 2.4740755558013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15928 , TFLOPS: 70.69224530395842, Tokens per sec: 175183.1712568959, Loss: 2.487997531890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15929 , TFLOPS: 71.54223790533032, Tokens per sec: 177289.54655751027, Loss: 2.500556230545044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15930 , TFLOPS: 70.0599459131166, Tokens per sec: 173616.26371286195, Loss: 2.4867002964019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15931 , TFLOPS: 69.56523104115062, Tokens per sec: 172390.30576278822, Loss: 2.4905612468719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15932 , TFLOPS: 71.52565672140851, Tokens per sec: 177248.45655718385, Loss: 2.482912302017212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15933 , TFLOPS: 70.5775998911614, Tokens per sec: 174899.06729475013, Loss: 2.4845337867736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15934 , TFLOPS: 68.83753273412395, Tokens per sec: 170586.98919539005, Loss: 2.4656150341033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15935 , TFLOPS: 71.5094374000521, Tokens per sec: 177208.2632921569, Loss: 2.4831337928771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15936 , TFLOPS: 70.69094563997005, Tokens per sec: 175179.95054636293, Loss: 2.4801747798919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15937 , TFLOPS: 68.81853750258973, Tokens per sec: 170539.91691915074, Loss: 2.4803786277770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15938 , TFLOPS: 70.44260965521663, Tokens per sec: 174564.5466196769, Loss: 2.493043899536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15939 , TFLOPS: 71.53103686665783, Tokens per sec: 177261.78914978358, Loss: 2.484384536743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15940 , TFLOPS: 69.65561095146553, Tokens per sec: 172614.2771367176, Loss: 2.50518798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15941 , TFLOPS: 68.68748106962154, Tokens per sec: 170215.14464119766, Loss: 2.5036685466766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15942 , TFLOPS: 71.48180453627602, Tokens per sec: 177139.78601170852, Loss: 2.467536211013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15943 , TFLOPS: 70.61082930505545, Tokens per sec: 174981.4134429017, Loss: 2.4760055541992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15944 , TFLOPS: 68.79791380556638, Tokens per sec: 170488.8091260391, Loss: 2.4926774501800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15945 , TFLOPS: 69.92844947434861, Tokens per sec: 173290.40105206633, Loss: 2.4952895641326904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15946 , TFLOPS: 71.4852161197489, Tokens per sec: 177148.24029137127, Loss: 2.4839229583740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15947 , TFLOPS: 69.13377463026089, Tokens per sec: 171321.1092477562, Loss: 2.476761817932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15948 , TFLOPS: 70.51088306070959, Tokens per sec: 174733.73563942497, Loss: 2.491222858428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15949 , TFLOPS: 70.54092877418485, Tokens per sec: 174808.1922272252, Loss: 2.5040736198425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15950 , TFLOPS: 69.83027829544184, Tokens per sec: 173047.12205628725, Loss: 2.478358268737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15951 , TFLOPS: 69.57476883406748, Tokens per sec: 172413.94146431083, Loss: 2.4891436100006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15952 , TFLOPS: 71.45370591243777, Tokens per sec: 177070.15452651842, Loss: 2.4830822944641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15953 , TFLOPS: 69.52501039656836, Tokens per sec: 172290.63457484366, Loss: 2.5138649940490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15954 , TFLOPS: 68.79149683724224, Tokens per sec: 170472.90717164447, Loss: 2.4966461658477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15955 , TFLOPS: 71.50953402457395, Tokens per sec: 177208.5027383661, Loss: 2.480567693710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15956 , TFLOPS: 69.69691961429662, Tokens per sec: 172716.6445537379, Loss: 2.4782474040985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15957 , TFLOPS: 68.85288053256919, Tokens per sec: 170625.0227306369, Loss: 2.4928715229034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15958 , TFLOPS: 71.4858560655681, Tokens per sec: 177149.82614760712, Loss: 2.4789834022521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15959 , TFLOPS: 71.51749244657358, Tokens per sec: 177228.22458477376, Loss: 2.4882876873016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15960 , TFLOPS: 68.80690452165086, Tokens per sec: 170511.08911090557, Loss: 2.4861602783203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15961 , TFLOPS: 69.73817460168253, Tokens per sec: 172818.87895709224, Loss: 2.48498797416687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15962 , TFLOPS: 71.5157216053535, Tokens per sec: 177223.83624515613, Loss: 2.4796221256256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15963 , TFLOPS: 70.65824903617026, Tokens per sec: 175098.9247603188, Loss: 2.498453140258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15964 , TFLOPS: 67.82097746206558, Tokens per sec: 168067.85324842174, Loss: 2.495604991912842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15965 , TFLOPS: 71.5232527332756, Tokens per sec: 177242.49920976933, Loss: 2.5117762088775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15966 , TFLOPS: 71.50676573240811, Tokens per sec: 177201.6426054266, Loss: 2.489375591278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15967 , TFLOPS: 69.78926514559967, Tokens per sec: 172945.4869530069, Loss: 2.494098424911499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15968 , TFLOPS: 68.87519989596773, Tokens per sec: 170680.3325718193, Loss: 2.4972732067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15969 , TFLOPS: 71.53349794764343, Tokens per sec: 177267.88797957634, Loss: 2.4951019287109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15970 , TFLOPS: 70.661600764839, Tokens per sec: 175107.23071318335, Loss: 2.49941349029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15971 , TFLOPS: 67.91645227716488, Tokens per sec: 168304.45035765693, Loss: 2.4851884841918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15972 , TFLOPS: 71.52397018690152, Tokens per sec: 177244.27713888843, Loss: 2.504365921020508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15973 , TFLOPS: 70.58862328847418, Tokens per sec: 174926.3844876187, Loss: 2.47573184967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15974 , TFLOPS: 68.95262201783822, Tokens per sec: 170872.1931185652, Loss: 2.5078210830688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15975 , TFLOPS: 70.598790155282, Tokens per sec: 174951.5791035413, Loss: 2.5008037090301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15976 , TFLOPS: 70.646817507788, Tokens per sec: 175070.5961737008, Loss: 2.474536657333374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15977 , TFLOPS: 69.68106076506118, Tokens per sec: 172677.34457833602, Loss: 2.4861819744110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15978 , TFLOPS: 69.79985157151604, Tokens per sec: 172971.72128261876, Loss: 2.474684715270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15979 , TFLOPS: 71.5131758443945, Tokens per sec: 177217.527569061, Loss: 2.475275754928589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15980 , TFLOPS: 69.10310667928107, Tokens per sec: 171245.11068687454, Loss: 2.5229427814483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15981 , TFLOPS: 69.62574647856968, Tokens per sec: 172540.2696830384, Loss: 2.4978270530700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15982 , TFLOPS: 70.50680154367554, Tokens per sec: 174723.62119059835, Loss: 2.4877357482910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15983 , TFLOPS: 70.69311663736923, Tokens per sec: 175185.33051707322, Loss: 2.467439651489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15984 , TFLOPS: 68.62531133125124, Tokens per sec: 170061.081180954, Loss: 2.479262351989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15985 , TFLOPS: 71.54479866218595, Tokens per sec: 177295.89239508918, Loss: 2.4953951835632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15986 , TFLOPS: 70.48809965390197, Tokens per sec: 174677.27584755624, Loss: 2.509284496307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15987 , TFLOPS: 69.12523870622682, Tokens per sec: 171299.95628769012, Loss: 2.510854959487915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15988 , TFLOPS: 69.72217368092915, Tokens per sec: 172779.2269128184, Loss: 2.5143046379089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15989 , TFLOPS: 71.5225215642214, Tokens per sec: 177240.6872923081, Loss: 2.494006395339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15990 , TFLOPS: 69.51164530725518, Tokens per sec: 172257.51441123945, Loss: 2.49544358253479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15991 , TFLOPS: 68.74610594234512, Tokens per sec: 170360.42353386956, Loss: 2.469364643096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15992 , TFLOPS: 71.49016642582336, Tokens per sec: 177160.50769514503, Loss: 2.484076976776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15993 , TFLOPS: 69.56497699015556, Tokens per sec: 172389.67619643643, Loss: 2.4639997482299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15994 , TFLOPS: 69.82712771896323, Tokens per sec: 173039.31458070857, Loss: 2.506476402282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15995 , TFLOPS: 70.76517632408763, Tokens per sec: 175363.90235879124, Loss: 2.4817936420440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15996 , TFLOPS: 71.51047711351548, Tokens per sec: 177210.83981665302, Loss: 2.5011096000671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15997 , TFLOPS: 67.95450113777238, Tokens per sec: 168398.7396256703, Loss: 2.4882147312164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15998 , TFLOPS: 70.51134808307405, Tokens per sec: 174734.88801607053, Loss: 2.4991846084594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15999 , TFLOPS: 71.52767994281461, Tokens per sec: 177253.47032270438, Loss: 2.499434471130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16000 , TFLOPS: 69.76983366965769, Tokens per sec: 172897.33361506893, Loss: 2.500061511993408 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/optimizer.pt +[rank0]:[2024-08-27 11:48:46,766] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006420184989110567, 'preprocessing_with_comm': 0.0012551330000860617, 'state_converting': 0.9960027540073497, : 1.005086849007057}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000, took 4.55s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014000 + eval ppl=9.570891380310059, eval loss=2.2587263584136963 +------------------------------------------------------------------ +iteration: 16001 , TFLOPS: 64.49628612651892, Tokens per sec: 159828.90187395219, Loss: 2.4989521503448486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16002 , TFLOPS: 71.49778306841492, Tokens per sec: 177179.38257452936, Loss: 2.4956552982330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16003 , TFLOPS: 71.47845281077242, Tokens per sec: 177131.48006668757, Loss: 2.475799560546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16004 , TFLOPS: 66.9947139789523, Tokens per sec: 166020.28131683043, Loss: 2.4834470748901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16005 , TFLOPS: 70.59145254742221, Tokens per sec: 174933.39570296046, Loss: 2.503845453262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16006 , TFLOPS: 71.53494926710233, Tokens per sec: 177271.48450907157, Loss: 2.5040783882141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16007 , TFLOPS: 69.75078691487101, Tokens per sec: 172850.1336585343, Loss: 2.529085397720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16008 , TFLOPS: 68.78323342506191, Tokens per sec: 170452.42952597162, Loss: 2.4966001510620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16009 , TFLOPS: 71.49899617881775, Tokens per sec: 177182.38879574274, Loss: 2.459205150604248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16010 , TFLOPS: 70.42691939173577, Tokens per sec: 174525.6644183461, Loss: 2.5021955966949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16011 , TFLOPS: 66.96136039337568, Tokens per sec: 165937.6274575708, Loss: 2.47733211517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16012 , TFLOPS: 71.4820979412549, Tokens per sec: 177140.51310156746, Loss: 2.48219895362854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16013 , TFLOPS: 71.494288058067, Tokens per sec: 177170.72155388005, Loss: 2.522190570831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16014 , TFLOPS: 68.05467325474334, Tokens per sec: 168646.97716639398, Loss: 2.4570910930633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16015 , TFLOPS: 69.12459633891342, Tokens per sec: 171298.3644307249, Loss: 2.495152235031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16016 , TFLOPS: 71.49234242170807, Tokens per sec: 177165.9000526531, Loss: 2.4738357067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16017 , TFLOPS: 68.95931441614852, Tokens per sec: 170888.77761880663, Loss: 2.480083703994751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16018 , TFLOPS: 69.82417826801579, Tokens per sec: 173032.005516352, Loss: 2.4809517860412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16019 , TFLOPS: 69.95036440133995, Tokens per sec: 173344.70865527936, Loss: 2.5090627670288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16020 , TFLOPS: 69.68770037910319, Tokens per sec: 172693.798273346, Loss: 2.496216297149658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16021 , TFLOPS: 68.99346783984035, Tokens per sec: 170973.4135650252, Loss: 2.4991090297698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16022 , TFLOPS: 71.50952100474844, Tokens per sec: 177208.47047380425, Loss: 2.4786455631256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16023 , TFLOPS: 69.68404619733181, Tokens per sec: 172684.74280837516, Loss: 2.4873623847961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16024 , TFLOPS: 68.80615000264362, Tokens per sec: 170509.21932968852, Loss: 2.486386299133301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16025 , TFLOPS: 70.64424529335392, Tokens per sec: 175064.2219429803, Loss: 2.49405837059021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16026 , TFLOPS: 70.48006023850195, Tokens per sec: 174657.35329058027, Loss: 2.470064401626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16027 , TFLOPS: 69.65314292988482, Tokens per sec: 172608.1611073656, Loss: 2.4714958667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16028 , TFLOPS: 69.69126373932735, Tokens per sec: 172702.62867251798, Loss: 2.508932590484619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16029 , TFLOPS: 71.57196555943707, Tokens per sec: 177363.21495915824, Loss: 2.4743056297302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16030 , TFLOPS: 69.64192231619376, Tokens per sec: 172580.35519058647, Loss: 2.4779717922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16031 , TFLOPS: 67.93251243084956, Tokens per sec: 168344.24918765953, Loss: 2.4875752925872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16032 , TFLOPS: 71.54923671014363, Tokens per sec: 177306.89036681966, Loss: 2.447049617767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16033 , TFLOPS: 70.4848998754395, Tokens per sec: 174669.3464440415, Loss: 2.497117280960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16034 , TFLOPS: 68.9897227977009, Tokens per sec: 170964.13293805323, Loss: 2.496002435684204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16035 , TFLOPS: 70.1016125786752, Tokens per sec: 173719.51830007829, Loss: 2.4948112964630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16036 , TFLOPS: 71.53218813034368, Tokens per sec: 177264.6421080193, Loss: 2.497976064682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16037 , TFLOPS: 68.7443337937626, Tokens per sec: 170356.0319544638, Loss: 2.482516288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16038 , TFLOPS: 69.26344859748214, Tokens per sec: 171642.4556232972, Loss: 2.480403184890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16039 , TFLOPS: 71.53004271119136, Tokens per sec: 177259.32552302242, Loss: 2.4820003509521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16040 , TFLOPS: 70.61913054854178, Tokens per sec: 175001.98483871826, Loss: 2.500012159347534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16041 , TFLOPS: 67.3547648413444, Tokens per sec: 166912.5270167148, Loss: 2.501595973968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16042 , TFLOPS: 71.44938361335441, Tokens per sec: 177059.4433932493, Loss: 2.4794540405273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16043 , TFLOPS: 71.45680650152991, Tokens per sec: 177077.83812784683, Loss: 2.4938998222351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16044 , TFLOPS: 69.16315480172024, Tokens per sec: 171393.9165491255, Loss: 2.479588031768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16045 , TFLOPS: 69.80631859928712, Tokens per sec: 172987.74728983743, Loss: 2.4896812438964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16046 , TFLOPS: 71.50869213199829, Tokens per sec: 177206.4164358215, Loss: 2.4665701389312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16047 , TFLOPS: 69.83369006280677, Tokens per sec: 173055.57679165475, Loss: 2.5055863857269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16048 , TFLOPS: 68.57350047950256, Tokens per sec: 169932.6881828843, Loss: 2.489978313446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16049 , TFLOPS: 71.52886301801088, Tokens per sec: 177256.40211336725, Loss: 2.463533878326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16050 , TFLOPS: 70.57976030959199, Tokens per sec: 174904.42104961086, Loss: 2.4875378608703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16051 , TFLOPS: 68.89362144181175, Tokens per sec: 170725.9831917216, Loss: 2.4932026863098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16052 , TFLOPS: 69.89403296573687, Tokens per sec: 173205.11315243502, Loss: 2.500004291534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16053 , TFLOPS: 71.54452860308702, Tokens per sec: 177295.22315889254, Loss: 2.4823670387268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16054 , TFLOPS: 69.69005143145884, Tokens per sec: 172699.6244400731, Loss: 2.4811367988586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16055 , TFLOPS: 69.17311104128434, Tokens per sec: 171418.5892075357, Loss: 2.5057716369628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16056 , TFLOPS: 70.5664156305842, Tokens per sec: 174871.35146499184, Loss: 2.4888057708740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16057 , TFLOPS: 70.58717281726635, Tokens per sec: 174922.79006018306, Loss: 2.4716479778289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16058 , TFLOPS: 68.30457585388497, Tokens per sec: 169266.26333611083, Loss: 2.478610038757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16059 , TFLOPS: 71.49724331428912, Tokens per sec: 177178.0450043468, Loss: 2.5100221633911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16060 , TFLOPS: 70.40079995113649, Tokens per sec: 174460.9375672472, Loss: 2.5044431686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16061 , TFLOPS: 67.60183008574298, Tokens per sec: 167524.78190881794, Loss: 2.4760959148406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16062 , TFLOPS: 71.53038757817654, Tokens per sec: 177260.1801414019, Loss: 2.505044460296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16063 , TFLOPS: 70.53919921561405, Tokens per sec: 174803.9061905602, Loss: 2.4963197708129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16064 , TFLOPS: 68.99592084900559, Tokens per sec: 170979.49239195793, Loss: 2.4836912155151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16065 , TFLOPS: 69.74792620235546, Tokens per sec: 172843.04449778722, Loss: 2.4725522994995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16066 , TFLOPS: 71.46179113807423, Tokens per sec: 177090.19060630669, Loss: 2.4968068599700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16067 , TFLOPS: 68.71333295936785, Tokens per sec: 170279.20847180972, Loss: 2.482795238494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16068 , TFLOPS: 69.52693645167629, Tokens per sec: 172295.4075515736, Loss: 2.5042788982391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16069 , TFLOPS: 71.52810085890692, Tokens per sec: 177254.51339914888, Loss: 2.4788284301757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16070 , TFLOPS: 70.49714504255297, Tokens per sec: 174699.69131706524, Loss: 2.4980533123016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16071 , TFLOPS: 68.99959785236848, Tokens per sec: 170988.60441134652, Loss: 2.494800090789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16072 , TFLOPS: 70.63062456005203, Tokens per sec: 175030.4682654106, Loss: 2.506757974624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16073 , TFLOPS: 71.5145053578403, Tokens per sec: 177220.8222498377, Loss: 2.483902931213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16074 , TFLOPS: 68.5501367723517, Tokens per sec: 169874.79034283964, Loss: 2.4801976680755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16075 , TFLOPS: 68.89258599490886, Tokens per sec: 170723.41724023232, Loss: 2.445922613143921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16076 , TFLOPS: 71.50720272438585, Tokens per sec: 177202.7255196861, Loss: 2.484717845916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16077 , TFLOPS: 70.58452488088092, Tokens per sec: 174916.22818212572, Loss: 2.5214054584503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16078 , TFLOPS: 67.3894744125493, Tokens per sec: 166998.54115773548, Loss: 2.46343994140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16079 , TFLOPS: 71.51777121854379, Tokens per sec: 177228.9154124252, Loss: 2.4779233932495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16080 , TFLOPS: 71.55222063786321, Tokens per sec: 177314.28486841556, Loss: 2.486116647720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16081 , TFLOPS: 67.98030074637225, Tokens per sec: 168462.6738978423, Loss: 2.491816997528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16082 , TFLOPS: 69.79357284537903, Tokens per sec: 172956.1619075925, Loss: 2.493863344192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16083 , TFLOPS: 71.42963031915707, Tokens per sec: 177010.49255422328, Loss: 2.489912986755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16084 , TFLOPS: 69.79991861942874, Tokens per sec: 172971.88743473234, Loss: 2.5050933361053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16085 , TFLOPS: 68.40342524765259, Tokens per sec: 169511.22302302663, Loss: 2.4955263137817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16086 , TFLOPS: 71.52169976209707, Tokens per sec: 177238.65077611504, Loss: 2.491109848022461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16087 , TFLOPS: 70.61737996309529, Tokens per sec: 174997.64669513897, Loss: 2.504580020904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16088 , TFLOPS: 68.87785824305699, Tokens per sec: 170686.9202487476, Loss: 2.5031466484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16089 , TFLOPS: 70.45167840599215, Tokens per sec: 174587.01998310402, Loss: 2.485292911529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16090 , TFLOPS: 71.45404647547672, Tokens per sec: 177070.99847924503, Loss: 2.494870185852051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16091 , TFLOPS: 68.4141476236039, Tokens per sec: 169537.79425180118, Loss: 2.4694931507110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16092 , TFLOPS: 70.54473347209513, Tokens per sec: 174817.620687769, Loss: 2.49462890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16093 , TFLOPS: 70.54698756487284, Tokens per sec: 174823.20657798095, Loss: 2.488140106201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16094 , TFLOPS: 69.66283553759031, Tokens per sec: 172632.18045698866, Loss: 2.4808521270751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16095 , TFLOPS: 68.94579633998177, Tokens per sec: 170855.27833692642, Loss: 2.500927209854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16096 , TFLOPS: 71.483738364511, Tokens per sec: 177144.5782511036, Loss: 2.477851629257202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16097 , TFLOPS: 69.67886357475558, Tokens per sec: 172671.89969871828, Loss: 2.512697458267212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16098 , TFLOPS: 68.19664706016545, Tokens per sec: 168998.80389593574, Loss: 2.49190092086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16099 , TFLOPS: 71.48636792625163, Tokens per sec: 177151.094594767, Loss: 2.4940032958984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16100 , TFLOPS: 70.49279326145509, Tokens per sec: 174688.9071241164, Loss: 2.491281747817993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16101 , TFLOPS: 68.81521243285532, Tokens per sec: 170531.67703006222, Loss: 2.4653072357177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16102 , TFLOPS: 70.03325883259207, Tokens per sec: 173550.13018749753, Loss: 2.495051383972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16103 , TFLOPS: 71.50875782839958, Tokens per sec: 177206.57923874096, Loss: 2.494746208190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16104 , TFLOPS: 68.75731072312367, Tokens per sec: 170388.19021494727, Loss: 2.4847073554992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16105 , TFLOPS: 69.03710713424887, Tokens per sec: 171081.55654383465, Loss: 2.502454996109009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16106 , TFLOPS: 71.51582703777937, Tokens per sec: 177224.0975183218, Loss: 2.501354932785034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16107 , TFLOPS: 70.40886290554985, Tokens per sec: 174480.91845649126, Loss: 2.490041732788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16108 , TFLOPS: 68.72196431268009, Tokens per sec: 170300.59791613955, Loss: 2.5018608570098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16109 , TFLOPS: 71.49322993171019, Tokens per sec: 177168.0994002056, Loss: 2.479555130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16110 , TFLOPS: 71.54911886043246, Tokens per sec: 177306.59832225315, Loss: 2.4855422973632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16111 , TFLOPS: 67.21641173487033, Tokens per sec: 166569.6727780192, Loss: 2.483492136001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16112 , TFLOPS: 70.63352575171945, Tokens per sec: 175037.65773795507, Loss: 2.49484920501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16113 , TFLOPS: 71.46138679192248, Tokens per sec: 177089.18859200025, Loss: 2.4698052406311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16114 , TFLOPS: 69.74860189281304, Tokens per sec: 172844.7189331746, Loss: 2.485042095184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16115 , TFLOPS: 68.80842040541692, Tokens per sec: 170514.84563786627, Loss: 2.4687016010284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16116 , TFLOPS: 71.5511239634562, Tokens per sec: 177311.56718842633, Loss: 2.4886507987976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16117 , TFLOPS: 70.60290845707358, Tokens per sec: 174961.7847090492, Loss: 2.4755425453186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16118 , TFLOPS: 68.91143494616642, Tokens per sec: 170770.12701783198, Loss: 2.4977498054504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16119 , TFLOPS: 69.84264176280008, Tokens per sec: 173077.76009034933, Loss: 2.478414535522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16120 , TFLOPS: 71.47005206878387, Tokens per sec: 177110.66210262183, Loss: 2.4705443382263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16121 , TFLOPS: 69.74436253973418, Tokens per sec: 172834.21334924185, Loss: 2.4859654903411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16122 , TFLOPS: 68.4307754262352, Tokens per sec: 169578.99977842523, Loss: 2.497403144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16123 , TFLOPS: 71.44700146845246, Tokens per sec: 177053.54017576735, Loss: 2.4649088382720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16124 , TFLOPS: 70.58212420340695, Tokens per sec: 174910.27903888872, Loss: 2.4849114418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16125 , TFLOPS: 68.25381634687278, Tokens per sec: 169140.47568611332, Loss: 2.5030598640441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16126 , TFLOPS: 69.82230493675085, Tokens per sec: 173027.363195113, Loss: 2.500547409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16127 , TFLOPS: 71.53369999687297, Tokens per sec: 177268.38867982468, Loss: 2.482240915298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16128 , TFLOPS: 68.15320220707888, Tokens per sec: 168891.14276414158, Loss: 2.491075038909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16129 , TFLOPS: 71.42131467761533, Tokens per sec: 176989.88547843933, Loss: 2.506951093673706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16130 , TFLOPS: 70.26429909046854, Tokens per sec: 174122.67339770088, Loss: 2.477961778640747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16131 , TFLOPS: 68.98587520307348, Tokens per sec: 170954.5981747189, Loss: 2.4871246814727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16132 , TFLOPS: 69.80339364700785, Tokens per sec: 172980.4989358793, Loss: 2.4831748008728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16133 , TFLOPS: 71.49075381939696, Tokens per sec: 177161.96332112717, Loss: 2.4772868156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16134 , TFLOPS: 68.67901012413007, Tokens per sec: 170194.15270511815, Loss: 2.4916515350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16135 , TFLOPS: 68.85808287371667, Tokens per sec: 170637.91470508554, Loss: 2.4770095348358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16136 , TFLOPS: 71.5385036910901, Tokens per sec: 177280.29276326534, Loss: 2.488070011138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16137 , TFLOPS: 69.7647153166987, Tokens per sec: 172884.64977260338, Loss: 2.5063588619232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16138 , TFLOPS: 69.00373609316833, Tokens per sec: 170998.85942791344, Loss: 2.4745523929595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16139 , TFLOPS: 70.51654322003985, Tokens per sec: 174747.76213776338, Loss: 2.4903156757354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16140 , TFLOPS: 71.49200703498943, Tokens per sec: 177165.06892741835, Loss: 2.4751572608947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16141 , TFLOPS: 69.446409789821, Tokens per sec: 172095.85361275205, Loss: 2.5084784030914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16142 , TFLOPS: 69.04428136402336, Tokens per sec: 171099.33507552248, Loss: 2.476437568664551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16143 , TFLOPS: 71.54856113906612, Tokens per sec: 177305.2162272693, Loss: 2.490483522415161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16144 , TFLOPS: 69.78329614023903, Tokens per sec: 172930.69510018302, Loss: 2.4960830211639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16145 , TFLOPS: 67.97046556396069, Tokens per sec: 168438.30123239165, Loss: 2.488499164581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16146 , TFLOPS: 71.51120618166921, Tokens per sec: 177212.6465278515, Loss: 2.483366012573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16147 , TFLOPS: 71.5720105020889, Tokens per sec: 177363.32633200032, Loss: 2.5015456676483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16148 , TFLOPS: 67.91811475295606, Tokens per sec: 168308.57015580428, Loss: 2.4773952960968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16149 , TFLOPS: 70.74512230822344, Tokens per sec: 175314.20629834753, Loss: 2.4557552337646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16150 , TFLOPS: 71.46970599850023, Tokens per sec: 177109.8045023362, Loss: 2.5267891883850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16151 , TFLOPS: 68.86283602258527, Tokens per sec: 170649.6935315845, Loss: 2.4879534244537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16152 , TFLOPS: 68.78719852979721, Tokens per sec: 170462.25549229802, Loss: 2.4926199913024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16153 , TFLOPS: 71.51201994770723, Tokens per sec: 177214.6631297374, Loss: 2.4734461307525635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16154 , TFLOPS: 69.13306017458808, Tokens per sec: 171319.3387478952, Loss: 2.4982950687408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16155 , TFLOPS: 69.03028455757777, Tokens per sec: 171064.6494472748, Loss: 2.4784798622131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16156 , TFLOPS: 70.4817609543128, Tokens per sec: 174661.56785170885, Loss: 2.472517728805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16157 , TFLOPS: 70.70106456637541, Tokens per sec: 175205.0263606872, Loss: 2.5149407386779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16158 , TFLOPS: 68.96132895603718, Tokens per sec: 170893.7698705706, Loss: 2.447190284729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16159 , TFLOPS: 69.63312265989975, Tokens per sec: 172558.54867292778, Loss: 2.501333475112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16160 , TFLOPS: 71.51024587136017, Tokens per sec: 177210.26677311875, Loss: 2.4791765213012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16161 , TFLOPS: 69.17109318251232, Tokens per sec: 171413.58873121312, Loss: 2.489044189453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16162 , TFLOPS: 68.99380782315231, Tokens per sec: 170974.2560811245, Loss: 2.4723057746887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16163 , TFLOPS: 69.91944026214412, Tokens per sec: 173268.07523177695, Loss: 2.4887495040893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16164 , TFLOPS: 70.68867182453593, Tokens per sec: 175174.31578123575, Loss: 2.4903931617736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16165 , TFLOPS: 68.0102265108253, Tokens per sec: 168536.83323875206, Loss: 2.487213373184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16166 , TFLOPS: 71.4850863578703, Tokens per sec: 177147.9187271417, Loss: 2.500216484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16167 , TFLOPS: 70.40698062563729, Tokens per sec: 174476.25395951772, Loss: 2.4725561141967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16168 , TFLOPS: 68.95517232306776, Tokens per sec: 170878.5130558633, Loss: 2.5042881965637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16169 , TFLOPS: 70.4456559116464, Tokens per sec: 174572.09557868273, Loss: 2.507707118988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16170 , TFLOPS: 71.51356763970426, Tokens per sec: 177218.49848099845, Loss: 2.5007424354553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16171 , TFLOPS: 68.83241948803465, Tokens per sec: 170574.31800831016, Loss: 2.4727396965026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16172 , TFLOPS: 69.28271297645004, Tokens per sec: 171690.19487652616, Loss: 2.482617139816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16173 , TFLOPS: 71.50601313198665, Tokens per sec: 177199.77757867647, Loss: 2.483158588409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16174 , TFLOPS: 69.58582193901529, Tokens per sec: 172441.3322759715, Loss: 2.4742279052734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16175 , TFLOPS: 68.34622844123379, Tokens per sec: 169369.4830359725, Loss: 2.5000925064086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16176 , TFLOPS: 71.52508342212917, Tokens per sec: 177247.03585841547, Loss: 2.4731242656707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16177 , TFLOPS: 71.50468422419206, Tokens per sec: 177196.48439877018, Loss: 2.4972023963928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16178 , TFLOPS: 67.43549601413024, Tokens per sec: 167112.58775615093, Loss: 2.4669904708862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16179 , TFLOPS: 70.59950772821445, Tokens per sec: 174953.35732831506, Loss: 2.4712040424346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16180 , TFLOPS: 71.56692878974103, Tokens per sec: 177350.73328900547, Loss: 2.480894088745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16181 , TFLOPS: 68.75538873135307, Tokens per sec: 170383.42730761485, Loss: 2.494126081466675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16182 , TFLOPS: 68.93384260965709, Tokens per sec: 170825.6556763698, Loss: 2.4896738529205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16183 , TFLOPS: 71.4477474083297, Tokens per sec: 177055.38869695523, Loss: 2.472255229949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16184 , TFLOPS: 70.42691877348935, Tokens per sec: 174525.6628862634, Loss: 2.4681613445281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16185 , TFLOPS: 68.03137542743457, Tokens per sec: 168589.24258385532, Loss: 2.5081138610839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16186 , TFLOPS: 70.9298151518263, Tokens per sec: 175771.8955103917, Loss: 2.493664264678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16187 , TFLOPS: 71.47831134850094, Tokens per sec: 177131.1295075955, Loss: 2.49422287940979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16188 , TFLOPS: 69.7470380987077, Tokens per sec: 172840.8436791154, Loss: 2.49503231048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16189 , TFLOPS: 68.84439604832266, Tokens per sec: 170603.99724403088, Loss: 2.480715751647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16190 , TFLOPS: 71.50328745800975, Tokens per sec: 177193.02305830395, Loss: 2.488675117492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16191 , TFLOPS: 70.51120715492613, Tokens per sec: 174734.53878059547, Loss: 2.494354248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16192 , TFLOPS: 68.31201022886881, Tokens per sec: 169284.68653628477, Loss: 2.483752727508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16193 , TFLOPS: 70.42347853788101, Tokens per sec: 174517.1376034527, Loss: 2.479857921600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16194 , TFLOPS: 71.50598313540561, Tokens per sec: 177199.70324384427, Loss: 2.4927823543548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16195 , TFLOPS: 67.53545946350614, Tokens per sec: 167360.30819558742, Loss: 2.4848830699920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16196 , TFLOPS: 70.41737302151677, Tokens per sec: 174502.00746132396, Loss: 2.492196798324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16197 , TFLOPS: 71.51719966867871, Tokens per sec: 177227.4990488981, Loss: 2.486412525177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16198 , TFLOPS: 68.80035504738109, Tokens per sec: 170494.85879218092, Loss: 2.506437301635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16199 , TFLOPS: 70.01516816315124, Tokens per sec: 173505.29951577145, Loss: 2.482809543609619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16200 , TFLOPS: 70.53127864336444, Tokens per sec: 174784.2781400026, Loss: 2.4725441932678223 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/optimizer.pt +[rank0]:[2024-08-27 12:14:06,802] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006382039995514788, 'preprocessing_with_comm': 0.0014667889918200672, 'state_converting': 0.9735825429961551, : 0.9827971210033866}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014200 + eval ppl=10.880582809448242, eval loss=2.386979818344116 +------------------------------------------------------------------ +iteration: 16201 , TFLOPS: 65.87432096999716, Tokens per sec: 163243.82401916094, Loss: 2.4871952533721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16202 , TFLOPS: 68.51105591563073, Tokens per sec: 169777.94367477333, Loss: 2.484020709991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16203 , TFLOPS: 71.49740141996826, Tokens per sec: 177178.43680763646, Loss: 2.4937288761138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16204 , TFLOPS: 70.11945995095206, Tokens per sec: 173763.7460546307, Loss: 2.489168643951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16205 , TFLOPS: 67.72155569014276, Tokens per sec: 167821.47514538467, Loss: 2.4921555519104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16206 , TFLOPS: 70.63883179946698, Tokens per sec: 175050.8067087829, Loss: 2.4847471714019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16207 , TFLOPS: 71.53511460509291, Tokens per sec: 177271.89423482568, Loss: 2.463599920272827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16208 , TFLOPS: 68.23346359894775, Tokens per sec: 169090.03933471482, Loss: 2.4941375255584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16209 , TFLOPS: 70.05021181197567, Tokens per sec: 173592.14153793495, Loss: 2.505054473876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16210 , TFLOPS: 71.47643573112367, Tokens per sec: 177126.48152111832, Loss: 2.5111289024353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16211 , TFLOPS: 70.49202798983255, Tokens per sec: 174687.01069673413, Loss: 2.506723165512085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16212 , TFLOPS: 68.37887156332603, Tokens per sec: 169450.37628845382, Loss: 2.4737563133239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16213 , TFLOPS: 71.53647875527906, Tokens per sec: 177275.27474926054, Loss: 2.4589972496032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16214 , TFLOPS: 70.58674820254056, Tokens per sec: 174921.7378181174, Loss: 2.4968860149383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16215 , TFLOPS: 68.84575036356897, Tokens per sec: 170607.35338640137, Loss: 2.5013375282287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16216 , TFLOPS: 70.74166921540083, Tokens per sec: 175305.64915394352, Loss: 2.4761619567871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16217 , TFLOPS: 71.47723170286045, Tokens per sec: 177128.4540267656, Loss: 2.505689859390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16218 , TFLOPS: 69.66145895822623, Tokens per sec: 172628.76914168175, Loss: 2.4792561531066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16219 , TFLOPS: 67.95657810058381, Tokens per sec: 168403.88656831338, Loss: 2.4925975799560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16220 , TFLOPS: 71.51481219786618, Tokens per sec: 177221.58263322304, Loss: 2.507395029067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16221 , TFLOPS: 70.59887390363009, Tokens per sec: 174951.7866411735, Loss: 2.4908254146575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16222 , TFLOPS: 68.47702530942404, Tokens per sec: 169693.6120838124, Loss: 2.5057971477508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16223 , TFLOPS: 70.38245813539756, Tokens per sec: 174415.48452732834, Loss: 2.5101728439331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16224 , TFLOPS: 71.50681919837477, Tokens per sec: 177201.77509998175, Loss: 2.500737428665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16225 , TFLOPS: 68.15890707110277, Tokens per sec: 168905.28004563545, Loss: 2.4804933071136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16226 , TFLOPS: 69.67156915578587, Tokens per sec: 172653.82332496476, Loss: 2.515589714050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16227 , TFLOPS: 71.54084937760705, Tokens per sec: 177286.10563285302, Loss: 2.498591423034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16228 , TFLOPS: 69.64897883406674, Tokens per sec: 172597.84201921537, Loss: 2.5019755363464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16229 , TFLOPS: 68.96440776108727, Tokens per sec: 170901.39948864313, Loss: 2.4886417388916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16230 , TFLOPS: 70.48969120571573, Tokens per sec: 174681.21988827462, Loss: 2.506986141204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16231 , TFLOPS: 70.53937410845134, Tokens per sec: 174804.339594277, Loss: 2.4793384075164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16232 , TFLOPS: 69.71027316766326, Tokens per sec: 172749.73612999578, Loss: 2.501840114593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16233 , TFLOPS: 71.56495634864997, Tokens per sec: 177345.8453626995, Loss: 2.5039172172546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16234 , TFLOPS: 70.52237187254599, Tokens per sec: 174762.20618075263, Loss: 2.4780869483947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16235 , TFLOPS: 69.10265093633836, Tokens per sec: 171243.98130565824, Loss: 2.4450337886810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16236 , TFLOPS: 70.03646090954857, Tokens per sec: 173558.06528693365, Loss: 2.5058062076568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16237 , TFLOPS: 70.4073802623109, Tokens per sec: 174477.2443032185, Loss: 2.486257314682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16238 , TFLOPS: 69.7294489728449, Tokens per sec: 172797.25588762522, Loss: 2.4916372299194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16239 , TFLOPS: 68.8688424008103, Tokens per sec: 170664.5779984833, Loss: 2.488532543182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16240 , TFLOPS: 71.51764168625135, Tokens per sec: 177228.59441713538, Loss: 2.4817209243774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16241 , TFLOPS: 70.42513182062078, Tokens per sec: 174521.23462020594, Loss: 2.4984397888183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16242 , TFLOPS: 68.09424600009646, Tokens per sec: 168745.0427886768, Loss: 2.502974510192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16243 , TFLOPS: 71.54189214119287, Tokens per sec: 177288.6897158886, Loss: 2.497863292694092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16244 , TFLOPS: 71.55518522108841, Tokens per sec: 177321.6314322234, Loss: 2.4862380027770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16245 , TFLOPS: 68.56914684248814, Tokens per sec: 169921.89939076983, Loss: 2.4955687522888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16246 , TFLOPS: 70.02592396426795, Tokens per sec: 173531.95357578795, Loss: 2.5104777812957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16247 , TFLOPS: 71.49726249941364, Tokens per sec: 177178.0925471988, Loss: 2.4776806831359863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16248 , TFLOPS: 69.69963147685844, Tokens per sec: 172723.3648479024, Loss: 2.4875845909118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16249 , TFLOPS: 68.87510202499061, Tokens per sec: 170680.09003675662, Loss: 2.4944541454315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16250 , TFLOPS: 71.5073556105092, Tokens per sec: 177203.1043883415, Loss: 2.475362539291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16251 , TFLOPS: 71.45927602860804, Tokens per sec: 177083.9578879873, Loss: 2.4873692989349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16252 , TFLOPS: 67.85178142867396, Tokens per sec: 168144.18887101434, Loss: 2.491548776626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16253 , TFLOPS: 71.50606361265139, Tokens per sec: 177199.9026753246, Loss: 2.4729137420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16254 , TFLOPS: 71.53790405799954, Tokens per sec: 177278.8068064119, Loss: 2.482726573944092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16255 , TFLOPS: 68.90972572925672, Tokens per sec: 170765.89139004395, Loss: 2.468209981918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16256 , TFLOPS: 68.84430557859346, Tokens per sec: 170603.77305007586, Loss: 2.4927070140838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16257 , TFLOPS: 71.50719626324343, Tokens per sec: 177202.70950826345, Loss: 2.4842753410339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16258 , TFLOPS: 69.67559026720292, Tokens per sec: 172663.78808201817, Loss: 2.487675189971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16259 , TFLOPS: 68.92898146813576, Tokens per sec: 170813.60923218096, Loss: 2.4776744842529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16260 , TFLOPS: 70.5818000876354, Tokens per sec: 174909.47584430256, Loss: 2.478349208831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16261 , TFLOPS: 70.62351985550114, Tokens per sec: 175012.8620248857, Loss: 2.5058064460754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16262 , TFLOPS: 67.73120577575735, Tokens per sec: 167845.3891205823, Loss: 2.484278678894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16263 , TFLOPS: 71.4649041785068, Tokens per sec: 177097.9050634283, Loss: 2.4967362880706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16264 , TFLOPS: 70.43911333431399, Tokens per sec: 174555.8823513286, Loss: 2.4870550632476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16265 , TFLOPS: 69.6738739345372, Tokens per sec: 172659.5348206029, Loss: 2.495736598968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16266 , TFLOPS: 69.06814349972845, Tokens per sec: 171158.46807643096, Loss: 2.4937546253204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16267 , TFLOPS: 70.55999320323203, Tokens per sec: 174855.43598252436, Loss: 2.494224786758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16268 , TFLOPS: 69.78659615188892, Tokens per sec: 172938.87289257772, Loss: 2.4884321689605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16269 , TFLOPS: 69.11338031257385, Tokens per sec: 171270.56988190816, Loss: 2.4791862964630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16270 , TFLOPS: 71.53088649830441, Tokens per sec: 177261.41652043938, Loss: 2.472424268722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16271 , TFLOPS: 70.45035421767906, Tokens per sec: 174583.7384985928, Loss: 2.4878406524658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16272 , TFLOPS: 69.41288962847693, Tokens per sec: 172012.78696039083, Loss: 2.4860520362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16273 , TFLOPS: 70.91511739966751, Tokens per sec: 175735.47286145118, Loss: 2.4984519481658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16274 , TFLOPS: 70.63203478125689, Tokens per sec: 175033.96294890493, Loss: 2.473392963409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16275 , TFLOPS: 68.95042450441784, Tokens per sec: 170866.7474382334, Loss: 2.483267068862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16276 , TFLOPS: 69.77572065266766, Tokens per sec: 172911.92220747232, Loss: 2.4766616821289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16277 , TFLOPS: 71.5388651113002, Tokens per sec: 177281.18840235972, Loss: 2.468273639678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16278 , TFLOPS: 69.46948560916469, Tokens per sec: 172153.03803509707, Loss: 2.4945850372314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16279 , TFLOPS: 68.88126613535378, Tokens per sec: 170695.3653812694, Loss: 2.4892563819885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16280 , TFLOPS: 71.50947023908378, Tokens per sec: 177208.34467089493, Loss: 2.48091983795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16281 , TFLOPS: 70.5684758010357, Tokens per sec: 174876.45679431717, Loss: 2.4957163333892822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16282 , TFLOPS: 68.1796034076713, Tokens per sec: 168956.56784754188, Loss: 2.4821765422821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16283 , TFLOPS: 70.7671784784934, Tokens per sec: 175368.86391796422, Loss: 2.487218141555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16284 , TFLOPS: 71.52529285453305, Tokens per sec: 177247.55485498317, Loss: 2.474677324295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16285 , TFLOPS: 69.64588905751123, Tokens per sec: 172590.18521254382, Loss: 2.482325315475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16286 , TFLOPS: 68.66461324254638, Tokens per sec: 170158.4756466063, Loss: 2.481274366378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16287 , TFLOPS: 71.49525854088829, Tokens per sec: 177173.1265172187, Loss: 2.495551109313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16288 , TFLOPS: 70.49376341134955, Tokens per sec: 174691.3112624257, Loss: 2.4832968711853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16289 , TFLOPS: 67.6512377386536, Tokens per sec: 167647.21951543275, Loss: 2.473276376724243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16290 , TFLOPS: 71.49521335668494, Tokens per sec: 177173.01454578535, Loss: 2.4762895107269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16291 , TFLOPS: 71.5546231694953, Tokens per sec: 177320.23860646057, Loss: 2.5041627883911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16292 , TFLOPS: 69.39866661122001, Tokens per sec: 171977.5407568335, Loss: 2.47377872467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16293 , TFLOPS: 69.74932060041823, Tokens per sec: 172846.4999697924, Loss: 2.498197078704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16294 , TFLOPS: 71.48638967287378, Tokens per sec: 177151.14848529233, Loss: 2.482734441757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16295 , TFLOPS: 69.51261574546876, Tokens per sec: 172259.9192640353, Loss: 2.494269371032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16296 , TFLOPS: 69.26915447434529, Tokens per sec: 171656.5954147183, Loss: 2.4837300777435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16297 , TFLOPS: 70.42003803903955, Tokens per sec: 174508.61166832133, Loss: 2.502087116241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16298 , TFLOPS: 69.8143129426751, Tokens per sec: 173007.5581534025, Loss: 2.4898476600646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16299 , TFLOPS: 68.84395595794935, Tokens per sec: 170602.90665160585, Loss: 2.490666151046753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16300 , TFLOPS: 70.541757629693, Tokens per sec: 174810.24622248037, Loss: 2.4862451553344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16301 , TFLOPS: 71.53915394870293, Tokens per sec: 177281.90417326172, Loss: 2.494185447692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16302 , TFLOPS: 68.37389401532563, Tokens per sec: 169438.04137618665, Loss: 2.5130350589752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16303 , TFLOPS: 70.63961630322032, Tokens per sec: 175052.7507955039, Loss: 2.4991025924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16304 , TFLOPS: 70.47470503962391, Tokens per sec: 174644.08251783665, Loss: 2.505392074584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16305 , TFLOPS: 69.74998465673708, Tokens per sec: 172848.14557450238, Loss: 2.4978246688842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16306 , TFLOPS: 68.9938341037911, Tokens per sec: 170974.3212074425, Loss: 2.4728872776031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16307 , TFLOPS: 71.48252210449837, Tokens per sec: 177141.56422481022, Loss: 2.477574348449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16308 , TFLOPS: 69.53809050279887, Tokens per sec: 172323.04851897547, Loss: 2.48828387260437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16309 , TFLOPS: 68.13082454868075, Tokens per sec: 168835.68846152842, Loss: 2.502439498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16310 , TFLOPS: 71.48441685254815, Tokens per sec: 177146.25961920127, Loss: 2.475684642791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16311 , TFLOPS: 70.44296782896431, Tokens per sec: 174565.43421367978, Loss: 2.4733262062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16312 , TFLOPS: 70.4385193005612, Tokens per sec: 174554.410270251, Loss: 2.5065550804138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16313 , TFLOPS: 69.94850601056017, Tokens per sec: 173340.10335820843, Loss: 2.498227596282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16314 , TFLOPS: 71.48374410722708, Tokens per sec: 177144.59248218665, Loss: 2.4687047004699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16315 , TFLOPS: 68.79660682090858, Tokens per sec: 170485.57027408082, Loss: 2.5089492797851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16316 , TFLOPS: 68.35950270440874, Tokens per sec: 169402.37812240078, Loss: 2.4883265495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16317 , TFLOPS: 71.48957686977045, Tokens per sec: 177159.04671030096, Loss: 2.4970836639404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16318 , TFLOPS: 70.58136226992872, Tokens per sec: 174908.39088379577, Loss: 2.4958505630493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16319 , TFLOPS: 67.4074206360444, Tokens per sec: 167043.01387649606, Loss: 2.49432635307312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16320 , TFLOPS: 71.46475396256955, Tokens per sec: 177097.53281178817, Loss: 2.5063626766204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16321 , TFLOPS: 71.41624004516164, Tokens per sec: 176977.3099802022, Loss: 2.491007089614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16322 , TFLOPS: 68.95672566981777, Tokens per sec: 170882.36242022508, Loss: 2.4729340076446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16323 , TFLOPS: 69.72803613021924, Tokens per sec: 172793.75470796134, Loss: 2.4893832206726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16324 , TFLOPS: 71.56229313494003, Tokens per sec: 177339.24562574908, Loss: 2.47841477394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16325 , TFLOPS: 69.67368259429783, Tokens per sec: 172659.06065841182, Loss: 2.4910502433776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16326 , TFLOPS: 68.87294251939677, Tokens per sec: 170674.73854400593, Loss: 2.494826078414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16327 , TFLOPS: 71.49345800343687, Tokens per sec: 177168.66458706852, Loss: 2.4736921787261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16328 , TFLOPS: 70.57182986427755, Tokens per sec: 174884.768532511, Loss: 2.4864678382873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16329 , TFLOPS: 68.09293612706077, Tokens per sec: 168741.7967789999, Loss: 2.4836394786834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16330 , TFLOPS: 70.47195014644744, Tokens per sec: 174637.2555890688, Loss: 2.489762306213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16331 , TFLOPS: 71.5350590477915, Tokens per sec: 177271.75655771265, Loss: 2.4842896461486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16332 , TFLOPS: 70.51862073955792, Tokens per sec: 174752.9104599868, Loss: 2.496835231781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16333 , TFLOPS: 69.73244887639989, Tokens per sec: 172804.68997910112, Loss: 2.4839377403259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16334 , TFLOPS: 70.59794556524021, Tokens per sec: 174949.48611637845, Loss: 2.488595962524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16335 , TFLOPS: 69.62594749045056, Tokens per sec: 172540.7678126225, Loss: 2.4929583072662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16336 , TFLOPS: 68.8511740490152, Tokens per sec: 170620.79387640543, Loss: 2.4978811740875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16337 , TFLOPS: 71.54778915238967, Tokens per sec: 177303.3031592437, Loss: 2.4814183712005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16338 , TFLOPS: 70.43106885105546, Tokens per sec: 174535.947235641, Loss: 2.487304925918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16339 , TFLOPS: 68.27378236803241, Tokens per sec: 169189.95368598567, Loss: 2.4864025115966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16340 , TFLOPS: 71.04025267683485, Tokens per sec: 176045.57186306058, Loss: 2.5024242401123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16341 , TFLOPS: 70.55241950156658, Tokens per sec: 174836.66751548526, Loss: 2.484994888305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16342 , TFLOPS: 68.79576867591436, Tokens per sec: 170483.49325845586, Loss: 2.4992892742156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16343 , TFLOPS: 69.70119732089144, Tokens per sec: 172727.24518190816, Loss: 2.48689341545105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16344 , TFLOPS: 71.53720991982863, Tokens per sec: 177277.08665555873, Loss: 2.515265703201294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16345 , TFLOPS: 68.43921990616236, Tokens per sec: 169599.926129921, Loss: 2.4785349369049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16346 , TFLOPS: 68.99235644879448, Tokens per sec: 170970.65941558374, Loss: 2.4765141010284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16347 , TFLOPS: 71.49190799587981, Tokens per sec: 177164.8234975947, Loss: 2.4977447986602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16348 , TFLOPS: 70.53438774333613, Tokens per sec: 174791.98283222804, Loss: 2.48559308052063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16349 , TFLOPS: 68.19367052102561, Tokens per sec: 168991.42770405434, Loss: 2.477600336074829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16350 , TFLOPS: 70.61544340908276, Tokens per sec: 174992.84770096687, Loss: 2.5113680362701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16351 , TFLOPS: 71.50306091698732, Tokens per sec: 177192.4616646948, Loss: 2.4768636226654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16352 , TFLOPS: 69.45996361761735, Tokens per sec: 172129.4414911088, Loss: 2.470564365386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16353 , TFLOPS: 69.73294017833368, Tokens per sec: 172805.90747941443, Loss: 2.500836133956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16354 , TFLOPS: 71.55851685303382, Tokens per sec: 177329.88758319366, Loss: 2.484729290008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16355 , TFLOPS: 70.60142844809755, Tokens per sec: 174958.11708377014, Loss: 2.50888991355896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16356 , TFLOPS: 67.30130978599372, Tokens per sec: 166780.05950099535, Loss: 2.4956588745117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16357 , TFLOPS: 71.47331587755573, Tokens per sec: 177118.75018029675, Loss: 2.479240894317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16358 , TFLOPS: 71.52603401379567, Tokens per sec: 177249.3915292814, Loss: 2.505631923675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16359 , TFLOPS: 68.08223404910504, Tokens per sec: 168715.2758508944, Loss: 2.4888784885406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16360 , TFLOPS: 70.519928981561, Tokens per sec: 174756.1524277852, Loss: 2.498216390609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16361 , TFLOPS: 71.55087801167626, Tokens per sec: 177310.95769282183, Loss: 2.466749906539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16362 , TFLOPS: 69.62410192063739, Tokens per sec: 172536.1942873192, Loss: 2.4954729080200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16363 , TFLOPS: 68.1347089941015, Tokens per sec: 168845.31454518804, Loss: 2.491084098815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16364 , TFLOPS: 71.51929545072905, Tokens per sec: 177232.69262768832, Loss: 2.4828293323516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16365 , TFLOPS: 69.71963779211818, Tokens per sec: 172772.94270099362, Loss: 2.4868946075439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16366 , TFLOPS: 69.06349305038712, Tokens per sec: 171146.94375067385, Loss: 2.499373435974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16367 , TFLOPS: 70.38704077482804, Tokens per sec: 174426.84081265648, Loss: 2.4716386795043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16368 , TFLOPS: 71.51796390444503, Tokens per sec: 177229.39290931457, Loss: 2.4891257286071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16369 , TFLOPS: 67.99201173316692, Tokens per sec: 168491.6950131912, Loss: 2.4783105850219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16370 , TFLOPS: 70.54936109042602, Tokens per sec: 174829.08843576716, Loss: 2.5050742626190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16371 , TFLOPS: 70.47812010024639, Tokens per sec: 174652.5454142591, Loss: 2.4740161895751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16372 , TFLOPS: 70.40540898531086, Tokens per sec: 174472.3592616583, Loss: 2.4945309162139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16373 , TFLOPS: 69.6533461983713, Tokens per sec: 172608.66482906695, Loss: 2.4859180450439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16374 , TFLOPS: 70.62084267385146, Tokens per sec: 175006.22767384184, Loss: 2.503840923309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16375 , TFLOPS: 70.57673912698803, Tokens per sec: 174896.9342263075, Loss: 2.4904873371124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16376 , TFLOPS: 68.08125317453533, Tokens per sec: 168712.84513565843, Loss: 2.492396831512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16377 , TFLOPS: 71.48821695975325, Tokens per sec: 177155.67670347187, Loss: 2.485668659210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16378 , TFLOPS: 70.45337318655598, Tokens per sec: 174591.21983603667, Loss: 2.506476640701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16379 , TFLOPS: 68.85676239003288, Tokens per sec: 170634.64240105383, Loss: 2.4833695888519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16380 , TFLOPS: 70.11874509173526, Tokens per sec: 173761.97455474324, Loss: 2.4943017959594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16381 , TFLOPS: 71.51521948657339, Tokens per sec: 177222.59193950606, Loss: 2.4975595474243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16382 , TFLOPS: 69.67199182589283, Tokens per sec: 172654.87074805071, Loss: 2.4934194087982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16383 , TFLOPS: 68.89256278548702, Tokens per sec: 170723.3597247283, Loss: 2.4938971996307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16384 , TFLOPS: 71.5133228250107, Tokens per sec: 177217.89180321922, Loss: 2.475421190261841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16385 , TFLOPS: 69.5550961760491, Tokens per sec: 172365.19045061912, Loss: 2.519462823867798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16386 , TFLOPS: 68.32920533143854, Tokens per sec: 169327.29789464953, Loss: 2.4974617958068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16387 , TFLOPS: 71.55806919423534, Tokens per sec: 177328.77823537737, Loss: 2.4774584770202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16388 , TFLOPS: 71.48872356430587, Tokens per sec: 177156.93212536038, Loss: 2.494537830352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16389 , TFLOPS: 67.78703554612717, Tokens per sec: 167983.74144171565, Loss: 2.487806797027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16390 , TFLOPS: 70.03786042906577, Tokens per sec: 173561.53345046708, Loss: 2.5043535232543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16391 , TFLOPS: 71.51069075858771, Tokens per sec: 177211.369252677, Loss: 2.4939165115356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16392 , TFLOPS: 70.4651317651488, Tokens per sec: 174620.35888910404, Loss: 2.4909908771514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16393 , TFLOPS: 68.6126579206601, Tokens per sec: 170029.72463562092, Loss: 2.5061233043670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16394 , TFLOPS: 71.4812708621751, Tokens per sec: 177138.46350849737, Loss: 2.489675283432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16395 , TFLOPS: 70.61686892342817, Tokens per sec: 174996.3802825482, Loss: 2.480302095413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16396 , TFLOPS: 68.0461340711811, Tokens per sec: 168625.81612885802, Loss: 2.4993491172790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16397 , TFLOPS: 70.48700198531552, Tokens per sec: 174674.55570388082, Loss: 2.4774935245513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16398 , TFLOPS: 71.53337093907167, Tokens per sec: 177267.5732383444, Loss: 2.4755702018737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16399 , TFLOPS: 69.72289489256598, Tokens per sec: 172781.01415470245, Loss: 2.5118820667266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16400 , TFLOPS: 68.10628335747194, Tokens per sec: 168774.8726862776, Loss: 2.4955496788024902 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/model.pt +[rank0]:[2024-08-27 12:39:25,459] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006387257992173545, 'preprocessing_with_comm': 0.0013651229965034872, 'state_converting': 0.975507880008081, : 0.9846067750040675}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400, took 4.28s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014400 + eval ppl=9.466789245605469, eval loss=2.2477898597717285 +------------------------------------------------------------------ +iteration: 16401 , TFLOPS: 66.4829118372865, Tokens per sec: 164751.97923012238, Loss: 2.5058746337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16402 , TFLOPS: 69.662675771117, Tokens per sec: 172631.78453806808, Loss: 2.5054516792297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16403 , TFLOPS: 67.69106054750284, Tokens per sec: 167745.9048226057, Loss: 2.4805827140808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16404 , TFLOPS: 71.53167938253175, Tokens per sec: 177263.38137489813, Loss: 2.5087597370147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16405 , TFLOPS: 70.63725461659588, Tokens per sec: 175046.89827588727, Loss: 2.479665756225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16406 , TFLOPS: 68.39654676407699, Tokens per sec: 169494.17738299875, Loss: 2.4844844341278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16407 , TFLOPS: 70.51748101859884, Tokens per sec: 174750.08610589957, Loss: 2.4973413944244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16408 , TFLOPS: 71.51145668068831, Tokens per sec: 177213.26729201557, Loss: 2.4942569732666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16409 , TFLOPS: 69.2236671583415, Tokens per sec: 171543.8728348194, Loss: 2.4813222885131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16410 , TFLOPS: 70.10923641094304, Tokens per sec: 173738.41099627246, Loss: 2.499178409576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16411 , TFLOPS: 70.18532370756601, Tokens per sec: 173926.96369901183, Loss: 2.4599618911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16412 , TFLOPS: 69.76323139175015, Tokens per sec: 172880.97244311307, Loss: 2.4930343627929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16413 , TFLOPS: 69.60064726167982, Tokens per sec: 172478.07105867838, Loss: 2.5047078132629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16414 , TFLOPS: 71.52743458712501, Tokens per sec: 177252.8623042774, Loss: 2.4955639839172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16415 , TFLOPS: 69.44852606980018, Tokens per sec: 172101.09798766795, Loss: 2.4813945293426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16416 , TFLOPS: 69.00701453003272, Tokens per sec: 171006.983755613, Loss: 2.483062982559204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16417 , TFLOPS: 71.4663135396498, Tokens per sec: 177101.39761559461, Loss: 2.4821972846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16418 , TFLOPS: 70.4663818438493, Tokens per sec: 174623.45672183146, Loss: 2.488825559616089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16419 , TFLOPS: 67.46432264319223, Tokens per sec: 167184.0233184819, Loss: 2.5120882987976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16420 , TFLOPS: 70.65700290939775, Tokens per sec: 175095.83672090436, Loss: 2.493840456008911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16421 , TFLOPS: 71.53432736672282, Tokens per sec: 177269.94337142314, Loss: 2.463942050933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16422 , TFLOPS: 68.70864343142809, Tokens per sec: 170267.58730498087, Loss: 2.4823882579803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16423 , TFLOPS: 70.09036559782321, Tokens per sec: 173691.64704256982, Loss: 2.5015110969543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16424 , TFLOPS: 71.53272265206535, Tokens per sec: 177265.966711728, Loss: 2.5073251724243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16425 , TFLOPS: 70.67252929877016, Tokens per sec: 175134.31282414214, Loss: 2.505638599395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16426 , TFLOPS: 67.94991192554576, Tokens per sec: 168387.3670522293, Loss: 2.4887967109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16427 , TFLOPS: 70.85682894447268, Tokens per sec: 175591.027648475, Loss: 2.4730758666992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16428 , TFLOPS: 71.49786504325746, Tokens per sec: 177179.58571721948, Loss: 2.4824483394622803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16429 , TFLOPS: 68.89788242003267, Tokens per sec: 170736.54236513856, Loss: 2.4924325942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16430 , TFLOPS: 68.35049285926647, Tokens per sec: 169380.05073362187, Loss: 2.4668145179748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16431 , TFLOPS: 71.5038654971935, Tokens per sec: 177194.45550307754, Loss: 2.485271692276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16432 , TFLOPS: 69.87521194434555, Tokens per sec: 173158.4725881207, Loss: 2.4801971912384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16433 , TFLOPS: 68.32937288506305, Tokens per sec: 169327.71311098858, Loss: 2.4885268211364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16434 , TFLOPS: 71.51763766726022, Tokens per sec: 177228.58445763262, Loss: 2.4939064979553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16435 , TFLOPS: 70.52291270512724, Tokens per sec: 174763.5464234666, Loss: 2.507857322692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16436 , TFLOPS: 68.8891505036458, Tokens per sec: 170714.90371443186, Loss: 2.495163679122925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16437 , TFLOPS: 70.56496839512447, Tokens per sec: 174867.76505609642, Loss: 2.4972071647644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16438 , TFLOPS: 71.55527341071445, Tokens per sec: 177321.84997583163, Loss: 2.495868444442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16439 , TFLOPS: 68.94070644372316, Tokens per sec: 170842.66501329924, Loss: 2.4744298458099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16440 , TFLOPS: 68.46014945618056, Tokens per sec: 169651.79186044636, Loss: 2.475842237472534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16441 , TFLOPS: 71.48464191473384, Tokens per sec: 177146.81734809006, Loss: 2.509781837463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16442 , TFLOPS: 70.58421278807278, Tokens per sec: 174915.45478176724, Loss: 2.493195056915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16443 , TFLOPS: 69.2571439818294, Tokens per sec: 171626.83209119894, Loss: 2.4964230060577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16444 , TFLOPS: 70.39083692010533, Tokens per sec: 174436.2480788335, Loss: 2.4805917739868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16445 , TFLOPS: 71.4805338826846, Tokens per sec: 177136.63719213495, Loss: 2.4848837852478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16446 , TFLOPS: 68.42075588211254, Tokens per sec: 169554.17024434538, Loss: 2.5053606033325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16447 , TFLOPS: 70.67871780091158, Tokens per sec: 175149.6486141684, Loss: 2.4879629611968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16448 , TFLOPS: 70.50300131170503, Tokens per sec: 174714.20379714534, Loss: 2.473045587539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16449 , TFLOPS: 68.92029025909522, Tokens per sec: 170792.0714587628, Loss: 2.494140863418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16450 , TFLOPS: 69.75181061477626, Tokens per sec: 172852.67049966933, Loss: 2.4804506301879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16451 , TFLOPS: 71.45465917893071, Tokens per sec: 177072.5168258989, Loss: 2.4977548122406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16452 , TFLOPS: 68.66249242805736, Tokens per sec: 170153.22003467553, Loss: 2.503129720687866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16453 , TFLOPS: 69.582365961004, Tokens per sec: 172432.76798175217, Loss: 2.503851890563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16454 , TFLOPS: 71.48824248186467, Tokens per sec: 177155.73995007548, Loss: 2.478250741958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16455 , TFLOPS: 69.34839859216201, Tokens per sec: 171852.97106812836, Loss: 2.483363628387451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16456 , TFLOPS: 69.014060364116, Tokens per sec: 171024.44410863408, Loss: 2.5113978385925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16457 , TFLOPS: 71.49712122331245, Tokens per sec: 177177.74244945718, Loss: 2.475259304046631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16458 , TFLOPS: 71.53848373000206, Tokens per sec: 177280.2432974903, Loss: 2.4908454418182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16459 , TFLOPS: 68.82016830658223, Tokens per sec: 170543.95823109202, Loss: 2.506164073944092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16460 , TFLOPS: 69.15363230208473, Tokens per sec: 171370.3187460388, Loss: 2.5132412910461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16461 , TFLOPS: 71.56888534712147, Tokens per sec: 177355.58185372676, Loss: 2.473564386367798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16462 , TFLOPS: 70.51457037108119, Tokens per sec: 174742.87320071657, Loss: 2.483510971069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16463 , TFLOPS: 67.95106650509234, Tokens per sec: 168390.22822753317, Loss: 2.4907023906707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16464 , TFLOPS: 71.47500592934185, Tokens per sec: 177122.9383147973, Loss: 2.497584342956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16465 , TFLOPS: 71.55810174931504, Tokens per sec: 177328.8589104511, Loss: 2.493655204772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16466 , TFLOPS: 67.18179464207113, Tokens per sec: 166483.8878086752, Loss: 2.480586528778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16467 , TFLOPS: 70.63440349752071, Tokens per sec: 175039.8328887423, Loss: 2.5099990367889404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16468 , TFLOPS: 71.55431138427703, Tokens per sec: 177319.46596834375, Loss: 2.4888901710510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16469 , TFLOPS: 69.58258833230246, Tokens per sec: 172433.31904232598, Loss: 2.4902279376983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16470 , TFLOPS: 69.05681782490903, Tokens per sec: 171130.40180660237, Loss: 2.46713924407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16471 , TFLOPS: 71.5130744478218, Tokens per sec: 177217.2762971841, Loss: 2.4715867042541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16472 , TFLOPS: 68.93104876599566, Tokens per sec: 170818.73222400944, Loss: 2.4791266918182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16473 , TFLOPS: 69.20858727156512, Tokens per sec: 171506.50321420084, Loss: 2.4887313842773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16474 , TFLOPS: 70.44982940281389, Tokens per sec: 174582.4379495438, Loss: 2.481536388397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16475 , TFLOPS: 70.60309470053558, Tokens per sec: 174962.24624086445, Loss: 2.482407569885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16476 , TFLOPS: 69.62561925013013, Tokens per sec: 172539.95439694985, Loss: 2.488870859146118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16477 , TFLOPS: 69.77557119156795, Tokens per sec: 172911.55182640275, Loss: 2.491786479949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16478 , TFLOPS: 71.530984069898, Tokens per sec: 177261.65831359665, Loss: 2.480208158493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16479 , TFLOPS: 69.79150055667087, Tokens per sec: 172951.0265478922, Loss: 2.4818203449249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16480 , TFLOPS: 68.86192624885626, Tokens per sec: 170647.4390120642, Loss: 2.488158702850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16481 , TFLOPS: 70.36339344981576, Tokens per sec: 174368.24013630816, Loss: 2.4761922359466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16482 , TFLOPS: 70.5878628333056, Tokens per sec: 174924.49999593987, Loss: 2.4895694255828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16483 , TFLOPS: 68.91851827889687, Tokens per sec: 170787.68029663156, Loss: 2.4883227348327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16484 , TFLOPS: 71.50128152033352, Tokens per sec: 177188.05212376997, Loss: 2.5023021697998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16485 , TFLOPS: 70.43238313206449, Tokens per sec: 174539.2041687614, Loss: 2.5093047618865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16486 , TFLOPS: 68.17672973037047, Tokens per sec: 168949.44655862876, Loss: 2.4972238540649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16487 , TFLOPS: 70.72737339721014, Tokens per sec: 175270.22254165262, Loss: 2.5085203647613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16488 , TFLOPS: 71.55094863982954, Tokens per sec: 177311.13271716598, Loss: 2.4757843017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16489 , TFLOPS: 68.77013462104732, Tokens per sec: 170419.96924667104, Loss: 2.475470542907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16490 , TFLOPS: 68.75545530572369, Tokens per sec: 170383.59228623903, Loss: 2.474764823913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16491 , TFLOPS: 71.54538620946687, Tokens per sec: 177297.348401975, Loss: 2.483668088912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16492 , TFLOPS: 69.5296791602582, Tokens per sec: 172302.20428557383, Loss: 2.4618113040924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16493 , TFLOPS: 69.74535676005148, Tokens per sec: 172836.67713671052, Loss: 2.5307555198669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16494 , TFLOPS: 70.96483153971664, Tokens per sec: 175858.66997695755, Loss: 2.4903368949890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16495 , TFLOPS: 71.52552848429198, Tokens per sec: 177248.1387714826, Loss: 2.491184711456299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16496 , TFLOPS: 68.263292031167, Tokens per sec: 169163.9574756272, Loss: 2.478090286254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16497 , TFLOPS: 70.61042121543478, Tokens per sec: 174980.40215186693, Loss: 2.489478588104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16498 , TFLOPS: 71.51231890377507, Tokens per sec: 177215.40397580626, Loss: 2.522432565689087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16499 , TFLOPS: 69.68162991225591, Tokens per sec: 172678.7549877813, Loss: 2.4961400032043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16500 , TFLOPS: 68.01253974055861, Tokens per sec: 168542.5656768545, Loss: 2.4856584072113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16501 , TFLOPS: 71.52340384258687, Tokens per sec: 177242.87367528875, Loss: 2.5075292587280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16502 , TFLOPS: 70.63108118430404, Tokens per sec: 175031.59983060838, Loss: 2.510401964187622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16503 , TFLOPS: 68.84863445661098, Tokens per sec: 170614.50048666855, Loss: 2.476310968399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16504 , TFLOPS: 70.56873913445148, Tokens per sec: 174877.10936353, Loss: 2.4796175956726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16505 , TFLOPS: 71.54345997865038, Tokens per sec: 177292.57498981987, Loss: 2.477247714996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16506 , TFLOPS: 68.89072058035394, Tokens per sec: 170718.7945374729, Loss: 2.4638888835906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16507 , TFLOPS: 69.9935549754603, Tokens per sec: 173451.7396557841, Loss: 2.5041487216949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16508 , TFLOPS: 71.48024603004986, Tokens per sec: 177135.92386159647, Loss: 2.477619171142578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16509 , TFLOPS: 70.51938908716514, Tokens per sec: 174754.81450999796, Loss: 2.5106444358825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16510 , TFLOPS: 68.35819896124809, Tokens per sec: 169399.14730322955, Loss: 2.4889180660247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16511 , TFLOPS: 70.32312585430374, Tokens per sec: 174268.45259878755, Loss: 2.483873128890991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16512 , TFLOPS: 71.53822884629298, Tokens per sec: 177279.61166758143, Loss: 2.491889476776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16513 , TFLOPS: 68.84993230765035, Tokens per sec: 170617.71670451423, Loss: 2.4744391441345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16514 , TFLOPS: 69.69228202317105, Tokens per sec: 172705.1520920559, Loss: 2.488572359085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16515 , TFLOPS: 71.45446296706511, Tokens per sec: 177072.03059128142, Loss: 2.483051061630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16516 , TFLOPS: 69.08396253643366, Tokens per sec: 171197.66939199722, Loss: 2.482893466949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16517 , TFLOPS: 70.59765156937338, Tokens per sec: 174948.7575622341, Loss: 2.4958925247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16518 , TFLOPS: 70.49687264985494, Tokens per sec: 174699.01629795294, Loss: 2.4805595874786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16519 , TFLOPS: 69.75088745563502, Tokens per sec: 172850.38280962297, Loss: 2.4915878772735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16520 , TFLOPS: 68.99999808340925, Tokens per sec: 170989.59622795464, Loss: 2.491152048110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16521 , TFLOPS: 71.51321498504859, Tokens per sec: 177217.62456391356, Loss: 2.4728474617004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16522 , TFLOPS: 69.79221871355325, Tokens per sec: 172952.80621975826, Loss: 2.475076913833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16523 , TFLOPS: 68.83971790174867, Tokens per sec: 170592.40428148015, Loss: 2.4888250827789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16524 , TFLOPS: 71.52432456127399, Tokens per sec: 177245.15531762093, Loss: 2.509552478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16525 , TFLOPS: 71.52287251361449, Tokens per sec: 177241.55698356446, Loss: 2.4899215698242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16526 , TFLOPS: 68.59490017227222, Tokens per sec: 169985.71897893827, Loss: 2.4871509075164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16527 , TFLOPS: 69.2197129442378, Tokens per sec: 171534.07385667777, Loss: 2.472151279449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16528 , TFLOPS: 71.53139298309536, Tokens per sec: 177262.67164554523, Loss: 2.4796152114868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16529 , TFLOPS: 70.52165981218732, Tokens per sec: 174760.44161674398, Loss: 2.4791438579559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16530 , TFLOPS: 68.07350414937049, Tokens per sec: 168693.64219767437, Loss: 2.4872546195983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16531 , TFLOPS: 71.5427372324745, Tokens per sec: 177290.78394517896, Loss: 2.4744131565093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16532 , TFLOPS: 71.52833526066556, Tokens per sec: 177255.09427252828, Loss: 2.4871835708618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16533 , TFLOPS: 66.79450967655788, Tokens per sec: 165524.1530011725, Loss: 2.4919209480285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16534 , TFLOPS: 70.7669484523622, Tokens per sec: 175368.29388787164, Loss: 2.4992785453796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16535 , TFLOPS: 71.48341295129279, Tokens per sec: 177143.77184130167, Loss: 2.500156879425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16536 , TFLOPS: 68.92651019965166, Tokens per sec: 170807.48515664466, Loss: 2.4959216117858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16537 , TFLOPS: 69.60177705588593, Tokens per sec: 172480.87081317822, Loss: 2.473278045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16538 , TFLOPS: 71.45587442373342, Tokens per sec: 177075.52833639024, Loss: 2.49135160446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16539 , TFLOPS: 69.80114453506336, Tokens per sec: 172974.92538872073, Loss: 2.4865598678588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16540 , TFLOPS: 68.72309720545036, Tokens per sec: 170303.4053492227, Loss: 2.4877138137817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16541 , TFLOPS: 71.50205269679128, Tokens per sec: 177189.9631839844, Loss: 2.4901912212371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16542 , TFLOPS: 70.6809404835183, Tokens per sec: 175155.15666651653, Loss: 2.485079050064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16543 , TFLOPS: 69.70803879919545, Tokens per sec: 172744.19911879653, Loss: 2.480809450149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16544 , TFLOPS: 70.1254868009091, Tokens per sec: 173778.6812527366, Loss: 2.4684524536132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16545 , TFLOPS: 71.48963914284218, Tokens per sec: 177159.20102983268, Loss: 2.503624677658081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16546 , TFLOPS: 69.75876188878922, Tokens per sec: 172869.8965224195, Loss: 2.4982447624206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16547 , TFLOPS: 68.96756270315309, Tokens per sec: 170909.2177826278, Loss: 2.486588478088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16548 , TFLOPS: 70.47114692491348, Tokens per sec: 174635.2651176255, Loss: 2.5197527408599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16549 , TFLOPS: 70.66330111437861, Tokens per sec: 175111.44436665147, Loss: 2.4903206825256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16550 , TFLOPS: 68.07007088467475, Tokens per sec: 168685.13418955298, Loss: 2.4959676265716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16551 , TFLOPS: 70.50442038405701, Tokens per sec: 174717.72041475726, Loss: 2.508392810821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16552 , TFLOPS: 71.5333917576326, Tokens per sec: 177267.62482903173, Loss: 2.4957685470581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16553 , TFLOPS: 68.25162072203929, Tokens per sec: 169135.03468590789, Loss: 2.504702568054199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16554 , TFLOPS: 70.64677833868384, Tokens per sec: 175070.49910834586, Loss: 2.482438564300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16555 , TFLOPS: 70.47798660395411, Tokens per sec: 174652.214595741, Loss: 2.4685099124908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16556 , TFLOPS: 69.72177453129909, Tokens per sec: 172778.23777606516, Loss: 2.488424301147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16557 , TFLOPS: 70.10087934337412, Tokens per sec: 173717.70126222965, Loss: 2.492716073989868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16558 , TFLOPS: 71.54245000014268, Tokens per sec: 177290.07215181948, Loss: 2.5083489418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16559 , TFLOPS: 69.63731516850012, Tokens per sec: 172568.93817108244, Loss: 2.5025525093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16560 , TFLOPS: 69.0374102689615, Tokens per sec: 171082.30774504557, Loss: 2.4931087493896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16561 , TFLOPS: 70.95901248196321, Tokens per sec: 175844.24971082242, Loss: 2.4684929847717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16562 , TFLOPS: 71.45622255895455, Tokens per sec: 177076.3910538185, Loss: 2.474313974380493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16563 , TFLOPS: 68.27347192117655, Tokens per sec: 169189.1843644783, Loss: 2.4778003692626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16564 , TFLOPS: 70.60542143014536, Tokens per sec: 174968.01213314792, Loss: 2.5035433769226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16565 , TFLOPS: 71.49918468875546, Tokens per sec: 177182.85594413444, Loss: 2.4990336894989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16566 , TFLOPS: 69.6263833420746, Tokens per sec: 172541.84790095993, Loss: 2.497170925140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16567 , TFLOPS: 68.9776733735295, Tokens per sec: 170934.2731376028, Loss: 2.50712513923645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16568 , TFLOPS: 71.52885424329166, Tokens per sec: 177256.38036864638, Loss: 2.4755825996398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16569 , TFLOPS: 70.4165336843047, Tokens per sec: 174499.92749125182, Loss: 2.4870657920837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16570 , TFLOPS: 68.01551973777207, Tokens per sec: 168549.95043822323, Loss: 2.505234479904175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16571 , TFLOPS: 71.48183823809106, Tokens per sec: 177139.86952851876, Loss: 2.5053277015686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16572 , TFLOPS: 71.51679822128239, Tokens per sec: 177226.50421802697, Loss: 2.4853687286376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16573 , TFLOPS: 69.67262085291063, Tokens per sec: 172656.42954629386, Loss: 2.4847960472106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16574 , TFLOPS: 69.60057662492919, Tokens per sec: 172477.896013029, Loss: 2.5062880516052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16575 , TFLOPS: 71.5057415789751, Tokens per sec: 177199.1046403998, Loss: 2.4794528484344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16576 , TFLOPS: 70.54842895836887, Tokens per sec: 174826.7785098467, Loss: 2.4971699714660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16577 , TFLOPS: 69.1130706731187, Tokens per sec: 171269.80256122872, Loss: 2.5093538761138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16578 , TFLOPS: 70.50557470560761, Tokens per sec: 174720.58095071866, Loss: 2.5077943801879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16579 , TFLOPS: 71.47303456694513, Tokens per sec: 177118.05306161472, Loss: 2.479264974594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16580 , TFLOPS: 68.19482018816925, Tokens per sec: 168994.27670588257, Loss: 2.5006179809570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16581 , TFLOPS: 69.63168102389349, Tokens per sec: 172554.97614009498, Loss: 2.4854953289031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16582 , TFLOPS: 71.41228693742907, Tokens per sec: 176967.51374377013, Loss: 2.4743857383728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16583 , TFLOPS: 68.99085307335196, Tokens per sec: 170966.93388562667, Loss: 2.4852662086486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16584 , TFLOPS: 70.61151630853077, Tokens per sec: 174983.11591319324, Loss: 2.4893252849578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16585 , TFLOPS: 70.56488173620714, Tokens per sec: 174867.55030575304, Loss: 2.4888675212860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16586 , TFLOPS: 69.64033657234809, Tokens per sec: 172576.4255426528, Loss: 2.474580764770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16587 , TFLOPS: 68.83721172429695, Tokens per sec: 170586.1936976766, Loss: 2.508340358734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16588 , TFLOPS: 70.48683394942157, Tokens per sec: 174674.13929242492, Loss: 2.501091241836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16589 , TFLOPS: 70.04541340723455, Tokens per sec: 173580.25056239855, Loss: 2.4887115955352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16590 , TFLOPS: 68.91126887585833, Tokens per sec: 170769.71547731437, Loss: 2.496098756790161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16591 , TFLOPS: 70.95702399380298, Tokens per sec: 175839.3220181113, Loss: 2.4839026927948 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16592 , TFLOPS: 70.50717727201942, Tokens per sec: 174724.5522868244, Loss: 2.4758830070495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16593 , TFLOPS: 69.5622943776236, Tokens per sec: 172383.02838706807, Loss: 2.5079989433288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16594 , TFLOPS: 69.60384122264287, Tokens per sec: 172485.98604578862, Loss: 2.4749600887298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16595 , TFLOPS: 71.56596146805546, Tokens per sec: 177348.33615930934, Loss: 2.485433578491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16596 , TFLOPS: 69.68727013444052, Tokens per sec: 172692.73207967656, Loss: 2.496860980987549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16597 , TFLOPS: 68.43290666416883, Tokens per sec: 169584.28122079998, Loss: 2.476961374282837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16598 , TFLOPS: 71.53426140096548, Tokens per sec: 177269.7799010097, Loss: 2.498427152633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16599 , TFLOPS: 71.50683388810447, Tokens per sec: 177201.81150275018, Loss: 2.4737155437469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16600 , TFLOPS: 67.80430951221948, Tokens per sec: 168026.54823257684, Loss: 2.4728405475616455 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/model.pt +[rank0]:[2024-08-27 13:04:43,635] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006359232007525861, 'preprocessing_with_comm': 0.002327887006686069, 'state_converting': 0.984421884000767, : 0.9944953679951141}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600, took 4.34s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014600 + eval ppl=9.80552864074707, eval loss=2.2829463481903076 +------------------------------------------------------------------ +iteration: 16601 , TFLOPS: 65.94344446658266, Tokens per sec: 163415.11965828156, Loss: 2.4704275131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16602 , TFLOPS: 71.54266358504539, Tokens per sec: 177290.60143873672, Loss: 2.475940704345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16603 , TFLOPS: 67.6430094048579, Tokens per sec: 167626.8287978612, Loss: 2.488323450088501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16604 , TFLOPS: 69.8205089035525, Tokens per sec: 173022.9124269981, Loss: 2.483106851577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16605 , TFLOPS: 71.46636921216425, Tokens per sec: 177101.53557821826, Loss: 2.478701114654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16606 , TFLOPS: 70.65378337492443, Tokens per sec: 175087.85835981846, Loss: 2.477682113647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16607 , TFLOPS: 67.38264241734221, Tokens per sec: 166981.61072100588, Loss: 2.4957592487335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16608 , TFLOPS: 71.5507785485069, Tokens per sec: 177310.71121213134, Loss: 2.5087289810180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16609 , TFLOPS: 70.50718732835274, Tokens per sec: 174724.57720752616, Loss: 2.4806149005889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16610 , TFLOPS: 67.0565696452889, Tokens per sec: 166173.5664719767, Loss: 2.5057668685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16611 , TFLOPS: 71.50692397675665, Tokens per sec: 177202.03475235435, Loss: 2.473694324493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16612 , TFLOPS: 71.47982547419552, Tokens per sec: 177134.88167786138, Loss: 2.4911231994628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16613 , TFLOPS: 68.74626170181182, Tokens per sec: 170360.80952298635, Loss: 2.4824132919311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16614 , TFLOPS: 69.59895364981524, Tokens per sec: 172473.87410191118, Loss: 2.4838197231292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16615 , TFLOPS: 71.53804628781563, Tokens per sec: 177279.1592675641, Loss: 2.49682879447937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16616 , TFLOPS: 70.60202832077427, Tokens per sec: 174959.60363434447, Loss: 2.496769905090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16617 , TFLOPS: 69.00910858486765, Tokens per sec: 171012.17305417315, Loss: 2.4745168685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16618 , TFLOPS: 70.50915907864864, Tokens per sec: 174729.4634219656, Loss: 2.5043184757232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16619 , TFLOPS: 70.6345142351125, Tokens per sec: 175040.10730869355, Loss: 2.478268623352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16620 , TFLOPS: 69.0355180634796, Tokens per sec: 171077.6186514178, Loss: 2.4938440322875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16621 , TFLOPS: 70.89841757448072, Tokens per sec: 175694.08885500068, Loss: 2.5016071796417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16622 , TFLOPS: 70.45304553622314, Tokens per sec: 174590.40788241828, Loss: 2.475311756134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16623 , TFLOPS: 69.98050255590891, Tokens per sec: 173419.39432229308, Loss: 2.483970880508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16624 , TFLOPS: 69.98614236353724, Tokens per sec: 173433.37038687215, Loss: 2.4731552600860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16625 , TFLOPS: 70.60427808266186, Tokens per sec: 174965.17879213367, Loss: 2.496424674987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16626 , TFLOPS: 70.67570520814697, Tokens per sec: 175142.18307743318, Loss: 2.4961352348327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16627 , TFLOPS: 67.59598162701724, Tokens per sec: 167510.2887838346, Loss: 2.471853256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16628 , TFLOPS: 71.50456570021454, Tokens per sec: 177196.19068329738, Loss: 2.485408067703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16629 , TFLOPS: 69.52874984611651, Tokens per sec: 172299.90134275833, Loss: 2.4795985221862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16630 , TFLOPS: 68.13959427387381, Tokens per sec: 168857.4208066218, Loss: 2.477635622024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16631 , TFLOPS: 71.45949699710957, Tokens per sec: 177084.50547227578, Loss: 2.4908206462860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16632 , TFLOPS: 71.51612625749102, Tokens per sec: 177224.83901772904, Loss: 2.4790215492248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16633 , TFLOPS: 68.8629171452017, Tokens per sec: 170649.8945623644, Loss: 2.4739339351654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16634 , TFLOPS: 69.84797144763306, Tokens per sec: 173090.9676364787, Loss: 2.482527494430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16635 , TFLOPS: 71.51874819363344, Tokens per sec: 177231.33646432005, Loss: 2.4969325065612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16636 , TFLOPS: 70.59184341924582, Tokens per sec: 174934.36432639748, Loss: 2.4963510036468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16637 , TFLOPS: 67.69381809123315, Tokens per sec: 167752.7383197379, Loss: 2.4831831455230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16638 , TFLOPS: 71.4737013212971, Tokens per sec: 177119.70535234798, Loss: 2.4934492111206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16639 , TFLOPS: 71.51259308580119, Tokens per sec: 177216.08342907063, Loss: 2.5079762935638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16640 , TFLOPS: 68.26518893244818, Tokens per sec: 169168.65820596289, Loss: 2.4854934215545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16641 , TFLOPS: 70.67650407014365, Tokens per sec: 175144.16274546288, Loss: 2.470080614089966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16642 , TFLOPS: 71.48481230961761, Tokens per sec: 177147.2396053826, Loss: 2.524043083190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16643 , TFLOPS: 69.72196209587457, Tokens per sec: 172778.70258174516, Loss: 2.4921324253082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16644 , TFLOPS: 68.80962280299111, Tokens per sec: 170517.82531151015, Loss: 2.5034327507019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16645 , TFLOPS: 71.54915759580678, Tokens per sec: 177306.69431277775, Loss: 2.488420009613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16646 , TFLOPS: 70.58671939556372, Tokens per sec: 174921.66643125543, Loss: 2.498460292816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16647 , TFLOPS: 67.06981337278872, Tokens per sec: 166206.38588763823, Loss: 2.5073182582855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16648 , TFLOPS: 71.4708146796188, Tokens per sec: 177112.55193627928, Loss: 2.4835143089294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16649 , TFLOPS: 70.51808265913051, Tokens per sec: 174751.57703741326, Loss: 2.482797145843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16650 , TFLOPS: 68.94962647181185, Tokens per sec: 170864.76982552523, Loss: 2.469820261001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16651 , TFLOPS: 70.00989569738253, Tokens per sec: 173492.2337647865, Loss: 2.501366138458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16652 , TFLOPS: 71.48220393099896, Tokens per sec: 177140.77575582894, Loss: 2.475572109222412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16653 , TFLOPS: 70.5521921104795, Tokens per sec: 174836.1040153223, Loss: 2.487494468688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16654 , TFLOPS: 68.78782937743983, Tokens per sec: 170463.81880225003, Loss: 2.49594783782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16655 , TFLOPS: 70.4878394277753, Tokens per sec: 174676.63097854806, Loss: 2.475019931793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16656 , TFLOPS: 70.58261419871253, Tokens per sec: 174911.4933012334, Loss: 2.519742727279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16657 , TFLOPS: 68.88386349593375, Tokens per sec: 170701.80192690942, Loss: 2.4762725830078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16658 , TFLOPS: 71.50459676772405, Tokens per sec: 177196.26767200834, Loss: 2.473381996154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16659 , TFLOPS: 70.57739232138411, Tokens per sec: 174898.5529139763, Loss: 2.472825765609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16660 , TFLOPS: 68.00726050029408, Tokens per sec: 168529.48313792268, Loss: 2.493046998977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16661 , TFLOPS: 70.64705672724729, Tokens per sec: 175071.1889858731, Loss: 2.465470314025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16662 , TFLOPS: 70.43079190919468, Tokens per sec: 174535.26094320233, Loss: 2.5198352336883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16663 , TFLOPS: 69.84334574225574, Tokens per sec: 173079.50462899017, Loss: 2.490699291229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16664 , TFLOPS: 69.6206113781486, Tokens per sec: 172527.54433851695, Loss: 2.4671387672424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16665 , TFLOPS: 71.49129265514375, Tokens per sec: 177163.29861546535, Loss: 2.507225513458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16666 , TFLOPS: 69.6566654845092, Tokens per sec: 172616.89038576622, Loss: 2.498671770095825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16667 , TFLOPS: 68.06195335205342, Tokens per sec: 168665.01804949727, Loss: 2.4829020500183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16668 , TFLOPS: 71.52902970981914, Tokens per sec: 177256.81519403073, Loss: 2.496316909790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16669 , TFLOPS: 71.54089062688573, Tokens per sec: 177286.20785310963, Loss: 2.4886655807495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16670 , TFLOPS: 68.18378778492767, Tokens per sec: 168966.9371953304, Loss: 2.490565776824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16671 , TFLOPS: 70.12290420535045, Tokens per sec: 173772.28129644587, Loss: 2.4770398139953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16672 , TFLOPS: 71.51123001059169, Tokens per sec: 177212.70557854636, Loss: 2.486830234527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16673 , TFLOPS: 69.63758990822087, Tokens per sec: 172569.61900637555, Loss: 2.459810972213745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16674 , TFLOPS: 68.65233449324222, Tokens per sec: 170128.0475532147, Loss: 2.480482578277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16675 , TFLOPS: 71.48037197189282, Tokens per sec: 177136.23595935697, Loss: 2.4980664253234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16676 , TFLOPS: 70.63492937907347, Tokens per sec: 175041.13608116066, Loss: 2.482938528060913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16677 , TFLOPS: 67.89655165722452, Tokens per sec: 168255.13442921045, Loss: 2.488276481628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16678 , TFLOPS: 71.54322753708149, Tokens per sec: 177291.99897400662, Loss: 2.4716897010803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16679 , TFLOPS: 71.47040128760365, Tokens per sec: 177111.52750532684, Loss: 2.4746689796447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16680 , TFLOPS: 68.02136606518349, Tokens per sec: 168564.43828157484, Loss: 2.4721803665161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16681 , TFLOPS: 69.73602190328637, Tokens per sec: 172813.5443333272, Loss: 2.501556396484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16682 , TFLOPS: 71.5483143445013, Tokens per sec: 177304.60464315102, Loss: 2.4797468185424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16683 , TFLOPS: 70.5685047999441, Tokens per sec: 174876.52865680662, Loss: 2.508835792541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16684 , TFLOPS: 68.36914745788651, Tokens per sec: 169426.27888397602, Loss: 2.4836347103118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16685 , TFLOPS: 71.38326385079112, Tokens per sec: 176895.59133792433, Loss: 2.4938268661499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16686 , TFLOPS: 70.61624060808796, Tokens per sec: 174994.82324792052, Loss: 2.497659206390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16687 , TFLOPS: 68.82588273727872, Tokens per sec: 170558.11921985442, Loss: 2.4852781295776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16688 , TFLOPS: 69.86777715931098, Tokens per sec: 173140.0483717958, Loss: 2.4972848892211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16689 , TFLOPS: 71.51819279082241, Tokens per sec: 177229.96011497168, Loss: 2.4831137657165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16690 , TFLOPS: 69.77352183824145, Tokens per sec: 172906.47330310478, Loss: 2.4973626136779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16691 , TFLOPS: 69.23388897940086, Tokens per sec: 171569.20363343103, Loss: 2.506411075592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16692 , TFLOPS: 70.54680705041834, Tokens per sec: 174822.7592432774, Loss: 2.468381643295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16693 , TFLOPS: 70.56937198510842, Tokens per sec: 174878.6776371722, Loss: 2.4905593395233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16694 , TFLOPS: 68.75510157765744, Tokens per sec: 170382.71570912452, Loss: 2.4968104362487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16695 , TFLOPS: 71.4774983977534, Tokens per sec: 177129.11492608915, Loss: 2.4851317405700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16696 , TFLOPS: 69.58388399609986, Tokens per sec: 172436.52983994505, Loss: 2.4926679134368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16697 , TFLOPS: 68.81237457878773, Tokens per sec: 170524.64451506728, Loss: 2.4865217208862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16698 , TFLOPS: 71.49817455035742, Tokens per sec: 177180.35270990818, Loss: 2.514314651489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16699 , TFLOPS: 70.56908600626495, Tokens per sec: 174877.9689500949, Loss: 2.4880218505859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16700 , TFLOPS: 68.80888221992154, Tokens per sec: 170515.99006508177, Loss: 2.499251127243042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16701 , TFLOPS: 70.12998997970492, Tokens per sec: 173789.8406258587, Loss: 2.4718875885009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16702 , TFLOPS: 71.5281803326675, Tokens per sec: 177254.71034388235, Loss: 2.4832406044006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16703 , TFLOPS: 69.5802312837297, Tokens per sec: 172427.47801631232, Loss: 2.468451499938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16704 , TFLOPS: 69.04114721940675, Tokens per sec: 171091.568319908, Loss: 2.499087333679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16705 , TFLOPS: 71.51005684896383, Tokens per sec: 177209.7983547981, Loss: 2.50691294670105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16706 , TFLOPS: 71.49554078908288, Tokens per sec: 177173.82595933712, Loss: 2.5068962574005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16707 , TFLOPS: 67.12767178120903, Tokens per sec: 166349.7653973336, Loss: 2.4792373180389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16708 , TFLOPS: 70.65177108510885, Tokens per sec: 175082.87168398342, Loss: 2.508669853210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16709 , TFLOPS: 71.53525602153599, Tokens per sec: 177272.24468034992, Loss: 2.493251323699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16710 , TFLOPS: 69.83637253516096, Tokens per sec: 173062.22425364715, Loss: 2.4825921058654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16711 , TFLOPS: 67.81566374134994, Tokens per sec: 168054.68526313372, Loss: 2.499300003051758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16712 , TFLOPS: 71.53810391849694, Tokens per sec: 177279.3020827409, Loss: 2.4843945503234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16713 , TFLOPS: 70.54221408328567, Tokens per sec: 174811.37736476553, Loss: 2.503814935684204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16714 , TFLOPS: 67.95511105659801, Tokens per sec: 168400.2510717083, Loss: 2.4739649295806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16715 , TFLOPS: 71.55654689712242, Tokens per sec: 177325.00581543666, Loss: 2.5033469200134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16716 , TFLOPS: 70.65841653235985, Tokens per sec: 175099.33983432787, Loss: 2.4848549365997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16717 , TFLOPS: 68.5526862037992, Tokens per sec: 169881.1081148096, Loss: 2.4715681076049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16718 , TFLOPS: 70.41887539873642, Tokens per sec: 174505.7305175748, Loss: 2.483018636703491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16719 , TFLOPS: 71.53509669491135, Tokens per sec: 177271.84985142277, Loss: 2.4881980419158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16720 , TFLOPS: 69.70590768347542, Tokens per sec: 172738.9179792805, Loss: 2.4949066638946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16721 , TFLOPS: 68.27816756840465, Tokens per sec: 169200.8206955778, Loss: 2.4929916858673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16722 , TFLOPS: 71.51619492361574, Tokens per sec: 177225.00917995028, Loss: 2.492784261703491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16723 , TFLOPS: 70.65153217315144, Tokens per sec: 175082.27963383464, Loss: 2.4942400455474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16724 , TFLOPS: 68.86909273633121, Tokens per sec: 170665.19835748087, Loss: 2.4857237339019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16725 , TFLOPS: 70.51639626934151, Tokens per sec: 174747.39797774493, Loss: 2.4873197078704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16726 , TFLOPS: 71.53609803715733, Tokens per sec: 177274.3312878153, Loss: 2.5079405307769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16727 , TFLOPS: 68.25154656763455, Tokens per sec: 169134.85092312415, Loss: 2.4899182319641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16728 , TFLOPS: 70.72731750960682, Tokens per sec: 175270.08404601508, Loss: 2.4769389629364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16729 , TFLOPS: 70.52869169428347, Tokens per sec: 174777.86739520397, Loss: 2.4836394786834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16730 , TFLOPS: 70.60898954835555, Tokens per sec: 174976.85432313342, Loss: 2.48360013961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16731 , TFLOPS: 68.14681584462757, Tokens per sec: 168875.3166544713, Loss: 2.4761834144592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16732 , TFLOPS: 71.49869037248723, Tokens per sec: 177181.63097396833, Loss: 2.477661371231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16733 , TFLOPS: 69.60384445283502, Tokens per sec: 172485.9940505606, Loss: 2.491330623626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16734 , TFLOPS: 68.85028827468504, Tokens per sec: 170618.59883003918, Loss: 2.495742082595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16735 , TFLOPS: 71.54716337393616, Tokens per sec: 177301.75241130072, Loss: 2.5031232833862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16736 , TFLOPS: 69.61445814768811, Tokens per sec: 172512.29595560156, Loss: 2.5066909790039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16737 , TFLOPS: 69.67098548532508, Tokens per sec: 172652.3769252663, Loss: 2.487215995788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16738 , TFLOPS: 70.90110816178228, Tokens per sec: 175700.7564267266, Loss: 2.48063325881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16739 , TFLOPS: 71.47602125126484, Tokens per sec: 177125.4543943694, Loss: 2.497014045715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16740 , TFLOPS: 69.68962317584133, Tokens per sec: 172698.56317547665, Loss: 2.511564254760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16741 , TFLOPS: 68.41959281957371, Tokens per sec: 169551.28804725225, Loss: 2.5045759677886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16742 , TFLOPS: 71.52732181477653, Tokens per sec: 177252.58284197497, Loss: 2.461801528930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16743 , TFLOPS: 70.68484916937646, Tokens per sec: 175164.84282065037, Loss: 2.4927072525024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16744 , TFLOPS: 67.84821610523929, Tokens per sec: 168135.35360679042, Loss: 2.4946389198303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16745 , TFLOPS: 71.50878428281987, Tokens per sec: 177206.64479570865, Loss: 2.491570472717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16746 , TFLOPS: 71.51780927245439, Tokens per sec: 177229.00971420764, Loss: 2.510190010070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16747 , TFLOPS: 68.14787834523406, Tokens per sec: 168877.94964801832, Loss: 2.485414981842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16748 , TFLOPS: 69.8106618652359, Tokens per sec: 172998.51038130876, Loss: 2.4782047271728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16749 , TFLOPS: 71.53724443741207, Tokens per sec: 177277.17219393296, Loss: 2.501338481903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16750 , TFLOPS: 70.00259379456025, Tokens per sec: 173474.1388452219, Loss: 2.47265362739563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16751 , TFLOPS: 67.9019935560946, Tokens per sec: 168268.62005408513, Loss: 2.4743239879608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16752 , TFLOPS: 71.51922967718188, Tokens per sec: 177232.5296335929, Loss: 2.4949018955230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16753 , TFLOPS: 70.48960628016499, Tokens per sec: 174681.00943340457, Loss: 2.4964120388031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16754 , TFLOPS: 69.55254649473846, Tokens per sec: 172358.87205946076, Loss: 2.5017127990722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16755 , TFLOPS: 70.01605072847674, Tokens per sec: 173507.48660987063, Loss: 2.4972434043884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16756 , TFLOPS: 71.51465105909901, Tokens per sec: 177221.1833136071, Loss: 2.495987892150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16757 , TFLOPS: 70.01080782941469, Tokens per sec: 173494.49412844185, Loss: 2.4816246032714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16758 , TFLOPS: 68.9262964888132, Tokens per sec: 170806.95555764475, Loss: 2.507479190826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16759 , TFLOPS: 71.45231128195465, Tokens per sec: 177066.69847855, Loss: 2.4987220764160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16760 , TFLOPS: 69.99053900452785, Tokens per sec: 173444.26574757686, Loss: 2.493374824523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16761 , TFLOPS: 68.26369673940417, Tokens per sec: 169164.9603872212, Loss: 2.477264404296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16762 , TFLOPS: 70.49459494141999, Tokens per sec: 174693.3718855409, Loss: 2.5071372985839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16763 , TFLOPS: 70.57946331315372, Tokens per sec: 174903.68505971998, Loss: 2.4543378353118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16764 , TFLOPS: 68.75590217656114, Tokens per sec: 170384.6996814011, Loss: 2.490879535675049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16765 , TFLOPS: 71.53909146372273, Tokens per sec: 177281.7493285975, Loss: 2.4667060375213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16766 , TFLOPS: 70.37367935497552, Tokens per sec: 174393.72974238003, Loss: 2.4988913536071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16767 , TFLOPS: 68.93963713875026, Tokens per sec: 170840.0151577827, Loss: 2.474935293197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16768 , TFLOPS: 70.19047743520224, Tokens per sec: 173939.73520382436, Loss: 2.4990875720977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16769 , TFLOPS: 71.52083412449801, Tokens per sec: 177236.5056307867, Loss: 2.4769835472106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16770 , TFLOPS: 69.65566583660245, Tokens per sec: 172614.41314813297, Loss: 2.4925315380096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16771 , TFLOPS: 67.96813416651622, Tokens per sec: 168432.52377269964, Loss: 2.481234550476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16772 , TFLOPS: 71.56904766886093, Tokens per sec: 177355.98410487841, Loss: 2.512819528579712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16773 , TFLOPS: 70.50914281473187, Tokens per sec: 174729.42311818825, Loss: 2.4821197986602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16774 , TFLOPS: 68.97657724703828, Tokens per sec: 170931.55681540928, Loss: 2.4849250316619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16775 , TFLOPS: 70.68224828182021, Tokens per sec: 175158.39753477453, Loss: 2.499748945236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16776 , TFLOPS: 71.51849988475907, Tokens per sec: 177230.72112757587, Loss: 2.494945526123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16777 , TFLOPS: 68.7687781517784, Tokens per sec: 170416.60776639526, Loss: 2.4708051681518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16778 , TFLOPS: 69.62355943161207, Tokens per sec: 172534.84993975432, Loss: 2.4863736629486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16779 , TFLOPS: 71.46127499959232, Tokens per sec: 177088.91155829118, Loss: 2.4849612712860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16780 , TFLOPS: 70.61109082626459, Tokens per sec: 174982.06152126688, Loss: 2.5013058185577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16781 , TFLOPS: 67.23406862253651, Tokens per sec: 166613.42849072273, Loss: 2.500718116760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16782 , TFLOPS: 71.56309091366047, Tokens per sec: 177341.2226093009, Loss: 2.5006954669952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16783 , TFLOPS: 70.57164456640133, Tokens per sec: 174884.3093439615, Loss: 2.4875056743621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16784 , TFLOPS: 68.86485933304203, Tokens per sec: 170654.7075177824, Loss: 2.474158525466919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16785 , TFLOPS: 70.52053246735802, Tokens per sec: 174757.64793206967, Loss: 2.5029594898223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16786 , TFLOPS: 71.51688674114544, Tokens per sec: 177226.72357999886, Loss: 2.473301410675049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16787 , TFLOPS: 69.6939650882837, Tokens per sec: 172709.32291281, Loss: 2.4796507358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16788 , TFLOPS: 69.0917875599821, Tokens per sec: 171217.06066234774, Loss: 2.49782657623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16789 , TFLOPS: 71.49134546075794, Tokens per sec: 177163.42947359435, Loss: 2.4978692531585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16790 , TFLOPS: 70.60895592074844, Tokens per sec: 174976.77099021862, Loss: 2.4989118576049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16791 , TFLOPS: 68.26403919759882, Tokens per sec: 169165.8090363524, Loss: 2.4811482429504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16792 , TFLOPS: 70.38706910095327, Tokens per sec: 174426.9110079152, Loss: 2.4837896823883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16793 , TFLOPS: 71.55801112534132, Tokens per sec: 177328.63433426115, Loss: 2.4909000396728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16794 , TFLOPS: 68.74923914777226, Tokens per sec: 170368.18796206906, Loss: 2.4835474491119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16795 , TFLOPS: 69.65291579133789, Tokens per sec: 172607.5982330249, Loss: 2.4701361656188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16796 , TFLOPS: 71.48106134018549, Tokens per sec: 177137.9442899264, Loss: 2.505547523498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16797 , TFLOPS: 69.81577212173404, Tokens per sec: 173011.17415985235, Loss: 2.4778852462768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16798 , TFLOPS: 69.59716144001729, Tokens per sec: 172469.43280860365, Loss: 2.4863200187683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16799 , TFLOPS: 70.45810847215013, Tokens per sec: 174602.95439537376, Loss: 2.5051608085632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16800 , TFLOPS: 70.65102262841808, Tokens per sec: 175081.01692585438, Loss: 2.5031609535217285 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/model.pt +[rank0]:[2024-08-27 13:30:02,497] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006372453004587442, 'preprocessing_with_comm': 0.0013618499942822382, 'state_converting': 0.9676613729970995, : 0.9767607829999179}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0014800 + eval ppl=10.413918495178223, eval loss=2.3431432247161865 +------------------------------------------------------------------ +iteration: 16801 , TFLOPS: 64.19277167726088, Tokens per sec: 159076.759633814, Loss: 2.4792838096618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16802 , TFLOPS: 71.55660170432631, Tokens per sec: 177325.1416337254, Loss: 2.4918150901794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16803 , TFLOPS: 69.20754025487811, Tokens per sec: 171503.90859151204, Loss: 2.495712995529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16804 , TFLOPS: 68.9898185763456, Tokens per sec: 170964.37028808566, Loss: 2.513151168823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16805 , TFLOPS: 71.52540484738972, Tokens per sec: 177247.83238561908, Loss: 2.484077215194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16806 , TFLOPS: 71.53310782839708, Tokens per sec: 177266.9212211087, Loss: 2.4721643924713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16807 , TFLOPS: 69.6852110967588, Tokens per sec: 172687.62955747952, Loss: 2.486647129058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16808 , TFLOPS: 68.93696713601503, Tokens per sec: 170833.3985968797, Loss: 2.499267578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16809 , TFLOPS: 71.52684190379203, Tokens per sec: 177251.3935696888, Loss: 2.491090774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16810 , TFLOPS: 69.42026630650386, Tokens per sec: 172031.06718114973, Loss: 2.49542236328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16811 , TFLOPS: 68.1156921350731, Tokens per sec: 168798.18867363568, Loss: 2.4870107173919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16812 , TFLOPS: 71.52097105963959, Tokens per sec: 177236.84497115223, Loss: 2.4833807945251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16813 , TFLOPS: 70.66397844388868, Tokens per sec: 175113.12286379142, Loss: 2.4984023571014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16814 , TFLOPS: 68.6295172188279, Tokens per sec: 170071.50383368574, Loss: 2.513643503189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16815 , TFLOPS: 70.83162420736242, Tokens per sec: 175528.56753338475, Loss: 2.470864772796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16816 , TFLOPS: 71.55058462209755, Tokens per sec: 177310.23064112623, Loss: 2.495558261871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16817 , TFLOPS: 70.03254220056978, Tokens per sec: 173548.3542944028, Loss: 2.495373010635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16818 , TFLOPS: 68.12755285415265, Tokens per sec: 168827.58084208088, Loss: 2.486585855484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16819 , TFLOPS: 71.58927533342276, Tokens per sec: 177406.1104859238, Loss: 2.4721755981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16820 , TFLOPS: 70.61986267410713, Tokens per sec: 175003.7991265195, Loss: 2.4760117530822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16821 , TFLOPS: 67.23601903110443, Tokens per sec: 166618.26181800928, Loss: 2.499091386795044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16822 , TFLOPS: 71.49904860955048, Tokens per sec: 177182.51872487416, Loss: 2.4758071899414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16823 , TFLOPS: 70.6052320106397, Tokens per sec: 174967.5427307466, Loss: 2.5026473999023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16824 , TFLOPS: 68.94366163095829, Tokens per sec: 170849.98829280862, Loss: 2.5043258666992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16825 , TFLOPS: 69.80580777220962, Tokens per sec: 172986.48140406713, Loss: 2.4822940826416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16826 , TFLOPS: 71.51578435850183, Tokens per sec: 177223.9917543705, Loss: 2.4865493774414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16827 , TFLOPS: 70.55996871565394, Tokens per sec: 174855.3752996083, Loss: 2.4950695037841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16828 , TFLOPS: 68.24910602136654, Tokens per sec: 169128.80298062367, Loss: 2.495478868484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16829 , TFLOPS: 70.50653847527774, Tokens per sec: 174722.96927812923, Loss: 2.4899051189422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16830 , TFLOPS: 70.53081950314925, Tokens per sec: 174783.14033997088, Loss: 2.501495122909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16831 , TFLOPS: 68.79957279591646, Tokens per sec: 170492.92028687947, Loss: 2.490734100341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16832 , TFLOPS: 70.46162027667286, Tokens per sec: 174611.65703383827, Loss: 2.4859890937805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16833 , TFLOPS: 71.50572513200646, Tokens per sec: 177199.06388299973, Loss: 2.4772326946258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16834 , TFLOPS: 68.23263766794179, Tokens per sec: 169087.99258669824, Loss: 2.505568742752075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16835 , TFLOPS: 70.62817511509989, Tokens per sec: 175024.39827099055, Loss: 2.4971001148223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16836 , TFLOPS: 70.58260099663046, Tokens per sec: 174911.46058501973, Loss: 2.478431224822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16837 , TFLOPS: 70.5545814530783, Tokens per sec: 174842.02506948993, Loss: 2.487070083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16838 , TFLOPS: 68.32691265236092, Tokens per sec: 169321.6163833357, Loss: 2.4857192039489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16839 , TFLOPS: 71.51760027047658, Tokens per sec: 177228.49178428316, Loss: 2.4816203117370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16840 , TFLOPS: 69.48940087277433, Tokens per sec: 172202.39025216913, Loss: 2.480489730834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16841 , TFLOPS: 68.07090559261181, Tokens per sec: 168687.20268777167, Loss: 2.4976720809936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16842 , TFLOPS: 71.545946723787, Tokens per sec: 177298.737418206, Loss: 2.5293030738830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16843 , TFLOPS: 70.56121407394575, Tokens per sec: 174858.46143464296, Loss: 2.491734266281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16844 , TFLOPS: 68.78404472353249, Tokens per sec: 170454.4400129536, Loss: 2.476193428039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16845 , TFLOPS: 69.96064685177102, Tokens per sec: 173370.189700153, Loss: 2.4637575149536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16846 , TFLOPS: 71.47416136622633, Tokens per sec: 177120.84539436072, Loss: 2.522618532180786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16847 , TFLOPS: 69.30017697378408, Tokens per sec: 171733.47258573803, Loss: 2.4869565963745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16848 , TFLOPS: 68.66552137222965, Tokens per sec: 170160.72609200017, Loss: 2.500058174133301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16849 , TFLOPS: 71.51009087680352, Tokens per sec: 177209.88267953333, Loss: 2.4784209728240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16850 , TFLOPS: 70.443426707188, Tokens per sec: 174566.57136446764, Loss: 2.5073955059051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16851 , TFLOPS: 67.7376449407527, Tokens per sec: 167861.3460807734, Loss: 2.49062180519104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16852 , TFLOPS: 71.55378707132584, Tokens per sec: 177318.16666309285, Loss: 2.4908580780029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16853 , TFLOPS: 71.55764203551725, Tokens per sec: 177327.7196890185, Loss: 2.493180513381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16854 , TFLOPS: 69.02313876504826, Tokens per sec: 171046.94138621265, Loss: 2.5039870738983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16855 , TFLOPS: 69.63385323120772, Tokens per sec: 172560.35910910802, Loss: 2.494088888168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16856 , TFLOPS: 71.50538089895618, Tokens per sec: 177198.2108355807, Loss: 2.4958343505859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16857 , TFLOPS: 70.62276052279319, Tokens per sec: 175010.98031478794, Loss: 2.476276397705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16858 , TFLOPS: 68.06033320833431, Tokens per sec: 168661.00315488776, Loss: 2.500455617904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16859 , TFLOPS: 70.45176764639821, Tokens per sec: 174587.2411306609, Loss: 2.496436834335327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16860 , TFLOPS: 70.57978565762386, Tokens per sec: 174904.48386482615, Loss: 2.5010650157928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16861 , TFLOPS: 68.2817194402505, Tokens per sec: 169209.6226252828, Loss: 2.486945629119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16862 , TFLOPS: 70.53692540620656, Tokens per sec: 174798.27144036765, Loss: 2.477625846862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16863 , TFLOPS: 70.59599853586792, Tokens per sec: 174944.66116311093, Loss: 2.488776683807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16864 , TFLOPS: 70.5815253840828, Tokens per sec: 174908.7950986381, Loss: 2.4813852310180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16865 , TFLOPS: 69.17982787893008, Tokens per sec: 171435.2342711426, Loss: 2.4818906784057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16866 , TFLOPS: 70.47937489932103, Tokens per sec: 174655.65494459358, Loss: 2.4835896492004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16867 , TFLOPS: 70.70247479771317, Tokens per sec: 175208.52106929195, Loss: 2.4991114139556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16868 , TFLOPS: 68.89984656694533, Tokens per sec: 170741.40973755677, Loss: 2.502610683441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16869 , TFLOPS: 70.56375888714446, Tokens per sec: 174864.7677620837, Loss: 2.5003061294555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16870 , TFLOPS: 70.50751818684438, Tokens per sec: 174725.3971113155, Loss: 2.4709677696228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16871 , TFLOPS: 68.77092695850487, Tokens per sec: 170421.93274617408, Loss: 2.494492530822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16872 , TFLOPS: 71.53140339165803, Tokens per sec: 177262.6974391101, Loss: 2.4946422576904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16873 , TFLOPS: 70.48903196615919, Tokens per sec: 174679.5862200323, Loss: 2.4876723289489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16874 , TFLOPS: 69.77054871423084, Tokens per sec: 172899.10557428983, Loss: 2.495465040206909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16875 , TFLOPS: 69.3321252382229, Tokens per sec: 171812.6439044093, Loss: 2.4903814792633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16876 , TFLOPS: 71.55178270865842, Tokens per sec: 177313.1996316042, Loss: 2.4684500694274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16877 , TFLOPS: 69.61237063386234, Tokens per sec: 172507.1228663827, Loss: 2.4680328369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16878 , TFLOPS: 68.02958851858666, Tokens per sec: 168584.81442688592, Loss: 2.474270820617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16879 , TFLOPS: 71.50530512637985, Tokens per sec: 177198.02306278955, Loss: 2.482758045196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16880 , TFLOPS: 70.60928635057834, Tokens per sec: 174977.58983173696, Loss: 2.4687068462371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16881 , TFLOPS: 68.03068914366919, Tokens per sec: 168587.5418970843, Loss: 2.4990673065185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16882 , TFLOPS: 70.98263967036486, Tokens per sec: 175902.80048643137, Loss: 2.4755895137786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16883 , TFLOPS: 71.48765098850116, Tokens per sec: 177154.27416436234, Loss: 2.4954099655151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16884 , TFLOPS: 69.9402996764796, Tokens per sec: 173319.76715835443, Loss: 2.5088982582092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16885 , TFLOPS: 68.30891302083025, Tokens per sec: 169277.0113135796, Loss: 2.4971134662628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16886 , TFLOPS: 71.54224054462553, Tokens per sec: 177289.55309797457, Loss: 2.477734327316284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16887 , TFLOPS: 70.59286687130621, Tokens per sec: 174936.90055334568, Loss: 2.483624219894409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16888 , TFLOPS: 68.0334585696724, Tokens per sec: 168594.40483979773, Loss: 2.5130972862243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16889 , TFLOPS: 71.53317519340604, Tokens per sec: 177267.08815902178, Loss: 2.501023769378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16890 , TFLOPS: 70.64962179296724, Tokens per sec: 175077.54550129245, Loss: 2.5202291011810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16891 , TFLOPS: 69.6109804430436, Tokens per sec: 172503.6778203917, Loss: 2.4922518730163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16892 , TFLOPS: 70.37323825978153, Tokens per sec: 174392.63665989868, Loss: 2.4790146350860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16893 , TFLOPS: 71.4947272436445, Tokens per sec: 177171.8099041214, Loss: 2.4828622341156006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16894 , TFLOPS: 70.54788414331104, Tokens per sec: 174825.42839811707, Loss: 2.4902946949005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16895 , TFLOPS: 67.514520143007, Tokens per sec: 167308.418252734, Loss: 2.496457815170288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16896 , TFLOPS: 71.52265764589745, Tokens per sec: 177241.024517692, Loss: 2.483229398727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16897 , TFLOPS: 70.60154995651637, Tokens per sec: 174958.41819501726, Loss: 2.495532989501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16898 , TFLOPS: 68.06366754100632, Tokens per sec: 168669.2659985561, Loss: 2.4962878227233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16899 , TFLOPS: 70.48055062846349, Tokens per sec: 174658.56853092587, Loss: 2.4833433628082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16900 , TFLOPS: 71.44901837193545, Tokens per sec: 177058.53828477848, Loss: 2.5197815895080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16901 , TFLOPS: 68.16174772480471, Tokens per sec: 168912.31949843268, Loss: 2.4986205101013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16902 , TFLOPS: 70.72709881603221, Tokens per sec: 175269.54209924611, Loss: 2.4928555488586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16903 , TFLOPS: 70.50383630841377, Tokens per sec: 174716.27301097207, Loss: 2.5042331218719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16904 , TFLOPS: 70.61912608352878, Tokens per sec: 175001.97377392423, Loss: 2.487661123275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16905 , TFLOPS: 67.95974189300838, Tokens per sec: 168411.72679446198, Loss: 2.4757978916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16906 , TFLOPS: 70.29668952832245, Tokens per sec: 174202.94047080417, Loss: 2.482372522354126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16907 , TFLOPS: 70.56562681711829, Tokens per sec: 174869.39669832832, Loss: 2.484142541885376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16908 , TFLOPS: 68.90777460643625, Tokens per sec: 170761.05629276088, Loss: 2.493666887283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16909 , TFLOPS: 71.53760151448894, Tokens per sec: 177278.05707026474, Loss: 2.504537343978882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16910 , TFLOPS: 69.74820606067132, Tokens per sec: 172843.7380175229, Loss: 2.4805524349212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16911 , TFLOPS: 70.5994130012515, Tokens per sec: 174953.12258446586, Loss: 2.4813029766082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16912 , TFLOPS: 69.82634625727395, Tokens per sec: 173037.37803255743, Loss: 2.496694326400757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16913 , TFLOPS: 71.51621977027075, Tokens per sec: 177225.07075269846, Loss: 2.4838614463806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16914 , TFLOPS: 69.61880387236305, Tokens per sec: 172523.06514006713, Loss: 2.498337984085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16915 , TFLOPS: 67.46391387809606, Tokens per sec: 167183.0103535444, Loss: 2.4972054958343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16916 , TFLOPS: 71.46520805600653, Tokens per sec: 177098.6581053475, Loss: 2.4849233627319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16917 , TFLOPS: 70.65965693736342, Tokens per sec: 175102.4136945686, Loss: 2.5132241249084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16918 , TFLOPS: 66.80729486785712, Tokens per sec: 165555.83611361712, Loss: 2.4861528873443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16919 , TFLOPS: 71.51067307315431, Tokens per sec: 177211.32542622482, Loss: 2.4920895099639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16920 , TFLOPS: 71.51843421881783, Tokens per sec: 177230.5584001398, Loss: 2.500999927520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16921 , TFLOPS: 67.80440478914197, Tokens per sec: 168026.7843392864, Loss: 2.491903305053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16922 , TFLOPS: 70.16298502545672, Tokens per sec: 173871.60598393602, Loss: 2.4758265018463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16923 , TFLOPS: 71.55799621162754, Tokens per sec: 177328.59737643556, Loss: 2.5126397609710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16924 , TFLOPS: 70.69790325726608, Tokens per sec: 175197.1922884673, Loss: 2.4814679622650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16925 , TFLOPS: 67.81969562996736, Tokens per sec: 168064.67672727685, Loss: 2.4841387271881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16926 , TFLOPS: 71.4612627207577, Tokens per sec: 177088.88112998637, Loss: 2.4829516410827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16927 , TFLOPS: 70.540047842408, Tokens per sec: 174806.00918123953, Loss: 2.4821062088012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16928 , TFLOPS: 68.89705703824463, Tokens per sec: 170734.49697814445, Loss: 2.5081887245178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16929 , TFLOPS: 70.55486361945513, Tokens per sec: 174842.7243088549, Loss: 2.5008654594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16930 , TFLOPS: 71.54191254900876, Tokens per sec: 177288.74028870446, Loss: 2.4875588417053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16931 , TFLOPS: 70.07912049086727, Tokens per sec: 173663.78042878222, Loss: 2.4903223514556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16932 , TFLOPS: 69.58864664881987, Tokens per sec: 172448.33221803475, Loss: 2.488499641418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16933 , TFLOPS: 70.14359493335468, Tokens per sec: 173823.55519971217, Loss: 2.4974827766418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16934 , TFLOPS: 70.51520031599034, Tokens per sec: 174744.43427359557, Loss: 2.4851152896881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16935 , TFLOPS: 68.20288913367136, Tokens per sec: 169014.27244170455, Loss: 2.503096103668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16936 , TFLOPS: 70.5571015280609, Tokens per sec: 174848.27009290596, Loss: 2.492338180541992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16937 , TFLOPS: 70.68110445467424, Tokens per sec: 175155.56300510388, Loss: 2.491777181625366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16938 , TFLOPS: 68.0718385650709, Tokens per sec: 168689.51469630073, Loss: 2.488903522491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16939 , TFLOPS: 71.49908576506957, Tokens per sec: 177182.61080034342, Loss: 2.479680061340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16940 , TFLOPS: 70.43584687742505, Tokens per sec: 174547.78771132787, Loss: 2.4834954738616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16941 , TFLOPS: 69.79930203363224, Tokens per sec: 172970.35946720612, Loss: 2.448301315307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16942 , TFLOPS: 69.2466145817051, Tokens per sec: 171600.73907778176, Loss: 2.480086326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16943 , TFLOPS: 70.48678922365069, Tokens per sec: 174674.02845703776, Loss: 2.483607053756714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16944 , TFLOPS: 70.64279300669982, Tokens per sec: 175060.62301666886, Loss: 2.4933664798736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16945 , TFLOPS: 67.97848000910568, Tokens per sec: 168458.16191032497, Loss: 2.519991874694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16946 , TFLOPS: 71.54127856909574, Tokens per sec: 177287.16921664137, Loss: 2.4788243770599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16947 , TFLOPS: 68.93132955634843, Tokens per sec: 170819.42805343523, Loss: 2.5055506229400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16948 , TFLOPS: 69.71673348446093, Tokens per sec: 172765.74550668304, Loss: 2.4778153896331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16949 , TFLOPS: 70.99426159836683, Tokens per sec: 175931.60090428134, Loss: 2.488837718963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16950 , TFLOPS: 71.45002870348591, Tokens per sec: 177061.04199765768, Loss: 2.4833428859710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16951 , TFLOPS: 68.54201513162609, Tokens per sec: 169854.66402244897, Loss: 2.476978302001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16952 , TFLOPS: 69.5199896825528, Tokens per sec: 172278.19269243575, Loss: 2.4905850887298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16953 , TFLOPS: 71.49767852926256, Tokens per sec: 177179.12351499384, Loss: 2.4821572303771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16954 , TFLOPS: 70.57275347105235, Tokens per sec: 174887.0573318427, Loss: 2.4920167922973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16955 , TFLOPS: 67.17803909914483, Tokens per sec: 166474.58115959662, Loss: 2.4851906299591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16956 , TFLOPS: 71.54535135522848, Tokens per sec: 177297.26202933275, Loss: 2.491614818572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16957 , TFLOPS: 70.64365349760357, Tokens per sec: 175062.75540791926, Loss: 2.4803731441497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16958 , TFLOPS: 68.2681123520919, Tokens per sec: 169175.9027618813, Loss: 2.4879560470581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16959 , TFLOPS: 70.88203981092651, Tokens per sec: 175653.50295275377, Loss: 2.4959301948547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16960 , TFLOPS: 71.52791236071003, Tokens per sec: 177254.04627985216, Loss: 2.4965708255767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16961 , TFLOPS: 70.62699038542222, Tokens per sec: 175021.4623803405, Loss: 2.494371175765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16962 , TFLOPS: 67.21450868815498, Tokens per sec: 166564.95681862065, Loss: 2.4832797050476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16963 , TFLOPS: 71.55993615705798, Tokens per sec: 177333.4047749148, Loss: 2.4850642681121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16964 , TFLOPS: 70.6477692388686, Tokens per sec: 175072.95466816035, Loss: 2.4900872707366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16965 , TFLOPS: 68.12896556349129, Tokens per sec: 168831.08169144482, Loss: 2.4861843585968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16966 , TFLOPS: 70.5146654435527, Tokens per sec: 174743.10880077403, Loss: 2.4882240295410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16967 , TFLOPS: 71.46965581035458, Tokens per sec: 177109.68013058256, Loss: 2.493077039718628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16968 , TFLOPS: 69.01394069602321, Tokens per sec: 171024.1475579176, Loss: 2.4881908893585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16969 , TFLOPS: 69.66148863724831, Tokens per sec: 172628.8426895679, Loss: 2.50809383392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16970 , TFLOPS: 71.54661033365579, Tokens per sec: 177300.38191656352, Loss: 2.494795083999634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16971 , TFLOPS: 70.64991002760358, Tokens per sec: 175078.25977847297, Loss: 2.491318702697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16972 , TFLOPS: 69.01136185282932, Tokens per sec: 171017.75690040016, Loss: 2.487884521484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16973 , TFLOPS: 70.50802799251437, Tokens per sec: 174726.6604659255, Loss: 2.504343271255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16974 , TFLOPS: 70.52048755802079, Tokens per sec: 174757.5366417848, Loss: 2.474642515182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16975 , TFLOPS: 68.19477980845888, Tokens per sec: 168994.1766405121, Loss: 2.485430955886841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16976 , TFLOPS: 71.5391873082151, Tokens per sec: 177281.98684180778, Loss: 2.5018463134765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16977 , TFLOPS: 69.68124214929036, Tokens per sec: 172677.79406843707, Loss: 2.4636070728302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16978 , TFLOPS: 69.94528228992964, Tokens per sec: 173332.11462336383, Loss: 2.5162343978881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16979 , TFLOPS: 69.90233053107548, Tokens per sec: 173225.67543339636, Loss: 2.4928436279296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16980 , TFLOPS: 70.17047856100571, Tokens per sec: 173890.1757905075, Loss: 2.4920811653137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16981 , TFLOPS: 70.65267237503059, Tokens per sec: 175085.10517969477, Loss: 2.4750325679779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16982 , TFLOPS: 68.16413117874438, Tokens per sec: 168918.2259598543, Loss: 2.4945991039276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16983 , TFLOPS: 71.54890962687443, Tokens per sec: 177306.07981844735, Loss: 2.489316701889038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16984 , TFLOPS: 69.66080295657956, Tokens per sec: 172627.14349733674, Loss: 2.475515365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16985 , TFLOPS: 68.22488703494851, Tokens per sec: 169068.78566433818, Loss: 2.4895365238189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16986 , TFLOPS: 71.53353608608316, Tokens per sec: 177267.9824908313, Loss: 2.504122495651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16987 , TFLOPS: 71.50927636280788, Tokens per sec: 177207.86422412598, Loss: 2.502847909927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16988 , TFLOPS: 68.76583693714873, Tokens per sec: 170409.3191125414, Loss: 2.485656261444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16989 , TFLOPS: 69.63211153488751, Tokens per sec: 172556.04299376285, Loss: 2.492763042449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16990 , TFLOPS: 71.50740194986435, Tokens per sec: 177203.2192223683, Loss: 2.473684072494507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16991 , TFLOPS: 70.54343944273892, Tokens per sec: 174814.41394047515, Loss: 2.5005903244018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16992 , TFLOPS: 67.8169705868661, Tokens per sec: 168057.92377028367, Loss: 2.49100661277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16993 , TFLOPS: 71.53712444261595, Tokens per sec: 177276.87483360965, Loss: 2.485112190246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16994 , TFLOPS: 70.60672904354675, Tokens per sec: 174971.25254320205, Loss: 2.484604835510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16995 , TFLOPS: 67.83379439003221, Tokens per sec: 168099.61500782418, Loss: 2.517625093460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16996 , TFLOPS: 71.52808903071927, Tokens per sec: 177254.4840875969, Loss: 2.4909963607788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16997 , TFLOPS: 70.55304224406223, Tokens per sec: 174838.2107399902, Loss: 2.476555347442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16998 , TFLOPS: 70.76406814579498, Tokens per sec: 175361.15617090612, Loss: 2.4805498123168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16999 , TFLOPS: 68.23603400364195, Tokens per sec: 169096.40908070048, Loss: 2.5013954639434814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17000 , TFLOPS: 71.52655411300384, Tokens per sec: 177250.6803924129, Loss: 2.4908177852630615 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/optimizer.pt +[rank0]:[2024-08-27 13:55:22,005] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006359279010212049, 'preprocessing_with_comm': 0.001375824009301141, 'state_converting': 0.983275987993693, : 0.9924055119918194}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000, took 4.55s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015000 + eval ppl=10.154667854309082, eval loss=2.3179335594177246 +------------------------------------------------------------------ +iteration: 17001 , TFLOPS: 65.31790831443406, Tokens per sec: 161864.9721647616, Loss: 2.496042013168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17002 , TFLOPS: 68.74824257526281, Tokens per sec: 170365.71834560912, Loss: 2.4901952743530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17003 , TFLOPS: 70.11596603071693, Tokens per sec: 173755.08773540097, Loss: 2.4887046813964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17004 , TFLOPS: 69.70464682333167, Tokens per sec: 172735.7934289491, Loss: 2.488494396209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17005 , TFLOPS: 69.0664763522815, Tokens per sec: 171154.33670140596, Loss: 2.4720561504364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17006 , TFLOPS: 69.83747043816652, Tokens per sec: 173064.94497823896, Loss: 2.495058298110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17007 , TFLOPS: 71.52579505656209, Tokens per sec: 177248.79936693326, Loss: 2.501349449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17008 , TFLOPS: 69.75232233350944, Tokens per sec: 172853.9385950606, Loss: 2.474254846572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17009 , TFLOPS: 69.06733776626604, Tokens per sec: 171156.47138015222, Loss: 2.494184970855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17010 , TFLOPS: 70.47936465376922, Tokens per sec: 174655.6295549876, Loss: 2.490143299102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17011 , TFLOPS: 69.65040355699571, Tokens per sec: 172601.37263957952, Loss: 2.5020744800567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17012 , TFLOPS: 68.49587009047916, Tokens per sec: 169740.311527191, Loss: 2.493898868560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17013 , TFLOPS: 71.45428340088947, Tokens per sec: 177071.58560651733, Loss: 2.495128870010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17014 , TFLOPS: 68.22000737113633, Tokens per sec: 169056.69331987228, Loss: 2.5024831295013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17015 , TFLOPS: 68.68803382535579, Tokens per sec: 170216.5144307982, Loss: 2.5053906440734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17016 , TFLOPS: 71.5020612042679, Tokens per sec: 177189.98426644862, Loss: 2.4964964389801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17017 , TFLOPS: 70.60836682541523, Tokens per sec: 174975.3111470887, Loss: 2.5062623023986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17018 , TFLOPS: 68.84425639613441, Tokens per sec: 170603.65117052448, Loss: 2.4977951049804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17019 , TFLOPS: 69.27236838642143, Tokens per sec: 171664.5598428846, Loss: 2.49226450920105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17020 , TFLOPS: 71.52965908512907, Tokens per sec: 177258.37485538013, Loss: 2.48700213432312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17021 , TFLOPS: 68.80703712773925, Tokens per sec: 170511.41772340034, Loss: 2.492898464202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17022 , TFLOPS: 68.06350195286798, Tokens per sec: 168668.85565290824, Loss: 2.4879531860351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17023 , TFLOPS: 71.43942151382973, Tokens per sec: 177034.7562132113, Loss: 2.484851360321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17024 , TFLOPS: 70.52611245057183, Tokens per sec: 174771.47574515938, Loss: 2.486588478088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17025 , TFLOPS: 67.98049585238059, Tokens per sec: 168463.15739202366, Loss: 2.4981698989868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17026 , TFLOPS: 70.57339820956602, Tokens per sec: 174888.65506490335, Loss: 2.49998140335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17027 , TFLOPS: 71.53459492274513, Tokens per sec: 177270.60640472014, Loss: 2.529358148574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17028 , TFLOPS: 69.7693229167004, Tokens per sec: 172896.06791297666, Loss: 2.478670597076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17029 , TFLOPS: 68.81760584741126, Tokens per sec: 170537.6081749882, Loss: 2.4700307846069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17030 , TFLOPS: 71.50087692009964, Tokens per sec: 177187.04947982, Loss: 2.492335081100464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17031 , TFLOPS: 69.81465257566025, Tokens per sec: 173008.3998013533, Loss: 2.47312593460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17032 , TFLOPS: 68.90908558520522, Tokens per sec: 170764.30504256664, Loss: 2.494466781616211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17033 , TFLOPS: 70.49809420030022, Tokens per sec: 174702.04343452124, Loss: 2.5081863403320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17034 , TFLOPS: 69.76546261641347, Tokens per sec: 172886.5016636185, Loss: 2.481865167617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17035 , TFLOPS: 69.68903255230775, Tokens per sec: 172697.09954529788, Loss: 2.481691837310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17036 , TFLOPS: 69.68289556181632, Tokens per sec: 172681.89140681483, Loss: 2.5059964656829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17037 , TFLOPS: 71.54731223461063, Tokens per sec: 177302.12130445038, Loss: 2.498469352722168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17038 , TFLOPS: 69.31346543392792, Tokens per sec: 171766.4028538192, Loss: 2.4756362438201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17039 , TFLOPS: 68.64486317279169, Tokens per sec: 170109.5327981044, Loss: 2.4710512161254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17040 , TFLOPS: 70.43929908965985, Tokens per sec: 174556.3426735383, Loss: 2.4903416633605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17041 , TFLOPS: 69.79748560574367, Tokens per sec: 172965.85815880224, Loss: 2.49229097366333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17042 , TFLOPS: 68.03261199885559, Tokens per sec: 168592.30694405618, Loss: 2.4982752799987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17043 , TFLOPS: 71.5607664213081, Tokens per sec: 177335.46226118773, Loss: 2.4617414474487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17044 , TFLOPS: 69.62116552168601, Tokens per sec: 172528.91756724662, Loss: 2.496044397354126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17045 , TFLOPS: 68.13653143675862, Tokens per sec: 168849.830758851, Loss: 2.488288402557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17046 , TFLOPS: 70.54199077132624, Tokens per sec: 174810.82397313023, Loss: 2.5283446311950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17047 , TFLOPS: 71.5376769881193, Tokens per sec: 177278.24410223486, Loss: 2.4834301471710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17048 , TFLOPS: 68.79201618474167, Tokens per sec: 170474.1941719589, Loss: 2.490626573562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17049 , TFLOPS: 68.43168667739985, Tokens per sec: 169581.25795919387, Loss: 2.4942095279693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17050 , TFLOPS: 71.5423967322762, Tokens per sec: 177289.94014817837, Loss: 2.4891655445098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17051 , TFLOPS: 69.48739149618069, Tokens per sec: 172197.410795619, Loss: 2.477726459503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17052 , TFLOPS: 68.80763609249016, Tokens per sec: 170512.90202403453, Loss: 2.4952409267425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17053 , TFLOPS: 71.44600978245604, Tokens per sec: 177051.08266862502, Loss: 2.482617139816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17054 , TFLOPS: 70.66089840461447, Tokens per sec: 175105.4901871756, Loss: 2.475342273712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17055 , TFLOPS: 68.16613364892793, Tokens per sec: 168923.18830155942, Loss: 2.510594367980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17056 , TFLOPS: 70.56679933105781, Tokens per sec: 174872.30231703352, Loss: 2.4796178340911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17057 , TFLOPS: 71.49823182709052, Tokens per sec: 177180.49464796236, Loss: 2.488769292831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17058 , TFLOPS: 68.85546052941562, Tokens per sec: 170631.41624703247, Loss: 2.5071585178375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17059 , TFLOPS: 68.34860134374107, Tokens per sec: 169375.36334978233, Loss: 2.4903151988983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17060 , TFLOPS: 71.53280852437304, Tokens per sec: 177266.17951276607, Loss: 2.4834229946136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17061 , TFLOPS: 69.76434258752926, Tokens per sec: 172883.72610866203, Loss: 2.4787702560424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17062 , TFLOPS: 67.94760125351483, Tokens per sec: 168381.6409523951, Loss: 2.476283073425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17063 , TFLOPS: 71.49706474008903, Tokens per sec: 177177.6024778074, Loss: 2.4997901916503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17064 , TFLOPS: 70.69525981661793, Tokens per sec: 175190.64155134218, Loss: 2.489950656890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17065 , TFLOPS: 68.89539734196177, Tokens per sec: 170730.3840679249, Loss: 2.493511199951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17066 , TFLOPS: 69.0724467032121, Tokens per sec: 171169.13188870042, Loss: 2.4923713207244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17067 , TFLOPS: 71.5355117637082, Tokens per sec: 177272.87843762498, Loss: 2.4901041984558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17068 , TFLOPS: 70.5983662008129, Tokens per sec: 174950.5284976644, Loss: 2.473012685775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17069 , TFLOPS: 68.1401537429228, Tokens per sec: 168858.80723255582, Loss: 2.4899628162384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17070 , TFLOPS: 70.36003353944663, Tokens per sec: 174359.91390828876, Loss: 2.4811906814575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17071 , TFLOPS: 69.74504959068325, Tokens per sec: 172835.91593717856, Loss: 2.4882349967956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17072 , TFLOPS: 69.67384208116145, Tokens per sec: 172659.45588442884, Loss: 2.492584466934204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17073 , TFLOPS: 70.02834320635729, Tokens per sec: 173537.94872418686, Loss: 2.491091728210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17074 , TFLOPS: 71.48501529027857, Tokens per sec: 177147.74261382062, Loss: 2.485039710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17075 , TFLOPS: 69.19444504410139, Tokens per sec: 171471.45721665077, Loss: 2.482541084289551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17076 , TFLOPS: 69.9001432577636, Tokens per sec: 173220.25512918152, Loss: 2.500966787338257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17077 , TFLOPS: 70.42731673474343, Tokens per sec: 174526.6490780901, Loss: 2.486415147781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17078 , TFLOPS: 69.75297134794182, Tokens per sec: 172855.54692431889, Loss: 2.473949909210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17079 , TFLOPS: 68.08390437974529, Tokens per sec: 168719.41511422305, Loss: 2.475053071975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17080 , TFLOPS: 71.48373958066674, Tokens per sec: 177144.58126487146, Loss: 2.4936623573303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17081 , TFLOPS: 68.82120980800195, Tokens per sec: 170546.53918634096, Loss: 2.4975554943084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17082 , TFLOPS: 68.08684719474178, Tokens per sec: 168726.7077339622, Loss: 2.4929494857788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17083 , TFLOPS: 71.54387206052138, Tokens per sec: 177293.59617409078, Loss: 2.4929234981536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17084 , TFLOPS: 70.51097181272209, Tokens per sec: 174733.95557668878, Loss: 2.493253231048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17085 , TFLOPS: 69.31983501021918, Tokens per sec: 171782.1873655357, Loss: 2.4905130863189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17086 , TFLOPS: 69.02091249590121, Tokens per sec: 171041.42444602246, Loss: 2.4857683181762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17087 , TFLOPS: 71.51717910106031, Tokens per sec: 177227.448080074, Loss: 2.4827165603637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17088 , TFLOPS: 69.1273394733124, Tokens per sec: 171305.1622199468, Loss: 2.4936270713806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17089 , TFLOPS: 68.13303818632731, Tokens per sec: 168841.17409946895, Loss: 2.499567985534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17090 , TFLOPS: 71.5412443955108, Tokens per sec: 177287.08453073335, Loss: 2.475661039352417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17091 , TFLOPS: 70.73217519360449, Tokens per sec: 175282.12192207936, Loss: 2.478621006011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17092 , TFLOPS: 68.07566811782483, Tokens per sec: 168699.004749885, Loss: 2.484930992126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17093 , TFLOPS: 70.53710473284254, Tokens per sec: 174798.71583152603, Loss: 2.468759059906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17094 , TFLOPS: 71.53169638247799, Tokens per sec: 177263.4235026376, Loss: 2.496093511581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17095 , TFLOPS: 67.96674748940428, Tokens per sec: 168429.08743406116, Loss: 2.49167799949646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17096 , TFLOPS: 69.7385869007293, Tokens per sec: 172819.90067954862, Loss: 2.5303735733032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17097 , TFLOPS: 71.50285078425433, Tokens per sec: 177191.94093263443, Loss: 2.4787774085998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17098 , TFLOPS: 69.86447163199955, Tokens per sec: 173131.85691098424, Loss: 2.497488498687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17099 , TFLOPS: 66.41971364184403, Tokens per sec: 164595.36713996, Loss: 2.48828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17100 , TFLOPS: 71.5179242562848, Tokens per sec: 177229.29465680593, Loss: 2.482050895690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17101 , TFLOPS: 69.70877236618811, Tokens per sec: 172746.01697861345, Loss: 2.460862159729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17102 , TFLOPS: 68.79066715597449, Tokens per sec: 170470.85113006603, Loss: 2.5085654258728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17103 , TFLOPS: 69.91284549867893, Tokens per sec: 173251.73268143705, Loss: 2.496128559112549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17104 , TFLOPS: 71.46977007009629, Tokens per sec: 177109.96327880927, Loss: 2.485293388366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17105 , TFLOPS: 69.0788788092588, Tokens per sec: 171185.07135603912, Loss: 2.500272750854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17106 , TFLOPS: 68.8886635796455, Tokens per sec: 170713.69706312002, Loss: 2.4791131019592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17107 , TFLOPS: 70.48124156386062, Tokens per sec: 174660.28074495276, Loss: 2.5059940814971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17108 , TFLOPS: 70.58875677117028, Tokens per sec: 174926.71527244392, Loss: 2.4822371006011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17109 , TFLOPS: 68.27003212814584, Tokens per sec: 169180.66017842377, Loss: 2.4809224605560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17110 , TFLOPS: 70.42205153777905, Tokens per sec: 174513.60134000302, Loss: 2.465078353881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17111 , TFLOPS: 70.66452733571475, Tokens per sec: 175114.48307820188, Loss: 2.4799022674560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17112 , TFLOPS: 68.80518935341703, Tokens per sec: 170506.8387350808, Loss: 2.4732978343963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17113 , TFLOPS: 70.59358107618559, Tokens per sec: 174938.67043171293, Loss: 2.4941928386688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17114 , TFLOPS: 70.54234798261919, Tokens per sec: 174811.70918206425, Loss: 2.492138147354126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17115 , TFLOPS: 69.73962177150887, Tokens per sec: 172822.4652033408, Loss: 2.468759775161743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17116 , TFLOPS: 69.19421543419857, Tokens per sec: 171470.888218018, Loss: 2.4976353645324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17117 , TFLOPS: 71.51915923732113, Tokens per sec: 177232.35507585836, Loss: 2.490753412246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17118 , TFLOPS: 68.83524297558519, Tokens per sec: 170581.31492149326, Loss: 2.484071969985962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17119 , TFLOPS: 68.12294450832022, Tokens per sec: 168816.1608534569, Loss: 2.499584436416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17120 , TFLOPS: 71.53218240881328, Tokens per sec: 177264.62792943668, Loss: 2.5209097862243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17121 , TFLOPS: 70.57327619610143, Tokens per sec: 174888.3527020972, Loss: 2.4916653633117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17122 , TFLOPS: 68.95116229147257, Tokens per sec: 170868.575755829, Loss: 2.515251398086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17123 , TFLOPS: 69.91141678562337, Tokens per sec: 173248.19217309964, Loss: 2.4743785858154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17124 , TFLOPS: 71.4613211151599, Tokens per sec: 177089.0258377477, Loss: 2.488306760787964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17125 , TFLOPS: 68.80837337561496, Tokens per sec: 170514.72909283626, Loss: 2.488306999206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17126 , TFLOPS: 68.26316808733219, Tokens per sec: 169163.65032915102, Loss: 2.484247922897339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17127 , TFLOPS: 71.52980033515321, Tokens per sec: 177258.72488849994, Loss: 2.490025043487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17128 , TFLOPS: 70.59462526857617, Tokens per sec: 174941.25805548404, Loss: 2.474463939666748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17129 , TFLOPS: 67.38985989660878, Tokens per sec: 166999.49642969941, Loss: 2.4918107986450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17130 , TFLOPS: 71.53344988334054, Tokens per sec: 177267.76887093903, Loss: 2.510647773742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17131 , TFLOPS: 70.65379753693408, Tokens per sec: 175087.89345483843, Loss: 2.4824507236480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17132 , TFLOPS: 69.64260794556137, Tokens per sec: 172582.05425568772, Loss: 2.4668123722076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17133 , TFLOPS: 69.87420603214198, Tokens per sec: 173155.97982686988, Loss: 2.4846582412719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17134 , TFLOPS: 71.55915802494765, Tokens per sec: 177331.47647782706, Loss: 2.4742188453674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17135 , TFLOPS: 69.71613118310702, Tokens per sec: 172764.2529375791, Loss: 2.4875006675720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17136 , TFLOPS: 67.90700644475731, Tokens per sec: 168281.04254440663, Loss: 2.498525857925415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17137 , TFLOPS: 71.47618151136673, Tokens per sec: 177125.851536556, Loss: 2.4677815437316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17138 , TFLOPS: 69.81047154942553, Tokens per sec: 172998.03875776564, Loss: 2.4952266216278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17139 , TFLOPS: 68.94073613999662, Tokens per sec: 170842.73860393616, Loss: 2.487661123275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17140 , TFLOPS: 69.84223771295684, Tokens per sec: 173076.758810328, Loss: 2.493316173553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17141 , TFLOPS: 71.50412242448513, Tokens per sec: 177195.09219720907, Loss: 2.501215696334839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17142 , TFLOPS: 69.0415137827557, Tokens per sec: 171092.47670426697, Loss: 2.482576608657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17143 , TFLOPS: 69.21602171914274, Tokens per sec: 171524.92659427022, Loss: 2.4818034172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17144 , TFLOPS: 70.53260282181984, Tokens per sec: 174787.55960008537, Loss: 2.4969749450683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17145 , TFLOPS: 69.82953875198163, Tokens per sec: 173045.2893861254, Loss: 2.4902756214141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17146 , TFLOPS: 67.98120754652187, Tokens per sec: 168464.92104850535, Loss: 2.493274450302124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17147 , TFLOPS: 70.39472038040465, Tokens per sec: 174445.8717212537, Loss: 2.473085880279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17148 , TFLOPS: 70.56756424221444, Tokens per sec: 174874.1978511416, Loss: 2.451885938644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17149 , TFLOPS: 68.10893649492282, Tokens per sec: 168781.44745314808, Loss: 2.4950640201568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17150 , TFLOPS: 71.4965699894821, Tokens per sec: 177176.37643130217, Loss: 2.5074141025543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17151 , TFLOPS: 69.72035193684175, Tokens per sec: 172774.71243028817, Loss: 2.4903171062469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17152 , TFLOPS: 69.76215279996651, Tokens per sec: 172878.29957385655, Loss: 2.491250991821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17153 , TFLOPS: 69.64358613914227, Tokens per sec: 172584.47832713815, Loss: 2.4825921058654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17154 , TFLOPS: 71.47543909805495, Tokens per sec: 177124.01175458584, Loss: 2.4910836219787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17155 , TFLOPS: 68.84053497286476, Tokens per sec: 170594.42907371366, Loss: 2.4772677421569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17156 , TFLOPS: 68.8732991274085, Tokens per sec: 170675.62225794262, Loss: 2.5042078495025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17157 , TFLOPS: 71.4700940277353, Tokens per sec: 177110.76608152562, Loss: 2.468255043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17158 , TFLOPS: 69.46628894561691, Tokens per sec: 172145.1163506841, Loss: 2.5244686603546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17159 , TFLOPS: 68.38728206333454, Tokens per sec: 169471.21843396794, Loss: 2.4882211685180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17160 , TFLOPS: 70.64414882494954, Tokens per sec: 175063.9828836474, Loss: 2.484419345855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17161 , TFLOPS: 71.54348867242794, Tokens per sec: 177292.64609616154, Loss: 2.519969940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17162 , TFLOPS: 68.9433123381133, Tokens per sec: 170849.12270666094, Loss: 2.4822046756744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17163 , TFLOPS: 68.2983904322303, Tokens per sec: 169250.9351798701, Loss: 2.489204168319702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17164 , TFLOPS: 71.49462248147456, Tokens per sec: 177171.55029192378, Loss: 2.504939079284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17165 , TFLOPS: 69.76861266877626, Tokens per sec: 172894.30784038035, Loss: 2.495311737060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17166 , TFLOPS: 67.14914718995031, Tokens per sec: 166402.9838259061, Loss: 2.4918723106384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17167 , TFLOPS: 71.49773247382964, Tokens per sec: 177179.25719557356, Loss: 2.4890594482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17168 , TFLOPS: 70.32853372197161, Tokens per sec: 174281.85389059433, Loss: 2.4922614097595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17169 , TFLOPS: 68.8668081965373, Tokens per sec: 170659.53701620872, Loss: 2.499826669692993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17170 , TFLOPS: 71.00888220684847, Tokens per sec: 175967.8323826918, Loss: 2.4738616943359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17171 , TFLOPS: 71.55940405279671, Tokens per sec: 177332.08616193922, Loss: 2.503406524658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17172 , TFLOPS: 69.97181647651435, Tokens per sec: 173397.8692607027, Loss: 2.487424612045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17173 , TFLOPS: 67.99789027171072, Tokens per sec: 168506.2626792772, Loss: 2.499241352081299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17174 , TFLOPS: 71.56450435331564, Tokens per sec: 177344.72526846966, Loss: 2.484527349472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17175 , TFLOPS: 69.77143875901515, Tokens per sec: 172901.3112033119, Loss: 2.483607769012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17176 , TFLOPS: 68.91403943533378, Tokens per sec: 170776.58122889686, Loss: 2.485787868499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17177 , TFLOPS: 70.49324128138595, Tokens per sec: 174690.01736685855, Loss: 2.509357452392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17178 , TFLOPS: 70.60247890745659, Tokens per sec: 174960.7202377795, Loss: 2.4765827655792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17179 , TFLOPS: 68.23445627777583, Tokens per sec: 169092.49930220342, Loss: 2.4999070167541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17180 , TFLOPS: 70.65553788746416, Tokens per sec: 175092.20623516745, Loss: 2.4887890815734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17181 , TFLOPS: 70.45317825344456, Tokens per sec: 174590.73677031294, Loss: 2.486478090286255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17182 , TFLOPS: 69.86730721765767, Tokens per sec: 173138.88380461058, Loss: 2.476295232772827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17183 , TFLOPS: 68.13958388170043, Tokens per sec: 168857.39505367138, Loss: 2.4878549575805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17184 , TFLOPS: 70.4332877816676, Tokens per sec: 174541.44599013327, Loss: 2.4958534240722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17185 , TFLOPS: 69.76377623425073, Tokens per sec: 172882.32262284885, Loss: 2.493380069732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17186 , TFLOPS: 67.95958371038617, Tokens per sec: 168411.33480049888, Loss: 2.486389636993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17187 , TFLOPS: 71.50579915747859, Tokens per sec: 177199.24732627423, Loss: 2.4823131561279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17188 , TFLOPS: 70.47570278914925, Tokens per sec: 174646.5550510716, Loss: 2.494673490524292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17189 , TFLOPS: 69.26893150090164, Tokens per sec: 171656.0428619624, Loss: 2.5036377906799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17190 , TFLOPS: 69.85317579217293, Tokens per sec: 173103.86457555438, Loss: 2.497288942337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17191 , TFLOPS: 71.45041357785632, Tokens per sec: 177061.995758745, Loss: 2.4907121658325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17192 , TFLOPS: 68.63677185577421, Tokens per sec: 170089.4816232029, Loss: 2.4861631393432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17193 , TFLOPS: 68.94772904593412, Tokens per sec: 170860.06779518147, Loss: 2.482893943786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17194 , TFLOPS: 71.51270808139382, Tokens per sec: 177216.368400817, Loss: 2.4931631088256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17195 , TFLOPS: 69.86125307121735, Tokens per sec: 173123.88096281217, Loss: 2.505148410797119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17196 , TFLOPS: 67.7249114549149, Tokens per sec: 167829.79110015804, Loss: 2.4770524501800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17197 , TFLOPS: 71.49653805039941, Tokens per sec: 177176.29728273684, Loss: 2.4786837100982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17198 , TFLOPS: 70.62862156964654, Tokens per sec: 175025.50463453817, Loss: 2.5084786415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17199 , TFLOPS: 67.99414769735162, Tokens per sec: 168496.98816773674, Loss: 2.491915702819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17200 , TFLOPS: 70.0106620531752, Tokens per sec: 173494.1328788618, Loss: 2.4927501678466797 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/optimizer.pt +[rank0]:[2024-08-27 14:20:45,291] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006397712015314028, 'preprocessing_with_comm': 0.0015415889793075621, 'state_converting': 0.9757956079847645, : 0.9851129970047623}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015200 + eval ppl=9.904108047485352, eval loss=2.292949676513672 +------------------------------------------------------------------ +iteration: 17201 , TFLOPS: 66.28450290236037, Tokens per sec: 164260.30003282696, Loss: 2.476351022720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17202 , TFLOPS: 68.56109928965775, Tokens per sec: 169901.956668345, Loss: 2.4808268547058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17203 , TFLOPS: 68.11241791561932, Tokens per sec: 168790.0747971442, Loss: 2.4569435119628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17204 , TFLOPS: 71.57033776427566, Tokens per sec: 177359.18110343182, Loss: 2.4915153980255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17205 , TFLOPS: 70.61243727194346, Tokens per sec: 174985.3981619824, Loss: 2.503492832183838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17206 , TFLOPS: 67.35988383488402, Tokens per sec: 166925.21244660963, Loss: 2.4762589931488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17207 , TFLOPS: 71.54076026666674, Tokens per sec: 177285.8848061266, Loss: 2.49965500831604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17208 , TFLOPS: 71.53988119483853, Tokens per sec: 177283.7063692985, Loss: 2.4892637729644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17209 , TFLOPS: 69.05952525199876, Tokens per sec: 171137.11110917843, Loss: 2.4870030879974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17210 , TFLOPS: 68.82105006780989, Tokens per sec: 170546.1433325482, Loss: 2.482567310333252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17211 , TFLOPS: 71.52104473004096, Tokens per sec: 177237.02753452226, Loss: 2.4922373294830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17212 , TFLOPS: 69.57565203485923, Tokens per sec: 172416.1301331654, Loss: 2.4909415245056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17213 , TFLOPS: 67.93972669326071, Tokens per sec: 168362.12692463098, Loss: 2.4755735397338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17214 , TFLOPS: 70.47462489712925, Tokens per sec: 174643.88391590642, Loss: 2.4719386100769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17215 , TFLOPS: 70.64034450853131, Tokens per sec: 175054.55536848298, Loss: 2.46767258644104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17216 , TFLOPS: 67.99744132269352, Tokens per sec: 168505.15013415675, Loss: 2.4913787841796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17217 , TFLOPS: 70.56639679148145, Tokens per sec: 174871.3047796199, Loss: 2.4886112213134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17218 , TFLOPS: 71.52184351204482, Tokens per sec: 177239.00700432094, Loss: 2.4791815280914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17219 , TFLOPS: 68.15689233780144, Tokens per sec: 168900.28731457354, Loss: 2.4874587059020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17220 , TFLOPS: 69.78207506402538, Tokens per sec: 172927.66913881295, Loss: 2.501699686050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17221 , TFLOPS: 70.30190269701122, Tokens per sec: 174215.85927709288, Loss: 2.5064377784729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17222 , TFLOPS: 69.8163480933877, Tokens per sec: 173012.60148105863, Loss: 2.4672908782958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17223 , TFLOPS: 68.92888380324439, Tokens per sec: 170813.3672078215, Loss: 2.479060411453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17224 , TFLOPS: 71.5211256085724, Tokens per sec: 177237.2279604327, Loss: 2.4766430854797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17225 , TFLOPS: 70.54553297626086, Tokens per sec: 174819.60194716428, Loss: 2.496035575866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17226 , TFLOPS: 67.63457421222341, Tokens per sec: 167605.92546129975, Loss: 2.4985406398773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17227 , TFLOPS: 70.84119419900132, Tokens per sec: 175552.28302688757, Loss: 2.511317014694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17228 , TFLOPS: 70.45910541434704, Tokens per sec: 174605.42492795995, Loss: 2.514719009399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17229 , TFLOPS: 69.81131062193901, Tokens per sec: 173000.1180718855, Loss: 2.4647040367126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17230 , TFLOPS: 68.9397415307833, Tokens per sec: 170840.2738527403, Loss: 2.4858760833740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17231 , TFLOPS: 71.51243847717515, Tokens per sec: 177215.70029186376, Loss: 2.5098397731781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17232 , TFLOPS: 68.88014492299713, Tokens per sec: 170692.5868935379, Loss: 2.495558023452759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17233 , TFLOPS: 68.03946937729984, Tokens per sec: 168609.30028324365, Loss: 2.5146725177764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17234 , TFLOPS: 71.56542704915705, Tokens per sec: 177347.011810408, Loss: 2.493952751159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17235 , TFLOPS: 71.51820397818265, Tokens per sec: 177229.98783848272, Loss: 2.500143527984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17236 , TFLOPS: 65.99668837560695, Tokens per sec: 163547.0639301445, Loss: 2.486699104309082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17237 , TFLOPS: 71.53157188337735, Tokens per sec: 177263.1149801515, Loss: 2.485480308532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17238 , TFLOPS: 71.53505920236766, Tokens per sec: 177271.7569407694, Loss: 2.519817352294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17239 , TFLOPS: 68.8101265502238, Tokens per sec: 170519.07365264508, Loss: 2.4843153953552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17240 , TFLOPS: 69.03418273627966, Tokens per sec: 171074.30956351792, Loss: 2.4985642433166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17241 , TFLOPS: 71.51428506308412, Tokens per sec: 177220.27633516444, Loss: 2.488365888595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17242 , TFLOPS: 69.78074230154269, Tokens per sec: 172924.36640656495, Loss: 2.494084596633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17243 , TFLOPS: 68.03450468685266, Tokens per sec: 168596.99723340973, Loss: 2.489131450653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17244 , TFLOPS: 71.5286816572351, Tokens per sec: 177255.95268138632, Loss: 2.5005545616149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17245 , TFLOPS: 71.52314147197592, Tokens per sec: 177242.22349201227, Loss: 2.5007593631744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17246 , TFLOPS: 68.09606617389417, Tokens per sec: 168749.55337985622, Loss: 2.481980085372925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17247 , TFLOPS: 69.02798294843586, Tokens per sec: 171058.945806278, Loss: 2.4645049571990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17248 , TFLOPS: 71.528718109347, Tokens per sec: 177256.04301373512, Loss: 2.497709274291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17249 , TFLOPS: 69.77999866607482, Tokens per sec: 172922.52359595732, Loss: 2.5062878131866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17250 , TFLOPS: 68.81613904996213, Tokens per sec: 170533.97328932816, Loss: 2.477510452270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17251 , TFLOPS: 70.5100198383813, Tokens per sec: 174731.59647940367, Loss: 2.507141590118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17252 , TFLOPS: 69.83121860711577, Tokens per sec: 173049.45225219912, Loss: 2.4937198162078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17253 , TFLOPS: 68.20858494661786, Tokens per sec: 169028.38729363156, Loss: 2.4782228469848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17254 , TFLOPS: 70.41400975206776, Tokens per sec: 174493.6729091346, Loss: 2.482940435409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17255 , TFLOPS: 71.46938915460811, Tokens per sec: 177109.01932826827, Loss: 2.4714481830596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17256 , TFLOPS: 68.01476555893178, Tokens per sec: 168548.08149997736, Loss: 2.4843740463256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17257 , TFLOPS: 70.87079190489048, Tokens per sec: 175625.6294025374, Loss: 2.5051841735839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17258 , TFLOPS: 70.57877186186306, Tokens per sec: 174901.97156725207, Loss: 2.483445405960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17259 , TFLOPS: 69.0407158910529, Tokens per sec: 171090.4994407324, Loss: 2.5048131942749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17260 , TFLOPS: 68.9404219141373, Tokens per sec: 170841.95991764194, Loss: 2.486635208129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17261 , TFLOPS: 71.52370287443951, Tokens per sec: 177243.61470916064, Loss: 2.5052974224090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17262 , TFLOPS: 69.67830811802628, Tokens per sec: 172670.52321575442, Loss: 2.5165822505950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17263 , TFLOPS: 68.22720386482956, Tokens per sec: 169074.52702400333, Loss: 2.496647834777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17264 , TFLOPS: 71.49878138694943, Tokens per sec: 177181.85651783174, Loss: 2.505769968032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17265 , TFLOPS: 71.47443552588918, Tokens per sec: 177121.52479220644, Loss: 2.5001213550567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17266 , TFLOPS: 67.36453990976715, Tokens per sec: 166936.75071307906, Loss: 2.495131492614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17267 , TFLOPS: 69.75674740046324, Tokens per sec: 172864.9043984335, Loss: 2.484865188598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17268 , TFLOPS: 71.50638858522683, Tokens per sec: 177200.7079931652, Loss: 2.493300437927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17269 , TFLOPS: 69.56282327187063, Tokens per sec: 172384.33904527474, Loss: 2.4956130981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17270 , TFLOPS: 68.65540528449321, Tokens per sec: 170135.65731221024, Loss: 2.494738817214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17271 , TFLOPS: 71.51945439809786, Tokens per sec: 177233.08651677772, Loss: 2.5006604194641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17272 , TFLOPS: 70.72099323647062, Tokens per sec: 175254.41180050737, Loss: 2.4780516624450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17273 , TFLOPS: 67.33942433069585, Tokens per sec: 166874.51154143093, Loss: 2.4766578674316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17274 , TFLOPS: 71.42894598982409, Tokens per sec: 177008.79671075108, Loss: 2.510612964630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17275 , TFLOPS: 71.45322073035193, Tokens per sec: 177068.9521918625, Loss: 2.465261459350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17276 , TFLOPS: 67.65674543051726, Tokens per sec: 167660.86818259323, Loss: 2.4955856800079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17277 , TFLOPS: 69.98551702510196, Tokens per sec: 173431.82072934278, Loss: 2.487766742706299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17278 , TFLOPS: 71.47164064849866, Tokens per sec: 177114.5987781516, Loss: 2.473421096801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17279 , TFLOPS: 69.78467366648269, Tokens per sec: 172934.1087619619, Loss: 2.474414825439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17280 , TFLOPS: 67.346348514901, Tokens per sec: 166891.67043265316, Loss: 2.514570474624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17281 , TFLOPS: 71.48202012358658, Tokens per sec: 177140.320260813, Loss: 2.4724292755126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17282 , TFLOPS: 70.48882794853972, Tokens per sec: 174679.08064189684, Loss: 2.488755702972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17283 , TFLOPS: 67.9656049847054, Tokens per sec: 168426.25618156005, Loss: 2.486297845840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17284 , TFLOPS: 70.54724529518157, Tokens per sec: 174823.84526207732, Loss: 2.480844736099243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17285 , TFLOPS: 71.52527181851715, Tokens per sec: 177247.50272541848, Loss: 2.5082955360412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17286 , TFLOPS: 69.06072046943709, Tokens per sec: 171140.07298965464, Loss: 2.5123977661132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17287 , TFLOPS: 69.18550943513037, Tokens per sec: 171449.313793282, Loss: 2.4791648387908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17288 , TFLOPS: 70.43967499858971, Tokens per sec: 174557.2742172763, Loss: 2.4661436080932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17289 , TFLOPS: 69.7182348032375, Tokens per sec: 172769.4659399953, Loss: 2.492342233657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17290 , TFLOPS: 68.76921913042803, Tokens per sec: 170417.70056006682, Loss: 2.4640097618103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17291 , TFLOPS: 70.5599952907788, Tokens per sec: 174855.44115569524, Loss: 2.4970576763153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17292 , TFLOPS: 70.72863130474919, Tokens per sec: 175273.3397751043, Loss: 2.480342388153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17293 , TFLOPS: 68.22071631804388, Tokens per sec: 169058.45016840613, Loss: 2.486286163330078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17294 , TFLOPS: 70.88316480272003, Tokens per sec: 175656.2908063462, Loss: 2.481898784637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17295 , TFLOPS: 70.4751531021457, Tokens per sec: 174645.19286612375, Loss: 2.4927003383636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17296 , TFLOPS: 69.682555595343, Tokens per sec: 172681.04893244355, Loss: 2.511180877685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17297 , TFLOPS: 69.05733592602549, Tokens per sec: 171131.68571824246, Loss: 2.490814208984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17298 , TFLOPS: 71.51367249591837, Tokens per sec: 177218.75832624786, Loss: 2.504122257232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17299 , TFLOPS: 68.91412158466956, Tokens per sec: 170776.78480400043, Loss: 2.4973673820495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17300 , TFLOPS: 68.09469149910413, Tokens per sec: 168746.14678429355, Loss: 2.4663476943969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17301 , TFLOPS: 71.53701624095879, Tokens per sec: 177276.6066979838, Loss: 2.5052614212036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17302 , TFLOPS: 70.51849683963012, Tokens per sec: 174752.603422317, Loss: 2.4909603595733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17303 , TFLOPS: 67.92590327359403, Tokens per sec: 168327.87096792154, Loss: 2.4726624488830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17304 , TFLOPS: 71.51910814867529, Tokens per sec: 177232.22847256606, Loss: 2.4845762252807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17305 , TFLOPS: 71.49572305341084, Tokens per sec: 177174.27763041988, Loss: 2.4830613136291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17306 , TFLOPS: 68.80599681516267, Tokens per sec: 170508.83971423568, Loss: 2.4946250915527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17307 , TFLOPS: 68.9351086160505, Tokens per sec: 170828.79297967476, Loss: 2.489696979522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17308 , TFLOPS: 71.48329927544728, Tokens per sec: 177143.49014003403, Loss: 2.5092339515686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17309 , TFLOPS: 69.68241582382994, Tokens per sec: 172680.70256323705, Loss: 2.503662586212158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17310 , TFLOPS: 68.65464852918937, Tokens per sec: 170133.78198920382, Loss: 2.463310956954956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17311 , TFLOPS: 71.43431631835337, Tokens per sec: 177022.10497643717, Loss: 2.4980828762054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17312 , TFLOPS: 71.4669443875387, Tokens per sec: 177102.96092615696, Loss: 2.48661732673645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17313 , TFLOPS: 67.54898527256294, Tokens per sec: 167393.82664042106, Loss: 2.4923996925354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17314 , TFLOPS: 69.97494633849584, Tokens per sec: 173405.62540347557, Loss: 2.4792823791503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17315 , TFLOPS: 71.48570564552023, Tokens per sec: 177149.45339015836, Loss: 2.488534450531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17316 , TFLOPS: 69.41432862586713, Tokens per sec: 172016.3529544424, Loss: 2.462844133377075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17317 , TFLOPS: 67.88695802374937, Tokens per sec: 168231.36034863305, Loss: 2.507760763168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17318 , TFLOPS: 71.49080733868465, Tokens per sec: 177162.0959478177, Loss: 2.518038272857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17319 , TFLOPS: 69.8034719297104, Tokens per sec: 172980.69292903977, Loss: 2.4881410598754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17320 , TFLOPS: 68.171210026869, Tokens per sec: 168935.7681253066, Loss: 2.4891202449798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17321 , TFLOPS: 70.50273896638464, Tokens per sec: 174713.5536765417, Loss: 2.5044150352478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17322 , TFLOPS: 71.53683248997754, Tokens per sec: 177276.15134281042, Loss: 2.489036798477173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17323 , TFLOPS: 68.16401587939156, Tokens per sec: 168917.9402353567, Loss: 2.4636268615722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17324 , TFLOPS: 70.84577863037285, Tokens per sec: 175563.64375284303, Loss: 2.487886428833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17325 , TFLOPS: 70.47604481618282, Tokens per sec: 174647.40263173816, Loss: 2.486279249191284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17326 , TFLOPS: 69.77932190759041, Tokens per sec: 172920.8465138819, Loss: 2.497821569442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17327 , TFLOPS: 68.35643151312928, Tokens per sec: 169394.7673720907, Loss: 2.5002148151397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17328 , TFLOPS: 70.46301754883824, Tokens per sec: 174615.1196281863, Loss: 2.4934563636779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17329 , TFLOPS: 70.6193811477867, Tokens per sec: 175002.60585125303, Loss: 2.487287998199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17330 , TFLOPS: 68.29560692807098, Tokens per sec: 169244.03734993504, Loss: 2.497958183288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17331 , TFLOPS: 71.5343420482066, Tokens per sec: 177269.97975375722, Loss: 2.4645533561706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17332 , TFLOPS: 70.521510210303, Tokens per sec: 174760.07088679462, Loss: 2.5097312927246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17333 , TFLOPS: 68.06763559858305, Tokens per sec: 168679.09928234885, Loss: 2.492037057876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17334 , TFLOPS: 69.63696374866608, Tokens per sec: 172568.0673140216, Loss: 2.495866537094116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17335 , TFLOPS: 71.55062933471224, Tokens per sec: 177310.34144391093, Loss: 2.488631248474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17336 , TFLOPS: 68.65223214476786, Tokens per sec: 170127.79392242062, Loss: 2.506401538848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17337 , TFLOPS: 68.93201169456181, Tokens per sec: 170821.1184670717, Loss: 2.479262113571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17338 , TFLOPS: 71.56038262031004, Tokens per sec: 177334.51116003538, Loss: 2.492130994796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17339 , TFLOPS: 70.68557206011742, Tokens per sec: 175166.63422353947, Loss: 2.4605331420898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17340 , TFLOPS: 67.07463639424995, Tokens per sec: 166218.33786611885, Loss: 2.4966349601745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17341 , TFLOPS: 71.4976682847863, Tokens per sec: 177179.09812805319, Loss: 2.4768166542053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17342 , TFLOPS: 71.50735409419377, Tokens per sec: 177203.1006307449, Loss: 2.482997417449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17343 , TFLOPS: 68.71694096348439, Tokens per sec: 170288.149503465, Loss: 2.4868531227111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17344 , TFLOPS: 69.99800126378575, Tokens per sec: 173462.75804805369, Loss: 2.4995169639587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17345 , TFLOPS: 71.54737009281989, Tokens per sec: 177302.26468346655, Loss: 2.479891300201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17346 , TFLOPS: 69.82882872404737, Tokens per sec: 173043.52985868827, Loss: 2.4919040203094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17347 , TFLOPS: 67.90322032514212, Tokens per sec: 168271.66012292466, Loss: 2.5086934566497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17348 , TFLOPS: 71.5518576126165, Tokens per sec: 177313.38525186366, Loss: 2.500912666320801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17349 , TFLOPS: 70.57753880160521, Tokens per sec: 174898.91590810035, Loss: 2.471421003341675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17350 , TFLOPS: 68.04978998022942, Tokens per sec: 168634.87587421137, Loss: 2.4815831184387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17351 , TFLOPS: 71.52288916123999, Tokens per sec: 177241.5982382143, Loss: 2.4976747035980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17352 , TFLOPS: 71.48459462043566, Tokens per sec: 177146.70014760926, Loss: 2.494896650314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17353 , TFLOPS: 68.18374240438334, Tokens per sec: 168966.82473734248, Loss: 2.486508846282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17354 , TFLOPS: 68.58287638728939, Tokens per sec: 169955.92271522156, Loss: 2.502892017364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17355 , TFLOPS: 71.50659343675201, Tokens per sec: 177201.21563781088, Loss: 2.502535820007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17356 , TFLOPS: 69.00742309513187, Tokens per sec: 171007.99622493587, Loss: 2.4798378944396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17357 , TFLOPS: 68.706851716365, Tokens per sec: 170263.14723768082, Loss: 2.4879150390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17358 , TFLOPS: 70.50296260248123, Tokens per sec: 174714.10787142458, Loss: 2.526524066925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17359 , TFLOPS: 70.56765662628104, Tokens per sec: 174874.42678903555, Loss: 2.5012927055358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17360 , TFLOPS: 68.30914751882449, Tokens per sec: 169277.59242544143, Loss: 2.494225263595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17361 , TFLOPS: 70.93016065065227, Tokens per sec: 175772.7516945423, Loss: 2.485515832901001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17362 , TFLOPS: 70.4953126036393, Tokens per sec: 174695.15033157734, Loss: 2.4963035583496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17363 , TFLOPS: 69.83869271703632, Tokens per sec: 173067.97391992342, Loss: 2.480907917022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17364 , TFLOPS: 68.2141176037125, Tokens per sec: 169042.09782738457, Loss: 2.4827675819396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17365 , TFLOPS: 70.58542065385629, Tokens per sec: 174918.4480062362, Loss: 2.4849178791046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17366 , TFLOPS: 69.77499778694519, Tokens per sec: 172910.13086658184, Loss: 2.487328052520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17367 , TFLOPS: 68.18184352896738, Tokens per sec: 168962.11911488333, Loss: 2.4866836071014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17368 , TFLOPS: 71.51854328202063, Tokens per sec: 177230.82867077063, Loss: 2.4688148498535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17369 , TFLOPS: 69.55465805465886, Tokens per sec: 172364.10473755104, Loss: 2.4712321758270264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17370 , TFLOPS: 68.00344947970386, Tokens per sec: 168520.03900908152, Loss: 2.5034422874450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17371 , TFLOPS: 71.51349129917604, Tokens per sec: 177218.3093007599, Loss: 2.49352765083313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17372 , TFLOPS: 71.53887127854692, Tokens per sec: 177281.20368547644, Loss: 2.4773476123809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17373 , TFLOPS: 67.9956198593021, Tokens per sec: 168500.63634722205, Loss: 2.491802453994751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17374 , TFLOPS: 69.16365498315439, Tokens per sec: 171395.15605381888, Loss: 2.4799985885620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17375 , TFLOPS: 71.51859371303748, Tokens per sec: 177230.9536443858, Loss: 2.4941954612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17376 , TFLOPS: 69.63653436817255, Tokens per sec: 172567.00326185845, Loss: 2.4756596088409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17377 , TFLOPS: 68.74582356144386, Tokens per sec: 170359.72376288942, Loss: 2.4752297401428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17378 , TFLOPS: 71.52904761696288, Tokens per sec: 177256.85956990553, Loss: 2.4981188774108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17379 , TFLOPS: 71.53398366020377, Tokens per sec: 177269.09162880652, Loss: 2.499101400375366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17380 , TFLOPS: 67.53836458625221, Tokens per sec: 167367.5074097779, Loss: 2.5065271854400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17381 , TFLOPS: 69.98579381606706, Tokens per sec: 173432.50664784538, Loss: 2.4907052516937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17382 , TFLOPS: 71.5307627268544, Tokens per sec: 177261.1098011517, Loss: 2.4987335205078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17383 , TFLOPS: 69.8127343348674, Tokens per sec: 173003.64618935695, Loss: 2.5019495487213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17384 , TFLOPS: 67.87731344237424, Tokens per sec: 168207.46001354716, Loss: 2.484337329864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17385 , TFLOPS: 71.51120200892828, Tokens per sec: 177212.63618733978, Loss: 2.4921720027923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17386 , TFLOPS: 69.71336768694707, Tokens per sec: 172757.4046896705, Loss: 2.492910146713257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17387 , TFLOPS: 67.93092830494125, Tokens per sec: 168340.32354915288, Loss: 2.5021352767944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17388 , TFLOPS: 71.53463459923681, Tokens per sec: 177270.70472743723, Loss: 2.459473133087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17389 , TFLOPS: 71.48413765528068, Tokens per sec: 177145.56773761637, Loss: 2.4968738555908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17390 , TFLOPS: 68.5104091850224, Tokens per sec: 169776.34100508262, Loss: 2.4843811988830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17391 , TFLOPS: 69.8013991636097, Tokens per sec: 172975.5563863082, Loss: 2.4881794452667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17392 , TFLOPS: 71.51172263615952, Tokens per sec: 177213.92635897064, Loss: 2.5042922496795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17393 , TFLOPS: 68.98087342480584, Tokens per sec: 170942.20321718033, Loss: 2.4675285816192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17394 , TFLOPS: 68.8362893100175, Tokens per sec: 170583.9078534798, Loss: 2.5074422359466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17395 , TFLOPS: 70.48964664525676, Tokens per sec: 174681.10946254854, Loss: 2.5018904209136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17396 , TFLOPS: 70.63942789711824, Tokens per sec: 175052.2839044283, Loss: 2.486616373062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17397 , TFLOPS: 68.13697459512102, Tokens per sec: 168850.92895409095, Loss: 2.5011892318725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17398 , TFLOPS: 71.50265600699483, Tokens per sec: 177191.45825312886, Loss: 2.499988555908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17399 , TFLOPS: 70.53684116571766, Tokens per sec: 174798.06268315628, Loss: 2.4645979404449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17400 , TFLOPS: 68.07968424367719, Tokens per sec: 168708.95715215997, Loss: 2.4946839809417725 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/model.pt +[rank0]:[2024-08-27 14:46:08,171] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006400172016583383, 'preprocessing_with_comm': 0.0013611179892905056, 'state_converting': 0.969978606008226, : 0.9791248610126786}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015400 + eval ppl=10.45468807220459, eval loss=2.347050428390503 +------------------------------------------------------------------ +iteration: 17401 , TFLOPS: 64.88504474277812, Tokens per sec: 160792.28855033935, Loss: 2.484372138977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17402 , TFLOPS: 70.15687187963815, Tokens per sec: 173856.45693517898, Loss: 2.481260299682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17403 , TFLOPS: 69.85142797323616, Tokens per sec: 173099.5332876909, Loss: 2.4806294441223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17404 , TFLOPS: 68.75079143464627, Tokens per sec: 170372.03469994137, Loss: 2.465747833251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17405 , TFLOPS: 71.48831073710438, Tokens per sec: 177155.9090940784, Loss: 2.494600772857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17406 , TFLOPS: 69.68462925219775, Tokens per sec: 172686.18768256178, Loss: 2.4892823696136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17407 , TFLOPS: 68.71784114829622, Tokens per sec: 170290.38026059227, Loss: 2.4874095916748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17408 , TFLOPS: 71.48129886933381, Tokens per sec: 177138.53291332195, Loss: 2.487887144088745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17409 , TFLOPS: 71.48077609840338, Tokens per sec: 177137.23742936837, Loss: 2.495896577835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17410 , TFLOPS: 67.95445636704841, Tokens per sec: 168398.62867888442, Loss: 2.47780179977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17411 , TFLOPS: 69.90088773070539, Tokens per sec: 173222.1000151417, Loss: 2.482146739959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17412 , TFLOPS: 71.47343169238071, Tokens per sec: 177119.0371821911, Loss: 2.4913265705108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17413 , TFLOPS: 69.6687236469722, Tokens per sec: 172646.77184066575, Loss: 2.4978268146514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17414 , TFLOPS: 68.73473706688237, Tokens per sec: 170332.25020808782, Loss: 2.492032051086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17415 , TFLOPS: 71.50090971290814, Tokens per sec: 177187.1307440119, Loss: 2.5001659393310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17416 , TFLOPS: 70.59265508905342, Tokens per sec: 174936.37573359354, Loss: 2.4702539443969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17417 , TFLOPS: 67.33340715001312, Tokens per sec: 166859.60030485195, Loss: 2.474898338317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17418 , TFLOPS: 71.53821488791493, Tokens per sec: 177279.577077183, Loss: 2.496825933456421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17419 , TFLOPS: 71.53652359542416, Tokens per sec: 177275.3858680796, Loss: 2.4999117851257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17420 , TFLOPS: 69.77882702742009, Tokens per sec: 172919.62014630425, Loss: 2.511012554168701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17421 , TFLOPS: 69.4099934033581, Tokens per sec: 172005.60979550084, Loss: 2.4918649196624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17422 , TFLOPS: 71.51273311890135, Tokens per sec: 177216.4304465188, Loss: 2.511453151702881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17423 , TFLOPS: 69.77195926193008, Tokens per sec: 172902.6010668732, Loss: 2.5019617080688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17424 , TFLOPS: 68.18285255791201, Tokens per sec: 168964.61959976188, Loss: 2.481261968612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17425 , TFLOPS: 70.54713227397096, Tokens per sec: 174823.5651830671, Loss: 2.500596523284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17426 , TFLOPS: 71.49470051962138, Tokens per sec: 177171.74367904817, Loss: 2.5089430809020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17427 , TFLOPS: 68.27365109626285, Tokens per sec: 169189.62838007996, Loss: 2.4859237670898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17428 , TFLOPS: 70.49778702991958, Tokens per sec: 174701.28223248041, Loss: 2.486232280731201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17429 , TFLOPS: 71.47758887590835, Tokens per sec: 177129.33914092416, Loss: 2.485604763031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17430 , TFLOPS: 68.92001104792668, Tokens per sec: 170791.37954272947, Loss: 2.4979331493377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17431 , TFLOPS: 69.82015335788235, Tokens per sec: 173022.03134566086, Loss: 2.497011184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17432 , TFLOPS: 70.37151708248007, Tokens per sec: 174388.37139294232, Loss: 2.4719998836517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17433 , TFLOPS: 69.49607198996225, Tokens per sec: 172218.92201544502, Loss: 2.488374710083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17434 , TFLOPS: 68.86364030591025, Tokens per sec: 170651.68663426294, Loss: 2.4901819229125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17435 , TFLOPS: 70.46976130818786, Tokens per sec: 174631.83140674108, Loss: 2.5105013847351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17436 , TFLOPS: 70.65798303522783, Tokens per sec: 175098.26558068107, Loss: 2.4979848861694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17437 , TFLOPS: 67.95421895146306, Tokens per sec: 168398.04033691043, Loss: 2.496812105178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17438 , TFLOPS: 70.93888168025754, Tokens per sec: 175794.36336660487, Loss: 2.4912188053131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17439 , TFLOPS: 70.5479033111019, Tokens per sec: 174825.47589801435, Loss: 2.5018484592437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17440 , TFLOPS: 69.85414153186781, Tokens per sec: 173106.25778490328, Loss: 2.4851174354553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17441 , TFLOPS: 69.68349900405998, Tokens per sec: 172683.38680316924, Loss: 2.4963455200195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17442 , TFLOPS: 71.56652365893088, Tokens per sec: 177349.72933022908, Loss: 2.4811859130859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17443 , TFLOPS: 69.67816223792055, Tokens per sec: 172670.1617087824, Loss: 2.491525173187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17444 , TFLOPS: 67.3412793134895, Tokens per sec: 166879.1083931368, Loss: 2.517712354660034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17445 , TFLOPS: 71.50433544276413, Tokens per sec: 177195.62007997034, Loss: 2.4733669757843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17446 , TFLOPS: 71.53199190777308, Tokens per sec: 177264.15584687362, Loss: 2.490825653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17447 , TFLOPS: 66.53881807351394, Tokens per sec: 164890.5210420743, Loss: 2.491415500640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17448 , TFLOPS: 71.50291402675062, Tokens per sec: 177192.09765450694, Loss: 2.493910074234009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17449 , TFLOPS: 71.5025990902219, Tokens per sec: 177191.31720709568, Loss: 2.4855265617370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17450 , TFLOPS: 69.72666087553945, Tokens per sec: 172790.34667536803, Loss: 2.496986150741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17451 , TFLOPS: 68.96281314208336, Tokens per sec: 170897.44784708964, Loss: 2.4744038581848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17452 , TFLOPS: 71.50432285935283, Tokens per sec: 177195.58889689093, Loss: 2.4783363342285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17453 , TFLOPS: 69.66350687594884, Tokens per sec: 172633.84410739513, Loss: 2.491013526916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17454 , TFLOPS: 68.76896760928611, Tokens per sec: 170417.07726296983, Loss: 2.489846706390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17455 , TFLOPS: 71.4171821488566, Tokens per sec: 176979.64461693956, Loss: 2.521328926086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17456 , TFLOPS: 70.66057144292573, Tokens per sec: 175104.6799400931, Loss: 2.4915013313293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17457 , TFLOPS: 68.0748831067216, Tokens per sec: 168697.0594058951, Loss: 2.492845058441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17458 , TFLOPS: 70.3573747821923, Tokens per sec: 174353.3252149262, Loss: 2.499415636062622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17459 , TFLOPS: 71.51451555864514, Tokens per sec: 177220.8475285558, Loss: 2.5115551948547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17460 , TFLOPS: 70.63883719999974, Tokens per sec: 175050.820091898, Loss: 2.487283229827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17461 , TFLOPS: 68.46671074421972, Tokens per sec: 169668.05145499296, Loss: 2.5023088455200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17462 , TFLOPS: 70.00443309544484, Tokens per sec: 173478.69683542964, Loss: 2.496417760848999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17463 , TFLOPS: 70.67953556479412, Tokens per sec: 175151.67512315346, Loss: 2.523942470550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17464 , TFLOPS: 67.26760998393071, Tokens per sec: 166696.54767914352, Loss: 2.483830690383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17465 , TFLOPS: 70.22067816201057, Tokens per sec: 174014.57593174008, Loss: 2.478543281555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17466 , TFLOPS: 71.55297248010835, Tokens per sec: 177316.14801632127, Loss: 2.4757654666900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17467 , TFLOPS: 68.77384513066633, Tokens per sec: 170429.16429825334, Loss: 2.504939317703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17468 , TFLOPS: 70.90832763419772, Tokens per sec: 175718.64707466264, Loss: 2.5143091678619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17469 , TFLOPS: 70.48654078429163, Tokens per sec: 174673.41279693815, Loss: 2.4899797439575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17470 , TFLOPS: 69.7239088763113, Tokens per sec: 172783.52691812295, Loss: 2.4986772537231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17471 , TFLOPS: 68.86088490315215, Tokens per sec: 170644.8584426957, Loss: 2.5078320503234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17472 , TFLOPS: 69.76961443972883, Tokens per sec: 172896.790339155, Loss: 2.4738495349884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17473 , TFLOPS: 70.57204236826045, Tokens per sec: 174885.2951407899, Loss: 2.503845691680908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17474 , TFLOPS: 68.30505676770399, Tokens per sec: 169267.4550935314, Loss: 2.4884846210479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17475 , TFLOPS: 71.53122287827365, Tokens per sec: 177262.25010705844, Loss: 2.483962297439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17476 , TFLOPS: 70.2140299452004, Tokens per sec: 173998.10091812297, Loss: 2.4785990715026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17477 , TFLOPS: 68.1738130607989, Tokens per sec: 168942.21873013276, Loss: 2.483398675918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17478 , TFLOPS: 70.03457628129935, Tokens per sec: 173553.39497052316, Loss: 2.4841463565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17479 , TFLOPS: 71.5498574290589, Tokens per sec: 177308.4285766695, Loss: 2.501443386077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17480 , TFLOPS: 69.57698756251098, Tokens per sec: 172419.43971780693, Loss: 2.48899245262146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17481 , TFLOPS: 68.1321363706532, Tokens per sec: 168838.93930088528, Loss: 2.489656925201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17482 , TFLOPS: 71.46699720534204, Tokens per sec: 177103.09181449199, Loss: 2.498784065246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17483 , TFLOPS: 70.68516645597822, Tokens per sec: 175165.62909180153, Loss: 2.523470878601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17484 , TFLOPS: 67.2924516604812, Tokens per sec: 166758.10809017223, Loss: 2.483217239379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17485 , TFLOPS: 71.54743863548197, Tokens per sec: 177302.43453973377, Loss: 2.4695682525634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17486 , TFLOPS: 71.51414988282865, Tokens per sec: 177219.94134360002, Loss: 2.502406597137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17487 , TFLOPS: 68.80746395845952, Tokens per sec: 170512.4754569445, Loss: 2.460632801055908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17488 , TFLOPS: 69.87254848996548, Tokens per sec: 173151.87225476315, Loss: 2.4849815368652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17489 , TFLOPS: 71.53668673672134, Tokens per sec: 177275.7901501855, Loss: 2.4978363513946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17490 , TFLOPS: 69.76573315548663, Tokens per sec: 172887.1720892441, Loss: 2.4944489002227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17491 , TFLOPS: 67.93125955309505, Tokens per sec: 168341.14441856797, Loss: 2.49611496925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17492 , TFLOPS: 71.5168307868991, Tokens per sec: 177226.5849192126, Loss: 2.5075528621673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17493 , TFLOPS: 71.52649053886897, Tokens per sec: 177250.52284870343, Loss: 2.4867615699768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17494 , TFLOPS: 67.50918528708891, Tokens per sec: 167295.19789208606, Loss: 2.462448835372925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17495 , TFLOPS: 70.39663753669517, Tokens per sec: 174450.6226457338, Loss: 2.4918065071105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17496 , TFLOPS: 71.48267308009453, Tokens per sec: 177141.93835896882, Loss: 2.477588176727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17497 , TFLOPS: 68.04575620742995, Tokens per sec: 168624.87974085755, Loss: 2.5181198120117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17498 , TFLOPS: 69.74686431949246, Tokens per sec: 172840.41303507666, Loss: 2.468787431716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17499 , TFLOPS: 70.54876341923904, Tokens per sec: 174827.6073407269, Loss: 2.4977049827575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17500 , TFLOPS: 70.5974146329064, Tokens per sec: 174948.17040756825, Loss: 2.4939732551574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17501 , TFLOPS: 68.70093581180932, Tokens per sec: 170248.48697450117, Loss: 2.51078200340271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17502 , TFLOPS: 70.47878134037822, Tokens per sec: 174654.1840401474, Loss: 2.508714437484741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17503 , TFLOPS: 70.65193732511455, Tokens per sec: 175083.2836450304, Loss: 2.496753215789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17504 , TFLOPS: 68.14437337106322, Tokens per sec: 168869.2639359182, Loss: 2.505540609359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17505 , TFLOPS: 70.83607186173342, Tokens per sec: 175539.58931086567, Loss: 2.4927825927734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17506 , TFLOPS: 70.36373935329225, Tokens per sec: 174369.09732322732, Loss: 2.474350690841675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17507 , TFLOPS: 69.7850763699132, Tokens per sec: 172935.1067054241, Loss: 2.480691432952881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17508 , TFLOPS: 69.57963226223389, Tokens per sec: 172425.99357505803, Loss: 2.4947800636291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17509 , TFLOPS: 70.46783661238369, Tokens per sec: 174627.06179851564, Loss: 2.499807119369507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17510 , TFLOPS: 70.63693507390975, Tokens per sec: 175046.10641391051, Loss: 2.5034782886505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17511 , TFLOPS: 68.28772001177313, Tokens per sec: 169224.4927025334, Loss: 2.491990327835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17512 , TFLOPS: 71.38254234567403, Tokens per sec: 176893.80336876382, Loss: 2.495607852935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17513 , TFLOPS: 70.55110428142785, Tokens per sec: 174833.40825509804, Loss: 2.487837314605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17514 , TFLOPS: 67.13398209745561, Tokens per sec: 166365.40305613138, Loss: 2.5056872367858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17515 , TFLOPS: 71.54622710038474, Tokens per sec: 177299.43222230123, Loss: 2.5061235427856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17516 , TFLOPS: 71.47390083304833, Tokens per sec: 177120.19976444554, Loss: 2.4746406078338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17517 , TFLOPS: 67.93662321536027, Tokens per sec: 168354.4361645174, Loss: 2.472233295440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17518 , TFLOPS: 68.57172498919837, Tokens per sec: 169928.28832232408, Loss: 2.515623092651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17519 , TFLOPS: 71.5033893774162, Tokens per sec: 177193.27562582045, Loss: 2.4890007972717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17520 , TFLOPS: 70.57933505946804, Tokens per sec: 174903.36723295855, Loss: 2.5246081352233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17521 , TFLOPS: 68.57686956162408, Tokens per sec: 169941.03713951644, Loss: 2.503817558288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17522 , TFLOPS: 71.5109686662404, Tokens per sec: 177212.0579384544, Loss: 2.4828040599823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17523 , TFLOPS: 70.66168173868806, Tokens per sec: 175107.43137530133, Loss: 2.4960615634918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17524 , TFLOPS: 67.18125612777611, Tokens per sec: 166482.55331092988, Loss: 2.4761619567871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17525 , TFLOPS: 70.64248706250271, Tokens per sec: 175059.86485324588, Loss: 2.481966733932495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17526 , TFLOPS: 71.51243530889195, Tokens per sec: 177215.69244050895, Loss: 2.520951509475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17527 , TFLOPS: 70.60526615235801, Tokens per sec: 174967.6273376856, Loss: 2.4832770824432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17528 , TFLOPS: 68.11766648473062, Tokens per sec: 168803.0813295796, Loss: 2.468933582305908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17529 , TFLOPS: 71.51974012228357, Tokens per sec: 177233.794572785, Loss: 2.495638370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17530 , TFLOPS: 70.64493687791912, Tokens per sec: 175065.9357657172, Loss: 2.4808266162872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17531 , TFLOPS: 68.02832070307065, Tokens per sec: 168581.67264037582, Loss: 2.468445062637329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17532 , TFLOPS: 70.53978158404469, Tokens per sec: 174805.3493636845, Loss: 2.495058298110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17533 , TFLOPS: 71.51289051085352, Tokens per sec: 177216.82048111438, Loss: 2.500840425491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17534 , TFLOPS: 67.96588876620227, Tokens per sec: 168426.9594233704, Loss: 2.4869885444641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17535 , TFLOPS: 70.8184908179478, Tokens per sec: 175496.02154765304, Loss: 2.473170042037964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17536 , TFLOPS: 70.38473594836664, Tokens per sec: 174421.12919878753, Loss: 2.4813525676727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17537 , TFLOPS: 69.38531222015216, Tokens per sec: 171944.44710465975, Loss: 2.512627124786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17538 , TFLOPS: 68.32295612769587, Tokens per sec: 169311.81167936855, Loss: 2.4711391925811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17539 , TFLOPS: 70.47772815058431, Tokens per sec: 174651.57411981968, Loss: 2.492962121963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17540 , TFLOPS: 70.5740005977753, Tokens per sec: 174890.14784924447, Loss: 2.492499589920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17541 , TFLOPS: 68.22191144106317, Tokens per sec: 169061.4118149016, Loss: 2.49263596534729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17542 , TFLOPS: 71.56814754650262, Tokens per sec: 177353.7535025178, Loss: 2.478504180908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17543 , TFLOPS: 70.44149984116204, Tokens per sec: 174561.7963781935, Loss: 2.506863832473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17544 , TFLOPS: 68.36375607650174, Tokens per sec: 169412.91844699488, Loss: 2.4912309646606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17545 , TFLOPS: 70.14429186116524, Tokens per sec: 173825.28226359969, Loss: 2.5025486946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17546 , TFLOPS: 70.47162445244517, Tokens per sec: 174636.44848345302, Loss: 2.5122151374816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17547 , TFLOPS: 70.63271495102458, Tokens per sec: 175035.64848451613, Loss: 2.4872026443481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17548 , TFLOPS: 68.67646324359673, Tokens per sec: 170187.84125459442, Loss: 2.465721607208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17549 , TFLOPS: 71.51459664451289, Tokens per sec: 177221.04846826845, Loss: 2.4672348499298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17550 , TFLOPS: 69.6116795220133, Tokens per sec: 172505.41021508852, Loss: 2.4787697792053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17551 , TFLOPS: 68.12386926575118, Tokens per sec: 168818.4525042413, Loss: 2.49788498878479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17552 , TFLOPS: 71.4992088862357, Tokens per sec: 177182.91590815605, Loss: 2.488565683364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17553 , TFLOPS: 71.46196293856973, Tokens per sec: 177090.6163468599, Loss: 2.4712400436401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17554 , TFLOPS: 68.62587852020019, Tokens per sec: 170062.4867376505, Loss: 2.487441062927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17555 , TFLOPS: 69.89209216504534, Tokens per sec: 173200.30363452458, Loss: 2.4640097618103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17556 , TFLOPS: 71.53802841484844, Tokens per sec: 177279.1149763825, Loss: 2.5004382133483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17557 , TFLOPS: 69.7067017785864, Tokens per sec: 172740.88583444234, Loss: 2.483229875564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17558 , TFLOPS: 67.93481555611868, Tokens per sec: 168349.9565857866, Loss: 2.500112295150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17559 , TFLOPS: 71.49049048172012, Tokens per sec: 177161.3107413549, Loss: 2.4795713424682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17560 , TFLOPS: 71.41534381061456, Tokens per sec: 176975.08901226614, Loss: 2.5005948543548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17561 , TFLOPS: 67.29143029127984, Tokens per sec: 166755.57702477818, Loss: 2.4835362434387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17562 , TFLOPS: 71.5206218389618, Tokens per sec: 177235.97956384282, Loss: 2.4749386310577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17563 , TFLOPS: 71.53734536113365, Tokens per sec: 177277.4222940327, Loss: 2.4845125675201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17564 , TFLOPS: 68.86030510828415, Tokens per sec: 170643.4216471431, Loss: 2.502835273742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17565 , TFLOPS: 68.33571726924183, Tokens per sec: 169343.43519387522, Loss: 2.481323719024658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17566 , TFLOPS: 71.51769833723256, Tokens per sec: 177228.73480450743, Loss: 2.4970669746398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17567 , TFLOPS: 70.66050882345391, Tokens per sec: 175104.52476214382, Loss: 2.511833667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17568 , TFLOPS: 68.72757702012265, Tokens per sec: 170314.50682347297, Loss: 2.5058696269989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17569 , TFLOPS: 70.44239199727608, Tokens per sec: 174564.0072393232, Loss: 2.5035765171051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17570 , TFLOPS: 70.56544583115979, Tokens per sec: 174868.94819518592, Loss: 2.463552713394165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17571 , TFLOPS: 68.88336893025999, Tokens per sec: 170700.57633868896, Loss: 2.4719908237457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17572 , TFLOPS: 70.99206266987139, Tokens per sec: 175926.1517172376, Loss: 2.4957568645477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17573 , TFLOPS: 70.58655037568757, Tokens per sec: 174921.2475813832, Loss: 2.480231523513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17574 , TFLOPS: 69.86785988294379, Tokens per sec: 173140.2533700707, Loss: 2.5009703636169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17575 , TFLOPS: 69.05851603341755, Tokens per sec: 171134.61015435957, Loss: 2.4991018772125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17576 , TFLOPS: 70.47778575524964, Tokens per sec: 174651.71687052603, Loss: 2.48124361038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17577 , TFLOPS: 70.60861778170606, Tokens per sec: 174975.9330444225, Loss: 2.50488018989563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17578 , TFLOPS: 67.56524286957472, Tokens per sec: 167434.1147567384, Loss: 2.4824647903442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17579 , TFLOPS: 71.49497395850595, Tokens per sec: 177172.42129072594, Loss: 2.4850988388061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17580 , TFLOPS: 70.50899293704273, Tokens per sec: 174729.05170476416, Loss: 2.487657308578491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17581 , TFLOPS: 67.93156679957376, Tokens per sec: 168341.9058091882, Loss: 2.473031520843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17582 , TFLOPS: 71.50141990623271, Tokens per sec: 177188.39505927262, Loss: 2.4858481884002686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17583 , TFLOPS: 70.52991835966515, Tokens per sec: 174780.90720714812, Loss: 2.4630043506622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17584 , TFLOPS: 68.9887239789801, Tokens per sec: 170961.6577552341, Loss: 2.461031913757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17585 , TFLOPS: 68.44277292877894, Tokens per sec: 169608.73091136263, Loss: 2.483750343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17586 , TFLOPS: 71.53228695254971, Tokens per sec: 177264.887000332, Loss: 2.4770607948303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17587 , TFLOPS: 70.36609654263185, Tokens per sec: 174374.9386980767, Loss: 2.500267505645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17588 , TFLOPS: 68.11831333254554, Tokens per sec: 168804.68428972116, Loss: 2.491729497909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17589 , TFLOPS: 71.53144992659126, Tokens per sec: 177262.81275780094, Loss: 2.4945573806762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17590 , TFLOPS: 71.47144560683513, Tokens per sec: 177114.11544342374, Loss: 2.5004777908325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17591 , TFLOPS: 66.88241169469511, Tokens per sec: 165741.98388532322, Loss: 2.4832606315612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17592 , TFLOPS: 70.5910926551658, Tokens per sec: 174932.5038503037, Loss: 2.4810445308685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17593 , TFLOPS: 71.53506689601339, Tokens per sec: 177271.77600647102, Loss: 2.471600294113159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17594 , TFLOPS: 69.75708441495608, Tokens per sec: 172865.739557472, Loss: 2.4771385192871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17595 , TFLOPS: 68.65670445232116, Tokens per sec: 170138.87679320303, Loss: 2.508497476577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17596 , TFLOPS: 71.5104858643299, Tokens per sec: 177210.86150213517, Loss: 2.506855010986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17597 , TFLOPS: 70.63518054273068, Tokens per sec: 175041.7584923712, Loss: 2.513885974884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17598 , TFLOPS: 66.39927443031519, Tokens per sec: 164544.71652222652, Loss: 2.47534441947937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17599 , TFLOPS: 71.49020566687936, Tokens per sec: 177160.60493880455, Loss: 2.4804437160491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17600 , TFLOPS: 71.53951716366991, Tokens per sec: 177282.80425996135, Loss: 2.498048782348633 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/model.pt +[rank0]:[2024-08-27 15:11:29,288] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00641143400571309, 'preprocessing_with_comm': 0.0013392670080065727, 'state_converting': 0.9910300920018926, : 1.0001573030021973}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600, took 4.46s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015600 + eval ppl=10.31536865234375, eval loss=2.333634853363037 +------------------------------------------------------------------ +iteration: 17601 , TFLOPS: 63.432124978400594, Tokens per sec: 157191.79332188712, Loss: 2.500889778137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17602 , TFLOPS: 70.02834871774745, Tokens per sec: 173537.96238201877, Loss: 2.510831117630005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17603 , TFLOPS: 71.48674132965941, Tokens per sec: 177152.01992954526, Loss: 2.464282274246216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17604 , TFLOPS: 68.94240646975317, Tokens per sec: 170846.87786507493, Loss: 2.489943027496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17605 , TFLOPS: 69.19018616072744, Tokens per sec: 171460.9032344954, Loss: 2.451415777206421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17606 , TFLOPS: 70.30336807427801, Tokens per sec: 174219.49064338463, Loss: 2.496591806411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17607 , TFLOPS: 69.72642674656878, Tokens per sec: 172789.76647798726, Loss: 2.4948935508728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17608 , TFLOPS: 68.8804346369245, Tokens per sec: 170693.30483656452, Loss: 2.4888179302215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17609 , TFLOPS: 70.36259953688669, Tokens per sec: 174366.2727326132, Loss: 2.502609968185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17610 , TFLOPS: 70.51784965677793, Tokens per sec: 174750.9996319162, Loss: 2.496699810028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17611 , TFLOPS: 68.21329226401652, Tokens per sec: 169040.05254469923, Loss: 2.4848146438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17612 , TFLOPS: 70.65027398947693, Tokens per sec: 175079.16171608883, Loss: 2.4983997344970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17613 , TFLOPS: 70.3876053615706, Tokens per sec: 174428.23992079892, Loss: 2.4769270420074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17614 , TFLOPS: 69.79198894668394, Tokens per sec: 172952.23683214522, Loss: 2.474825143814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17615 , TFLOPS: 69.75998965908617, Tokens per sec: 172872.9390724654, Loss: 2.4624736309051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17616 , TFLOPS: 70.43477169432705, Tokens per sec: 174545.1232891699, Loss: 2.4732449054718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17617 , TFLOPS: 69.7373731938651, Tokens per sec: 172816.89298023828, Loss: 2.499088764190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17618 , TFLOPS: 68.13650256963275, Tokens per sec: 168849.75922293315, Loss: 2.4689695835113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17619 , TFLOPS: 71.46125697182788, Tokens per sec: 177088.86688350496, Loss: 2.501919984817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17620 , TFLOPS: 70.41773149133735, Tokens per sec: 174502.89578902806, Loss: 2.5029690265655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17621 , TFLOPS: 68.51093880130219, Tokens per sec: 169777.65345256598, Loss: 2.4738352298736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17622 , TFLOPS: 70.80150293374048, Tokens per sec: 175453.92369921738, Loss: 2.5061891078948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17623 , TFLOPS: 71.50888657503141, Tokens per sec: 177206.89828707726, Loss: 2.4785561561584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17624 , TFLOPS: 67.97838655512602, Tokens per sec: 168457.93032106856, Loss: 2.4846620559692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17625 , TFLOPS: 68.95237070657873, Tokens per sec: 170871.57034157997, Loss: 2.4811010360717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17626 , TFLOPS: 71.50335432265898, Tokens per sec: 177193.18875627045, Loss: 2.491292715072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17627 , TFLOPS: 70.62354804216763, Tokens per sec: 175012.9318745502, Loss: 2.482682228088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17628 , TFLOPS: 67.84696652605288, Tokens per sec: 168132.257011914, Loss: 2.505751132965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17629 , TFLOPS: 71.50870530612481, Tokens per sec: 177206.44908275825, Loss: 2.5039377212524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17630 , TFLOPS: 70.58922935301267, Tokens per sec: 174927.88638230856, Loss: 2.495335817337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17631 , TFLOPS: 67.82188096376947, Tokens per sec: 168070.0922251729, Loss: 2.4873952865600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17632 , TFLOPS: 70.64706215729711, Tokens per sec: 175071.20244213476, Loss: 2.4787795543670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17633 , TFLOPS: 71.55495959852583, Tokens per sec: 177321.07231465893, Loss: 2.471118450164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17634 , TFLOPS: 69.80138553462795, Tokens per sec: 172975.52261219008, Loss: 2.478381395339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17635 , TFLOPS: 68.66621713985987, Tokens per sec: 170162.450280833, Loss: 2.495896100997925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17636 , TFLOPS: 71.4687013427568, Tokens per sec: 177107.31485473202, Loss: 2.5034878253936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17637 , TFLOPS: 70.64114550283124, Tokens per sec: 175056.540320595, Loss: 2.4764928817749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17638 , TFLOPS: 67.23852700349502, Tokens per sec: 166624.47684986916, Loss: 2.4734253883361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17639 , TFLOPS: 70.38901931488688, Tokens per sec: 174431.74385287645, Loss: 2.4815452098846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17640 , TFLOPS: 71.53291819230611, Tokens per sec: 177266.45128198515, Loss: 2.487886905670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17641 , TFLOPS: 68.90699295941756, Tokens per sec: 170759.11928534307, Loss: 2.4713196754455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17642 , TFLOPS: 70.15296803635883, Tokens per sec: 173846.782781489, Loss: 2.5035390853881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17643 , TFLOPS: 70.54570445057612, Tokens per sec: 174820.026879407, Loss: 2.5048234462738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17644 , TFLOPS: 69.66479674586249, Tokens per sec: 172637.04054713206, Loss: 2.4660840034484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17645 , TFLOPS: 68.02225175393313, Tokens per sec: 168566.63311586314, Loss: 2.4840288162231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17646 , TFLOPS: 70.48685528908491, Tokens per sec: 174674.19217446147, Loss: 2.491978406906128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17647 , TFLOPS: 70.71802428761258, Tokens per sec: 175247.05441818043, Loss: 2.4940686225891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17648 , TFLOPS: 68.85048950768487, Tokens per sec: 170619.0975075804, Loss: 2.4828805923461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17649 , TFLOPS: 71.53364530210091, Tokens per sec: 177268.2531401544, Loss: 2.4719042778015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17650 , TFLOPS: 70.4693641332114, Tokens per sec: 174630.847163397, Loss: 2.488947868347168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17651 , TFLOPS: 68.31009487776782, Tokens per sec: 169279.94008526322, Loss: 2.4670515060424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17652 , TFLOPS: 69.88006952425641, Tokens per sec: 173170.51020624643, Loss: 2.485368013381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17653 , TFLOPS: 70.39803782103421, Tokens per sec: 174454.09270458002, Loss: 2.4915223121643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17654 , TFLOPS: 69.8356665592702, Tokens per sec: 173060.47476762015, Loss: 2.4911608695983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17655 , TFLOPS: 69.4672767211947, Tokens per sec: 172147.56416737832, Loss: 2.4761006832122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17656 , TFLOPS: 71.5146787061308, Tokens per sec: 177221.25182599746, Loss: 2.4867844581604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17657 , TFLOPS: 69.60969860568161, Tokens per sec: 172500.50128620263, Loss: 2.4917643070220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17658 , TFLOPS: 68.09852747748872, Tokens per sec: 168755.65276129846, Loss: 2.468482255935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17659 , TFLOPS: 70.94552443766612, Tokens per sec: 175810.8248512235, Loss: 2.5203158855438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17660 , TFLOPS: 71.55311354190226, Tokens per sec: 177316.49758298582, Loss: 2.483558416366577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17661 , TFLOPS: 67.94138458930082, Tokens per sec: 168366.2353736507, Loss: 2.4951188564300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17662 , TFLOPS: 69.61069548850251, Tokens per sec: 172502.97167164853, Loss: 2.4919612407684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17663 , TFLOPS: 71.51391411549412, Tokens per sec: 177219.3570861731, Loss: 2.4781978130340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17664 , TFLOPS: 69.70346812366957, Tokens per sec: 172732.8724813419, Loss: 2.4875662326812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17665 , TFLOPS: 67.91098191022276, Tokens per sec: 168290.89418576367, Loss: 2.4782848358154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17666 , TFLOPS: 71.48760368052919, Tokens per sec: 177154.1569299964, Loss: 2.4807143211364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17667 , TFLOPS: 70.19150659096604, Tokens per sec: 173942.28556517794, Loss: 2.5029561519622803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17668 , TFLOPS: 67.3462254712199, Tokens per sec: 166891.36551685698, Loss: 2.4937305450439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17669 , TFLOPS: 71.50973616038357, Tokens per sec: 177209.00365316946, Loss: 2.4803643226623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17670 , TFLOPS: 71.45815237970683, Tokens per sec: 177081.17336222998, Loss: 2.4781646728515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17671 , TFLOPS: 68.07677258894472, Tokens per sec: 168701.74175098762, Loss: 2.4640135765075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17672 , TFLOPS: 68.87060049445239, Tokens per sec: 170668.93474819843, Loss: 2.490230083465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17673 , TFLOPS: 71.52798114607799, Tokens per sec: 177254.21673757132, Loss: 2.493666887283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17674 , TFLOPS: 69.76635546448298, Tokens per sec: 172888.71423949004, Loss: 2.5029704570770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17675 , TFLOPS: 69.50323886410231, Tokens per sec: 172236.6823190605, Loss: 2.507662773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17676 , TFLOPS: 70.39773496682528, Tokens per sec: 174453.34219848816, Loss: 2.501818895339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17677 , TFLOPS: 70.70545526413348, Tokens per sec: 175215.906993407, Loss: 2.5238523483276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17678 , TFLOPS: 68.0830461691778, Tokens per sec: 168717.28837389714, Loss: 2.481501579284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17679 , TFLOPS: 70.62681349378207, Tokens per sec: 175021.02402337, Loss: 2.494776487350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17680 , TFLOPS: 70.45577974002077, Tokens per sec: 174597.18354062623, Loss: 2.481447219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17681 , TFLOPS: 69.67947711748064, Tokens per sec: 172673.4201251783, Loss: 2.493151903152466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17682 , TFLOPS: 69.30157225864248, Tokens per sec: 171736.9302553205, Loss: 2.4740846157073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17683 , TFLOPS: 70.29356991425269, Tokens per sec: 174195.20972348546, Loss: 2.476886749267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17684 , TFLOPS: 69.77469024590404, Tokens per sec: 172909.36874600337, Loss: 2.489102363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17685 , TFLOPS: 68.01965921441477, Tokens per sec: 168560.20851734484, Loss: 2.47619891166687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17686 , TFLOPS: 71.54490684883748, Tokens per sec: 177296.16049352946, Loss: 2.4908952713012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17687 , TFLOPS: 70.54008136693804, Tokens per sec: 174806.092258718, Loss: 2.4859225749969482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17688 , TFLOPS: 68.34967571379823, Tokens per sec: 169378.0257571356, Loss: 2.4905149936676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17689 , TFLOPS: 71.46287944248863, Tokens per sec: 177092.88754453222, Loss: 2.486208438873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17690 , TFLOPS: 70.40525564395841, Tokens per sec: 174471.979264895, Loss: 2.496735095977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17691 , TFLOPS: 68.99272333037734, Tokens per sec: 170971.56858856138, Loss: 2.4983596801757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17692 , TFLOPS: 69.05726081215691, Tokens per sec: 171131.49957780167, Loss: 2.4791479110717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17693 , TFLOPS: 71.50204776222532, Tokens per sec: 177189.95095558633, Loss: 2.476529359817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17694 , TFLOPS: 68.79453055003668, Tokens per sec: 170480.42504614015, Loss: 2.4819464683532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17695 , TFLOPS: 69.68803652492832, Tokens per sec: 172694.63127973018, Loss: 2.497035264968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17696 , TFLOPS: 71.48006443318957, Tokens per sec: 177135.47384457212, Loss: 2.5057361125946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17697 , TFLOPS: 70.70050774246714, Tokens per sec: 175203.64648970327, Loss: 2.497729778289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17698 , TFLOPS: 67.97448784917871, Tokens per sec: 168448.26889825324, Loss: 2.4734201431274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17699 , TFLOPS: 70.61872825004025, Tokens per sec: 175000.98789871464, Loss: 2.4957478046417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17700 , TFLOPS: 71.50200889446214, Tokens per sec: 177189.85463698753, Loss: 2.485567569732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17701 , TFLOPS: 70.5814902018099, Tokens per sec: 174908.7079130902, Loss: 2.4743871688842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17702 , TFLOPS: 67.17210913703906, Tokens per sec: 166459.88606025928, Loss: 2.4886953830718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17703 , TFLOPS: 71.52128601307383, Tokens per sec: 177237.62546045723, Loss: 2.5054373741149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17704 , TFLOPS: 70.73431111219979, Tokens per sec: 175287.41496364938, Loss: 2.4834322929382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17705 , TFLOPS: 66.70994450742089, Tokens per sec: 165314.5911964286, Loss: 2.493770122528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17706 , TFLOPS: 71.48206765604077, Tokens per sec: 177140.43805147064, Loss: 2.5038559436798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17707 , TFLOPS: 71.40322365274086, Tokens per sec: 176945.0539259095, Loss: 2.4572978019714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17708 , TFLOPS: 68.9348611841498, Tokens per sec: 170828.17981616795, Loss: 2.4748926162719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17709 , TFLOPS: 69.71362897271231, Tokens per sec: 172758.05218457978, Loss: 2.509781837463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17710 , TFLOPS: 71.49217632552876, Tokens per sec: 177165.4884480237, Loss: 2.4982943534851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17711 , TFLOPS: 68.9047491560904, Tokens per sec: 170753.558893523, Loss: 2.473308801651001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17712 , TFLOPS: 68.84970632162444, Tokens per sec: 170617.1566862476, Loss: 2.4787960052490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17713 , TFLOPS: 70.52266728977827, Tokens per sec: 174762.93825719712, Loss: 2.50797700881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17714 , TFLOPS: 69.73716422618317, Tokens per sec: 172816.37513530275, Loss: 2.4838690757751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17715 , TFLOPS: 69.60881249933573, Tokens per sec: 172498.30541706487, Loss: 2.4871537685394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17716 , TFLOPS: 71.5162727961692, Tokens per sec: 177225.20215671614, Loss: 2.5058248043060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17717 , TFLOPS: 69.60603996146591, Tokens per sec: 172491.4347628032, Loss: 2.4987123012542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17718 , TFLOPS: 68.96089657643576, Tokens per sec: 170892.69838628778, Loss: 2.491751194000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17719 , TFLOPS: 69.85768857601744, Tokens per sec: 173115.04775104526, Loss: 2.476635694503784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17720 , TFLOPS: 70.55798999486107, Tokens per sec: 174850.47181150952, Loss: 2.4844870567321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17721 , TFLOPS: 70.58662021380069, Tokens per sec: 174921.42064792084, Loss: 2.478360652923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17722 , TFLOPS: 68.08377723393143, Tokens per sec: 168719.10003289004, Loss: 2.4683077335357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17723 , TFLOPS: 71.5165515778905, Tokens per sec: 177225.89300853186, Loss: 2.482844352722168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17724 , TFLOPS: 69.67190857504575, Tokens per sec: 172654.66444328084, Loss: 2.479440450668335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17725 , TFLOPS: 68.13668055259555, Tokens per sec: 168850.20028432138, Loss: 2.498739242553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17726 , TFLOPS: 70.90062522324219, Tokens per sec: 175699.55965182418, Loss: 2.4651923179626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17727 , TFLOPS: 70.45636116327428, Tokens per sec: 174598.62437149743, Loss: 2.4758377075195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17728 , TFLOPS: 68.8247459969804, Tokens per sec: 170555.30225217293, Loss: 2.4919986724853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17729 , TFLOPS: 70.5399585004836, Tokens per sec: 174805.78778210905, Loss: 2.490818500518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17730 , TFLOPS: 71.54739915752424, Tokens per sec: 177302.33670900564, Loss: 2.4825491905212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17731 , TFLOPS: 69.50055562811889, Tokens per sec: 172230.03296471073, Loss: 2.4646122455596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17732 , TFLOPS: 68.44886865531662, Tokens per sec: 169623.83679322273, Loss: 2.5053420066833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17733 , TFLOPS: 71.50153067503058, Tokens per sec: 177188.6695565559, Loss: 2.5050032138824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17734 , TFLOPS: 70.55115309353263, Tokens per sec: 174833.52921687078, Loss: 2.476581573486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17735 , TFLOPS: 68.00337584630088, Tokens per sec: 168519.85653739754, Loss: 2.4742703437805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17736 , TFLOPS: 71.39264061735918, Tokens per sec: 176918.8279984161, Loss: 2.484985828399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17737 , TFLOPS: 71.49271377309081, Tokens per sec: 177166.82030228726, Loss: 2.492793560028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17738 , TFLOPS: 68.45262829396242, Tokens per sec: 169633.1535919454, Loss: 2.51355242729187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17739 , TFLOPS: 68.64372604078893, Tokens per sec: 170106.71485973612, Loss: 2.504265546798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17740 , TFLOPS: 71.53333375775566, Tokens per sec: 177267.48109894744, Loss: 2.504624843597412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17741 , TFLOPS: 70.60939908112614, Tokens per sec: 174977.86919045274, Loss: 2.4960391521453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17742 , TFLOPS: 67.16851869066159, Tokens per sec: 166450.9885386156, Loss: 2.474130630493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17743 , TFLOPS: 71.5577566250366, Tokens per sec: 177328.00365447052, Loss: 2.4976613521575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17744 , TFLOPS: 70.69169906767127, Tokens per sec: 175181.81762320347, Loss: 2.5155184268951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17745 , TFLOPS: 68.06118843985683, Tokens per sec: 168663.12251281214, Loss: 2.4715795516967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17746 , TFLOPS: 69.65250161810876, Tokens per sec: 172606.57186613817, Loss: 2.4665651321411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17747 , TFLOPS: 71.5435297026087, Tokens per sec: 177292.7477734693, Loss: 2.4838428497314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17748 , TFLOPS: 69.78895836204434, Tokens per sec: 172944.7267095618, Loss: 2.479254961013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17749 , TFLOPS: 69.58281464670148, Tokens per sec: 172433.87987433732, Loss: 2.498168468475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17750 , TFLOPS: 70.4327511901665, Tokens per sec: 174540.11625728314, Loss: 2.492784023284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17751 , TFLOPS: 69.87794337141831, Tokens per sec: 173165.2413652985, Loss: 2.5024969577789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17752 , TFLOPS: 68.23603000216175, Tokens per sec: 169096.39916459177, Loss: 2.504042863845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17753 , TFLOPS: 71.43222629458067, Tokens per sec: 177016.92566729328, Loss: 2.4825439453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17754 , TFLOPS: 70.50120521023801, Tokens per sec: 174709.75285985315, Loss: 2.500753402709961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17755 , TFLOPS: 68.09187668764186, Tokens per sec: 168739.171371413, Loss: 2.489407539367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17756 , TFLOPS: 71.47561373322102, Tokens per sec: 177124.444519765, Loss: 2.4830400943756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17757 , TFLOPS: 70.41764739479143, Tokens per sec: 174502.6873885233, Loss: 2.468782424926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17758 , TFLOPS: 69.07379396503079, Tokens per sec: 171172.47055190025, Loss: 2.5024220943450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17759 , TFLOPS: 69.20281386590736, Tokens per sec: 171492.1960789878, Loss: 2.506188154220581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17760 , TFLOPS: 71.5243032648567, Tokens per sec: 177245.1025427529, Loss: 2.489757537841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17761 , TFLOPS: 69.61405750303243, Tokens per sec: 172511.30311401014, Loss: 2.479029417037964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17762 , TFLOPS: 67.97146299707124, Tokens per sec: 168440.77298151594, Loss: 2.4787278175354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17763 , TFLOPS: 71.53540567377073, Tokens per sec: 177272.61553507316, Loss: 2.4923651218414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17764 , TFLOPS: 69.60351169907904, Tokens per sec: 172485.16945009743, Loss: 2.4906721115112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17765 , TFLOPS: 68.28298241682378, Tokens per sec: 169212.75242035973, Loss: 2.490255117416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17766 , TFLOPS: 70.62109601726311, Tokens per sec: 175006.855486724, Loss: 2.5083227157592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17767 , TFLOPS: 71.490155778229, Tokens per sec: 177160.48130923326, Loss: 2.47563099861145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17768 , TFLOPS: 69.61978039939649, Tokens per sec: 172525.48508162936, Loss: 2.4919350147247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17769 , TFLOPS: 69.05821490224228, Tokens per sec: 171133.8639181349, Loss: 2.4656996726989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17770 , TFLOPS: 71.52054639161314, Tokens per sec: 177235.7925970016, Loss: 2.4741404056549072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17771 , TFLOPS: 70.57205617801851, Tokens per sec: 174885.32936289167, Loss: 2.511005163192749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17772 , TFLOPS: 66.73161301282555, Tokens per sec: 165368.28813980432, Loss: 2.4837965965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17773 , TFLOPS: 71.53656874479081, Tokens per sec: 177275.4977531838, Loss: 2.4767258167266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17774 , TFLOPS: 71.53558259346651, Tokens per sec: 177273.05396156863, Loss: 2.4914472103118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17775 , TFLOPS: 68.87217278091413, Tokens per sec: 170672.83104725226, Loss: 2.497126817703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17776 , TFLOPS: 69.76400599920001, Tokens per sec: 172882.89200570388, Loss: 2.494959831237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17777 , TFLOPS: 71.52478463970711, Tokens per sec: 177246.29544266005, Loss: 2.4836363792419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17778 , TFLOPS: 69.06079102338906, Tokens per sec: 171140.2478301198, Loss: 2.488088846206665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17779 , TFLOPS: 67.99996633314305, Tokens per sec: 168511.40738820354, Loss: 2.486254930496216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17780 , TFLOPS: 71.53257541743858, Tokens per sec: 177265.60184810363, Loss: 2.491370916366577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17781 , TFLOPS: 70.57013717803261, Tokens per sec: 174880.57386953125, Loss: 2.4774298667907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17782 , TFLOPS: 68.0728973723707, Tokens per sec: 168692.1385374269, Loss: 2.502779960632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17783 , TFLOPS: 70.40846763404855, Tokens per sec: 174479.93893016828, Loss: 2.4958231449127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17784 , TFLOPS: 70.64191897327676, Tokens per sec: 175058.45706556382, Loss: 2.501537799835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17785 , TFLOPS: 69.09113235281598, Tokens per sec: 171215.43698681306, Loss: 2.510518789291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17786 , TFLOPS: 69.92193749742701, Tokens per sec: 173274.26365590113, Loss: 2.4966297149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17787 , TFLOPS: 70.57283883225472, Tokens per sec: 174887.26886630544, Loss: 2.4789981842041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17788 , TFLOPS: 70.54803536160796, Tokens per sec: 174825.80313371494, Loss: 2.5082974433898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17789 , TFLOPS: 68.86424703721036, Tokens per sec: 170653.19018126157, Loss: 2.4977598190307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17790 , TFLOPS: 71.53468295501176, Tokens per sec: 177270.8245583744, Loss: 2.4981701374053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17791 , TFLOPS: 69.48442660180858, Tokens per sec: 172190.0634607548, Loss: 2.4696810245513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17792 , TFLOPS: 68.16600309781693, Tokens per sec: 168922.86478152496, Loss: 2.5187809467315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17793 , TFLOPS: 70.81199471405002, Tokens per sec: 175479.9234865894, Loss: 2.5020196437835693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17794 , TFLOPS: 70.57008448285136, Tokens per sec: 174880.44328506725, Loss: 2.483154058456421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17795 , TFLOPS: 68.52160689414306, Tokens per sec: 169804.09016181313, Loss: 2.485978364944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17796 , TFLOPS: 70.61463106439531, Tokens per sec: 174990.83462134423, Loss: 2.476612091064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17797 , TFLOPS: 71.45281488985627, Tokens per sec: 177067.94647440722, Loss: 2.4888036251068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17798 , TFLOPS: 68.79655380872357, Tokens per sec: 170485.4389040466, Loss: 2.5124435424804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17799 , TFLOPS: 68.34075633038623, Tokens per sec: 169355.92254248945, Loss: 2.4804322719573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17800 , TFLOPS: 71.46439164126728, Tokens per sec: 177096.63493968814, Loss: 2.492520809173584 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/model.pt +[rank0]:[2024-08-27 15:36:51,282] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00637099301093258, 'preprocessing_with_comm': 0.0012645219976548105, 'state_converting': 0.9669691469753161, : 0.9760003290139139}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0015800 + eval ppl=9.982572555541992, eval loss=2.3008408546447754 +------------------------------------------------------------------ +iteration: 17801 , TFLOPS: 63.62100518659239, Tokens per sec: 157659.85928465897, Loss: 2.4714207649230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17802 , TFLOPS: 68.07593991680749, Tokens per sec: 168699.67829770516, Loss: 2.4793522357940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17803 , TFLOPS: 71.53777558383383, Tokens per sec: 177278.48843327668, Loss: 2.4877400398254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17804 , TFLOPS: 71.48905554203566, Tokens per sec: 177157.75480274513, Loss: 2.4952821731567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17805 , TFLOPS: 68.79161132850625, Tokens per sec: 170473.1908936089, Loss: 2.4792990684509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17806 , TFLOPS: 69.77801191093948, Tokens per sec: 172917.60019787285, Loss: 2.5010924339294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17807 , TFLOPS: 71.53560607139008, Tokens per sec: 177273.11214244956, Loss: 2.4803037643432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17808 , TFLOPS: 69.81866726684444, Tokens per sec: 173018.3486483625, Loss: 2.500284433364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17809 , TFLOPS: 67.9371612929698, Tokens per sec: 168355.76958010797, Loss: 2.4897117614746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17810 , TFLOPS: 71.44200527308564, Tokens per sec: 177041.15905327204, Loss: 2.4895472526550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17811 , TFLOPS: 70.60094522427528, Tokens per sec: 174956.91960190705, Loss: 2.511659860610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17812 , TFLOPS: 68.1113122962639, Tokens per sec: 168787.33495058742, Loss: 2.4832136631011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17813 , TFLOPS: 70.17776874119689, Tokens per sec: 173908.2416601009, Loss: 2.5095584392547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17814 , TFLOPS: 71.44342941633778, Tokens per sec: 177044.68823713314, Loss: 2.503889322280884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17815 , TFLOPS: 69.22940443930948, Tokens per sec: 171558.09044907172, Loss: 2.485929489135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17816 , TFLOPS: 68.97093613446263, Tokens per sec: 170917.57751702698, Loss: 2.457681655883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17817 , TFLOPS: 70.39339111629467, Tokens per sec: 174442.5776583586, Loss: 2.5031867027282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17818 , TFLOPS: 69.72313438764846, Tokens per sec: 172781.60764989947, Loss: 2.4944276809692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17819 , TFLOPS: 68.81390130272007, Tokens per sec: 170528.42790515412, Loss: 2.4999139308929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17820 , TFLOPS: 70.27404949527305, Tokens per sec: 174146.83597490223, Loss: 2.517702341079712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17821 , TFLOPS: 69.78279113195872, Tokens per sec: 172929.44363403314, Loss: 2.480903387069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17822 , TFLOPS: 69.62469436496792, Tokens per sec: 172537.66242963338, Loss: 2.469116687774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17823 , TFLOPS: 70.8332689798708, Tokens per sec: 175532.64346084715, Loss: 2.4840188026428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17824 , TFLOPS: 70.46487639785089, Tokens per sec: 174619.7260608088, Loss: 2.475412130355835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17825 , TFLOPS: 68.49729921529236, Tokens per sec: 169743.8530559092, Loss: 2.483722686767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17826 , TFLOPS: 69.53201500530804, Tokens per sec: 172307.9927669219, Loss: 2.491978168487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17827 , TFLOPS: 70.52744765296946, Tokens per sec: 174774.78452378538, Loss: 2.500988006591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17828 , TFLOPS: 69.75402932715033, Tokens per sec: 172858.16871334365, Loss: 2.500793933868408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17829 , TFLOPS: 68.97785408906176, Tokens per sec: 170934.7209705992, Loss: 2.492875814437866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17830 , TFLOPS: 71.52027394566338, Tokens per sec: 177235.1174459257, Loss: 2.4845643043518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17831 , TFLOPS: 69.54976687049756, Tokens per sec: 172351.98384439092, Loss: 2.5008037090301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17832 , TFLOPS: 67.37901648117298, Tokens per sec: 166972.6252517487, Loss: 2.5040383338928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17833 , TFLOPS: 70.66179615514581, Tokens per sec: 175107.71491188763, Loss: 2.4911584854125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17834 , TFLOPS: 71.51029101679849, Tokens per sec: 177210.3786484881, Loss: 2.480497360229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17835 , TFLOPS: 67.91516309605603, Tokens per sec: 168301.25562485415, Loss: 2.497236967086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17836 , TFLOPS: 68.97162685561578, Tokens per sec: 170919.2892001337, Loss: 2.4934463500976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17837 , TFLOPS: 71.50470236058266, Tokens per sec: 177196.52934274398, Loss: 2.483484983444214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17838 , TFLOPS: 69.7193650027766, Tokens per sec: 172772.26669895495, Loss: 2.474623441696167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17839 , TFLOPS: 67.86542214190858, Tokens per sec: 168177.99206105835, Loss: 2.4898271560668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17840 , TFLOPS: 71.51315329880343, Tokens per sec: 177217.47169860275, Loss: 2.504967451095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17841 , TFLOPS: 70.70564111448732, Tokens per sec: 175216.3675510569, Loss: 2.500612497329712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17842 , TFLOPS: 68.85354276235407, Tokens per sec: 170626.66380899443, Loss: 2.510195255279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17843 , TFLOPS: 70.59412857411138, Tokens per sec: 174940.0271918849, Loss: 2.4645795822143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17844 , TFLOPS: 71.53092335825602, Tokens per sec: 177261.50786345993, Loss: 2.4775712490081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17845 , TFLOPS: 69.80433652056858, Tokens per sec: 172982.8354804289, Loss: 2.487910270690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17846 , TFLOPS: 68.00524610920488, Tokens per sec: 168524.4912549001, Loss: 2.4722461700439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17847 , TFLOPS: 71.49583103366813, Tokens per sec: 177174.5452173924, Loss: 2.4875664710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17848 , TFLOPS: 69.79329132553258, Tokens per sec: 172955.46427040102, Loss: 2.498462200164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17849 , TFLOPS: 68.82440849191207, Tokens per sec: 170554.46587743436, Loss: 2.498187780380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17850 , TFLOPS: 70.32148649241111, Tokens per sec: 174264.39007942696, Loss: 2.486499547958374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17851 , TFLOPS: 71.53822003059308, Tokens per sec: 177279.58982130597, Loss: 2.4751853942871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17852 , TFLOPS: 68.23768429070763, Tokens per sec: 169100.4986738434, Loss: 2.521584987640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17853 , TFLOPS: 69.05741504840177, Tokens per sec: 171131.88179220696, Loss: 2.486786365509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17854 , TFLOPS: 71.55373765482715, Tokens per sec: 177318.0442035654, Loss: 2.4939332008361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17855 , TFLOPS: 69.90290556909119, Tokens per sec: 173227.100440945, Loss: 2.5167503356933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17856 , TFLOPS: 68.34040881474478, Tokens per sec: 169355.06136044793, Loss: 2.4775688648223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17857 , TFLOPS: 70.55075273502966, Tokens per sec: 174832.53708439728, Loss: 2.498629570007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17858 , TFLOPS: 70.50907407816946, Tokens per sec: 174729.25278141463, Loss: 2.501969337463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17859 , TFLOPS: 68.18554974987892, Tokens per sec: 168971.3035385761, Loss: 2.48037052154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17860 , TFLOPS: 71.52376911051662, Tokens per sec: 177243.77884945623, Loss: 2.4997520446777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17861 , TFLOPS: 69.6108660363745, Tokens per sec: 172503.39430806274, Loss: 2.470492362976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17862 , TFLOPS: 69.84739826230809, Tokens per sec: 173089.54722010184, Loss: 2.494358539581299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17863 , TFLOPS: 70.02539835929178, Tokens per sec: 173530.65106875682, Loss: 2.4957656860351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17864 , TFLOPS: 70.5209438395926, Tokens per sec: 174758.66735778336, Loss: 2.4875259399414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17865 , TFLOPS: 69.70796388758842, Tokens per sec: 172744.01347958212, Loss: 2.491830348968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17866 , TFLOPS: 68.17984635168969, Tokens per sec: 168957.16988958197, Loss: 2.4895825386047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17867 , TFLOPS: 71.52774458217279, Tokens per sec: 177253.63050615456, Loss: 2.4973766803741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17868 , TFLOPS: 68.86714988877888, Tokens per sec: 170660.38376722386, Loss: 2.5048327445983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17869 , TFLOPS: 68.7182208764442, Tokens per sec: 170291.3212687735, Loss: 2.469191551208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17870 , TFLOPS: 71.526860042003, Tokens per sec: 177251.43851817367, Loss: 2.485668420791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17871 , TFLOPS: 71.5392359988866, Tokens per sec: 177282.10750265542, Loss: 2.5012829303741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17872 , TFLOPS: 67.97869527592312, Tokens per sec: 168458.6953652122, Loss: 2.4772121906280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17873 , TFLOPS: 68.67889317425042, Tokens per sec: 170193.86289043323, Loss: 2.4894232749938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17874 , TFLOPS: 71.52072697680813, Tokens per sec: 177236.2401070079, Loss: 2.4810855388641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17875 , TFLOPS: 69.81569750303059, Tokens per sec: 173010.9892464853, Loss: 2.4748992919921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17876 , TFLOPS: 66.91296085355789, Tokens per sec: 165817.68806625286, Loss: 2.499675750732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17877 , TFLOPS: 71.51006232562521, Tokens per sec: 177209.81192656833, Loss: 2.486621379852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17878 , TFLOPS: 70.56042275316965, Tokens per sec: 174856.5004545886, Loss: 2.461381196975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17879 , TFLOPS: 67.86501052857244, Tokens per sec: 168176.97203786846, Loss: 2.4845387935638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17880 , TFLOPS: 70.90519817771131, Tokens per sec: 175710.89193675123, Loss: 2.485656976699829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17881 , TFLOPS: 70.67348739229817, Tokens per sec: 175136.68708544728, Loss: 2.483121633529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17882 , TFLOPS: 69.91270683254241, Tokens per sec: 173251.3890514754, Loss: 2.476329803466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17883 , TFLOPS: 68.65650272115325, Tokens per sec: 170138.37688114643, Loss: 2.4766082763671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17884 , TFLOPS: 71.51266572477766, Tokens per sec: 177216.26343645618, Loss: 2.479828357696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17885 , TFLOPS: 69.71743286209396, Tokens per sec: 172767.47864150052, Loss: 2.5046002864837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17886 , TFLOPS: 68.83424743456105, Tokens per sec: 170578.84786116754, Loss: 2.486682891845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17887 , TFLOPS: 70.57747878512842, Tokens per sec: 174898.7671806595, Loss: 2.4834935665130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17888 , TFLOPS: 70.69643038363832, Tokens per sec: 175193.5423453673, Loss: 2.4974470138549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17889 , TFLOPS: 69.066319498831, Tokens per sec: 171153.94800127682, Loss: 2.5063703060150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17890 , TFLOPS: 70.97531253623819, Tokens per sec: 175884.64304091502, Loss: 2.4764959812164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17891 , TFLOPS: 70.46832990870132, Tokens per sec: 174628.2842411318, Loss: 2.498492479324341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17892 , TFLOPS: 69.68471154297049, Tokens per sec: 172686.39160816165, Loss: 2.447204351425171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17893 , TFLOPS: 68.26545606805809, Tokens per sec: 169169.32019743166, Loss: 2.4772064685821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17894 , TFLOPS: 70.5362197992325, Tokens per sec: 174796.5228685569, Loss: 2.4906094074249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17895 , TFLOPS: 69.82524988273809, Tokens per sec: 173034.66109568198, Loss: 2.5108227729797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17896 , TFLOPS: 68.22233818162655, Tokens per sec: 169062.46932502702, Loss: 2.498364210128784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17897 , TFLOPS: 71.49217204894363, Tokens per sec: 177165.47785017468, Loss: 2.5160865783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17898 , TFLOPS: 69.66290759301228, Tokens per sec: 172632.35901826186, Loss: 2.4854063987731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17899 , TFLOPS: 68.76244572591752, Tokens per sec: 170400.9153175377, Loss: 2.5060324668884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17900 , TFLOPS: 70.50920534439632, Tokens per sec: 174729.57807358497, Loss: 2.4799883365631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17901 , TFLOPS: 70.48420889658097, Tokens per sec: 174667.63412231248, Loss: 2.5002970695495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17902 , TFLOPS: 69.36131302886953, Tokens per sec: 171884.97446493222, Loss: 2.487294912338257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17903 , TFLOPS: 69.63968365252717, Tokens per sec: 172574.8075354115, Loss: 2.4658000469207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17904 , TFLOPS: 71.48410343036336, Tokens per sec: 177145.48292450106, Loss: 2.496504783630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17905 , TFLOPS: 68.80444110445251, Tokens per sec: 170504.98449172024, Loss: 2.4847753047943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17906 , TFLOPS: 68.68431060445025, Tokens per sec: 170207.28787924763, Loss: 2.455617904663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17907 , TFLOPS: 71.53162986112983, Tokens per sec: 177263.25865540892, Loss: 2.497286558151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17908 , TFLOPS: 70.62950329242656, Tokens per sec: 175027.68964071682, Loss: 2.5024118423461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17909 , TFLOPS: 67.93972412137835, Tokens per sec: 168362.12055122317, Loss: 2.509225845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17910 , TFLOPS: 70.6133441324903, Tokens per sec: 174987.6454623164, Loss: 2.510040760040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17911 , TFLOPS: 71.51234268494511, Tokens per sec: 177215.46290816535, Loss: 2.479912042617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17912 , TFLOPS: 69.7975172779576, Tokens per sec: 172965.93664603727, Loss: 2.4925754070281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17913 , TFLOPS: 68.03919932967383, Tokens per sec: 168608.6310754782, Loss: 2.501009941101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17914 , TFLOPS: 71.48690877107688, Tokens per sec: 177152.43486782297, Loss: 2.4961493015289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17915 , TFLOPS: 69.18445894007857, Tokens per sec: 171446.71055082206, Loss: 2.4776604175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17916 , TFLOPS: 67.23593413120697, Tokens per sec: 166618.0514267109, Loss: 2.481451988220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17917 , TFLOPS: 71.51432356746695, Tokens per sec: 177220.3717532667, Loss: 2.482391595840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17918 , TFLOPS: 71.51187192934724, Tokens per sec: 177214.2963239358, Loss: 2.5003631114959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17919 , TFLOPS: 68.18511531151186, Tokens per sec: 168970.22695244517, Loss: 2.472792863845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17920 , TFLOPS: 69.56352529574747, Tokens per sec: 172386.07873777585, Loss: 2.496922492980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17921 , TFLOPS: 71.46081810093528, Tokens per sec: 177087.7793130875, Loss: 2.520129442214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17922 , TFLOPS: 69.82008786394609, Tokens per sec: 173021.86904447206, Loss: 2.467144727706909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17923 , TFLOPS: 69.18006947890078, Tokens per sec: 171435.83298248443, Loss: 2.467029333114624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17924 , TFLOPS: 70.43034230259872, Tokens per sec: 174534.14676852926, Loss: 2.4582626819610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17925 , TFLOPS: 70.65958606828245, Tokens per sec: 175102.2380731791, Loss: 2.4803905487060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17926 , TFLOPS: 68.20078992763435, Tokens per sec: 169009.0703778982, Loss: 2.50040340423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17927 , TFLOPS: 70.59694695901285, Tokens per sec: 174947.0114601415, Loss: 2.4861669540405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17928 , TFLOPS: 70.56425602901454, Tokens per sec: 174865.99973440246, Loss: 2.5002784729003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17929 , TFLOPS: 69.64707925140705, Tokens per sec: 172593.13464412832, Loss: 2.4946489334106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17930 , TFLOPS: 69.95943884498469, Tokens per sec: 173367.19612626353, Loss: 2.473005771636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17931 , TFLOPS: 70.4698747660088, Tokens per sec: 174632.11256771983, Loss: 2.5079002380371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17932 , TFLOPS: 69.79290861206141, Tokens per sec: 172954.5158642601, Loss: 2.501882314682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17933 , TFLOPS: 67.9474523622465, Tokens per sec: 168381.27198343046, Loss: 2.4856009483337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17934 , TFLOPS: 71.45317494322585, Tokens per sec: 177068.8387263202, Loss: 2.4743998050689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17935 , TFLOPS: 68.71774849604584, Tokens per sec: 170290.15065810937, Loss: 2.509065866470337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17936 , TFLOPS: 67.95135685435284, Tokens per sec: 168390.9477449853, Loss: 2.508370876312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17937 , TFLOPS: 71.50203195436272, Tokens per sec: 177189.91178196145, Loss: 2.474414348602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17938 , TFLOPS: 70.40351903503421, Tokens per sec: 174467.67575667758, Loss: 2.496028423309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17939 , TFLOPS: 68.8148720161896, Tokens per sec: 170530.83344006454, Loss: 2.4952049255371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17940 , TFLOPS: 69.81716874594426, Tokens per sec: 173014.63514849613, Loss: 2.494590997695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17941 , TFLOPS: 71.52635380476673, Tokens per sec: 177250.1840065356, Loss: 2.4796478748321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17942 , TFLOPS: 68.87555538072537, Tokens per sec: 170681.21350220864, Loss: 2.4623169898986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17943 , TFLOPS: 68.46390019688832, Tokens per sec: 169661.0866090984, Loss: 2.4897165298461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17944 , TFLOPS: 71.46568920469049, Tokens per sec: 177099.85044478928, Loss: 2.4872937202453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17945 , TFLOPS: 70.66941369678301, Tokens per sec: 175126.59201920612, Loss: 2.486414909362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17946 , TFLOPS: 67.7776483829883, Tokens per sec: 167960.47901737617, Loss: 2.504037857055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17947 , TFLOPS: 70.85051504445876, Tokens per sec: 175575.381108707, Loss: 2.492276430130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17948 , TFLOPS: 69.8936995449014, Tokens per sec: 173204.2868988754, Loss: 2.4694619178771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17949 , TFLOPS: 69.6094258526433, Tokens per sec: 172499.82537412748, Loss: 2.510162830352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17950 , TFLOPS: 68.56568486437854, Tokens per sec: 169913.3202276459, Loss: 2.489149570465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17951 , TFLOPS: 71.5165842014823, Tokens per sec: 177225.97385338612, Loss: 2.4615612030029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17952 , TFLOPS: 69.79296362193429, Tokens per sec: 172954.6521847849, Loss: 2.503575086593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17953 , TFLOPS: 67.21545058094081, Tokens per sec: 166567.29093270202, Loss: 2.497974395751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17954 , TFLOPS: 71.54742789329491, Tokens per sec: 177302.4079194109, Loss: 2.496115207672119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17955 , TFLOPS: 69.72214257302713, Tokens per sec: 172779.14982401038, Loss: 2.4914166927337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17956 , TFLOPS: 68.74698619032692, Tokens per sec: 170362.60488533578, Loss: 2.496187448501587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17957 , TFLOPS: 69.73687157809248, Tokens per sec: 172815.64992109616, Loss: 2.5028529167175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17958 , TFLOPS: 71.52295255532707, Tokens per sec: 177241.75533574555, Loss: 2.491086483001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17959 , TFLOPS: 69.02950576300859, Tokens per sec: 171062.71950854105, Loss: 2.476682662963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17960 , TFLOPS: 68.92781649822402, Tokens per sec: 170810.72230840762, Loss: 2.4904727935791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17961 , TFLOPS: 70.14908889617611, Tokens per sec: 173837.16984479365, Loss: 2.497962236404419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17962 , TFLOPS: 69.77690915413882, Tokens per sec: 172914.8674450421, Loss: 2.482663154602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17963 , TFLOPS: 68.65444876892991, Tokens per sec: 170133.2869612755, Loss: 2.512563943862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17964 , TFLOPS: 71.50475809292325, Tokens per sec: 177196.6674536234, Loss: 2.464723587036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17965 , TFLOPS: 69.62883124481219, Tokens per sec: 172547.9140736024, Loss: 2.4793334007263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17966 , TFLOPS: 68.79637651828097, Tokens per sec: 170484.99955879967, Loss: 2.480781316757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17967 , TFLOPS: 70.55985197711398, Tokens per sec: 174855.0860086465, Loss: 2.502098798751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17968 , TFLOPS: 70.47968273488262, Tokens per sec: 174656.4177950261, Loss: 2.5103819370269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17969 , TFLOPS: 70.03007919643228, Tokens per sec: 173542.25069882776, Loss: 2.4834046363830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17970 , TFLOPS: 68.88810309950878, Tokens per sec: 170712.30813159927, Loss: 2.47407603263855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17971 , TFLOPS: 71.4825044616459, Tokens per sec: 177141.52050387827, Loss: 2.4975974559783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17972 , TFLOPS: 69.03534705580661, Tokens per sec: 171077.19487556609, Loss: 2.490814685821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17973 , TFLOPS: 67.93983498399594, Tokens per sec: 168362.39528100207, Loss: 2.4857754707336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17974 , TFLOPS: 71.51643643470007, Tokens per sec: 177225.6076710219, Loss: 2.490622043609619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17975 , TFLOPS: 69.92796518541262, Tokens per sec: 173289.2009307341, Loss: 2.4901256561279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17976 , TFLOPS: 69.07429102537438, Tokens per sec: 171173.70232218743, Loss: 2.468393564224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17977 , TFLOPS: 70.06949588551552, Tokens per sec: 173639.9295964828, Loss: 2.512625217437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17978 , TFLOPS: 71.50310908534371, Tokens per sec: 177192.58103118808, Loss: 2.4949188232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17979 , TFLOPS: 68.19236790455159, Tokens per sec: 168988.1996769367, Loss: 2.483083724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17980 , TFLOPS: 69.0745353138073, Tokens per sec: 171174.3076958348, Loss: 2.5042872428894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17981 , TFLOPS: 71.51047390675343, Tokens per sec: 177210.83186994345, Loss: 2.5049710273742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17982 , TFLOPS: 69.69629799305604, Tokens per sec: 172715.10410782648, Loss: 2.5012853145599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17983 , TFLOPS: 67.60687846104555, Tokens per sec: 167537.29233894218, Loss: 2.4869308471679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17984 , TFLOPS: 71.51174881263744, Tokens per sec: 177213.99122716647, Loss: 2.4922587871551514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17985 , TFLOPS: 71.54289857521108, Tokens per sec: 177291.1837702536, Loss: 2.487602710723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17986 , TFLOPS: 68.37584327945136, Tokens per sec: 169442.87186741937, Loss: 2.4703338146209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17987 , TFLOPS: 68.88948153549424, Tokens per sec: 170715.7240478185, Loss: 2.5081934928894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17988 , TFLOPS: 71.50138430285212, Tokens per sec: 177188.30683017345, Loss: 2.483124256134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17989 , TFLOPS: 69.65670090548996, Tokens per sec: 172616.97816285846, Loss: 2.4829697608947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17990 , TFLOPS: 67.60353839716325, Tokens per sec: 167529.01529270338, Loss: 2.4877727031707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17991 , TFLOPS: 71.48557136490409, Tokens per sec: 177149.12062799933, Loss: 2.5070574283599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17992 , TFLOPS: 70.64041889258388, Tokens per sec: 175054.73970035932, Loss: 2.4694337844848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17993 , TFLOPS: 67.70243070084555, Tokens per sec: 167774.08131511472, Loss: 2.4818286895751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17994 , TFLOPS: 70.41712007922355, Tokens per sec: 174501.3806424575, Loss: 2.5307602882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17995 , TFLOPS: 70.50224483135646, Tokens per sec: 174712.3291555084, Loss: 2.499420642852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17996 , TFLOPS: 69.7030373628586, Tokens per sec: 172731.80500860012, Loss: 2.4912867546081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17997 , TFLOPS: 69.02339473209669, Tokens per sec: 171047.57570075575, Loss: 2.49790620803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17998 , TFLOPS: 70.4284442773811, Tokens per sec: 174529.4432529553, Loss: 2.482342004776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17999 , TFLOPS: 69.8291122709475, Tokens per sec: 173044.2325191421, Loss: 2.465695858001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18000 , TFLOPS: 68.94058481275795, Tokens per sec: 170842.36359836865, Loss: 2.4984076023101807 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/model.pt +[rank0]:[2024-08-27 16:02:15,796] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006438056996557862, 'preprocessing_with_comm': 0.0012785629951395094, 'state_converting': 1.017449838022003, : 1.026579572004266}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000, took 4.45s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016000 + eval ppl=9.925458908081055, eval loss=2.295103073120117 +------------------------------------------------------------------ +iteration: 18001 , TFLOPS: 64.71486706071698, Tokens per sec: 160370.5695075726, Loss: 2.4817495346069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18002 , TFLOPS: 69.74387243665973, Tokens per sec: 172832.99881983406, Loss: 2.488941192626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18003 , TFLOPS: 67.90407292809255, Tokens per sec: 168273.7729669578, Loss: 2.482468366622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18004 , TFLOPS: 71.50773676651478, Tokens per sec: 177204.04893491117, Loss: 2.4997310638427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18005 , TFLOPS: 70.40115216707106, Tokens per sec: 174461.81039713288, Loss: 2.49373197555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18006 , TFLOPS: 68.3780247377164, Tokens per sec: 169448.2777613076, Loss: 2.4779272079467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18007 , TFLOPS: 70.08766969300866, Tokens per sec: 173684.96629346095, Loss: 2.4975528717041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18008 , TFLOPS: 71.42620913201552, Tokens per sec: 177002.01447561168, Loss: 2.48486590385437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18009 , TFLOPS: 68.70871516721611, Tokens per sec: 170267.76507416595, Loss: 2.4836158752441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18010 , TFLOPS: 68.2846233500601, Tokens per sec: 169216.81883368306, Loss: 2.4985570907592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18011 , TFLOPS: 71.48810778604894, Tokens per sec: 177155.40615900615, Loss: 2.491736888885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18012 , TFLOPS: 70.55137150865731, Tokens per sec: 174834.07047361013, Loss: 2.4835193157196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18013 , TFLOPS: 67.37203811696006, Tokens per sec: 166955.33209649878, Loss: 2.4801039695739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18014 , TFLOPS: 71.4296058990726, Tokens per sec: 177010.4320385639, Loss: 2.5053555965423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18015 , TFLOPS: 70.68382888998393, Tokens per sec: 175162.3144559226, Loss: 2.4862349033355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18016 , TFLOPS: 67.71757431667469, Tokens per sec: 167811.6088633463, Loss: 2.4760665893554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18017 , TFLOPS: 70.0210164704498, Tokens per sec: 173519.79226550146, Loss: 2.5025782585144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18018 , TFLOPS: 71.4910966665655, Tokens per sec: 177162.8129341785, Loss: 2.4681742191314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18019 , TFLOPS: 69.06617735918901, Tokens per sec: 171153.59576358594, Loss: 2.4873979091644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18020 , TFLOPS: 68.38983311485836, Tokens per sec: 169477.5402206655, Loss: 2.512784719467163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18021 , TFLOPS: 71.47803056785882, Tokens per sec: 177130.4337022338, Loss: 2.514538288116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18022 , TFLOPS: 69.75950837172049, Tokens per sec: 172871.74638935504, Loss: 2.487736940383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18023 , TFLOPS: 68.59503336657353, Tokens per sec: 169986.04904908946, Loss: 2.4827680587768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18024 , TFLOPS: 71.44057007135645, Tokens per sec: 177037.60246528656, Loss: 2.48721981048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18025 , TFLOPS: 71.51481600399481, Tokens per sec: 177221.59206522908, Loss: 2.495885133743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18026 , TFLOPS: 68.95293466652365, Tokens per sec: 170872.96789644877, Loss: 2.4916300773620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18027 , TFLOPS: 68.07546729188428, Tokens per sec: 168698.5070810815, Loss: 2.4719910621643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18028 , TFLOPS: 71.48993051724842, Tokens per sec: 177159.92308770886, Loss: 2.4905178546905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18029 , TFLOPS: 69.83469330723287, Tokens per sec: 173058.0629418591, Loss: 2.485377073287964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18030 , TFLOPS: 68.0869636253628, Tokens per sec: 168726.99626186697, Loss: 2.477036476135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18031 , TFLOPS: 70.4620811796133, Tokens per sec: 174612.79920209735, Loss: 2.48341965675354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18032 , TFLOPS: 70.59109975182422, Tokens per sec: 174932.52143660502, Loss: 2.490633010864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18033 , TFLOPS: 68.81439142756427, Tokens per sec: 170529.64248850977, Loss: 2.483294725418091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18034 , TFLOPS: 70.80212443821567, Tokens per sec: 175455.46385577126, Loss: 2.4991965293884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18035 , TFLOPS: 69.61863870523919, Tokens per sec: 172522.65583773958, Loss: 2.4665277004241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18036 , TFLOPS: 68.58084430312881, Tokens per sec: 169950.8869868191, Loss: 2.471403121948242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18037 , TFLOPS: 69.70489437000359, Tokens per sec: 172736.40687687165, Loss: 2.4967501163482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18038 , TFLOPS: 70.40309272285157, Tokens per sec: 174466.61930812683, Loss: 2.4929654598236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18039 , TFLOPS: 69.71370945010781, Tokens per sec: 172758.25161643117, Loss: 2.470292806625366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18040 , TFLOPS: 69.05988252456629, Tokens per sec: 171137.9964699576, Loss: 2.4925026893615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18041 , TFLOPS: 71.52128503732727, Tokens per sec: 177237.6230424498, Loss: 2.484808921813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18042 , TFLOPS: 69.56174378467169, Tokens per sec: 172381.66395708043, Loss: 2.5165278911590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18043 , TFLOPS: 69.38737765341374, Tokens per sec: 171949.56547580822, Loss: 2.479915142059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18044 , TFLOPS: 70.85344656100936, Tokens per sec: 175582.6457296526, Loss: 2.4632463455200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18045 , TFLOPS: 71.49678666047431, Tokens per sec: 177176.91336588914, Loss: 2.481935977935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18046 , TFLOPS: 67.92599824780433, Tokens per sec: 168328.10632447695, Loss: 2.5020103454589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18047 , TFLOPS: 68.7945355204422, Tokens per sec: 170480.4373633525, Loss: 2.4886350631713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18048 , TFLOPS: 71.53419076721475, Tokens per sec: 177269.60486279448, Loss: 2.4966256618499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18049 , TFLOPS: 69.84191136462924, Tokens per sec: 173075.950083222, Loss: 2.509557008743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18050 , TFLOPS: 66.90253649977697, Tokens per sec: 165791.85536924636, Loss: 2.4786391258239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18051 , TFLOPS: 71.4462218567989, Tokens per sec: 177051.6082122085, Loss: 2.4787065982818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18052 , TFLOPS: 70.51452902869643, Tokens per sec: 174742.77074973285, Loss: 2.4934895038604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18053 , TFLOPS: 68.60240584867337, Tokens per sec: 170004.31887187884, Loss: 2.5069780349731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18054 , TFLOPS: 70.58613130198017, Tokens per sec: 174920.20907057155, Loss: 2.5027222633361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18055 , TFLOPS: 71.55280177954943, Tokens per sec: 177315.72500153203, Loss: 2.4803364276885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18056 , TFLOPS: 69.34747882266755, Tokens per sec: 171850.6917780001, Loss: 2.495779514312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18057 , TFLOPS: 68.28728102036033, Tokens per sec: 169223.40483345356, Loss: 2.492764949798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18058 , TFLOPS: 71.51769501412272, Tokens per sec: 177228.7265694752, Loss: 2.4648866653442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18059 , TFLOPS: 69.71109194322065, Tokens per sec: 172751.76514602258, Loss: 2.491034984588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18060 , TFLOPS: 67.80318576475865, Tokens per sec: 168023.76346257792, Loss: 2.4966909885406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18061 , TFLOPS: 71.51805343181255, Tokens per sec: 177229.61476799357, Loss: 2.505709409713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18062 , TFLOPS: 70.70143149119491, Tokens per sec: 175205.93564081, Loss: 2.491049289703369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18063 , TFLOPS: 68.90260847855282, Tokens per sec: 170748.2540587698, Loss: 2.466238260269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18064 , TFLOPS: 69.76150120638324, Tokens per sec: 172876.68485317822, Loss: 2.4945006370544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18065 , TFLOPS: 71.4959099047949, Tokens per sec: 177174.74066873293, Loss: 2.4871020317077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18066 , TFLOPS: 68.951151057668, Tokens per sec: 170868.54791722377, Loss: 2.5052976608276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18067 , TFLOPS: 68.87166221200278, Tokens per sec: 170671.56580124612, Loss: 2.4809482097625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18068 , TFLOPS: 70.30347822450506, Tokens per sec: 174219.7636077813, Loss: 2.4732511043548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18069 , TFLOPS: 69.25408961480488, Tokens per sec: 171619.26303324592, Loss: 2.5049147605895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18070 , TFLOPS: 68.01720466597808, Tokens per sec: 168554.1258759279, Loss: 2.487352132797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18071 , TFLOPS: 71.518094452225, Tokens per sec: 177229.71642109434, Loss: 2.475372552871704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18072 , TFLOPS: 70.48936407119231, Tokens per sec: 174680.40921288874, Loss: 2.4991350173950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18073 , TFLOPS: 68.1551410575031, Tokens per sec: 168895.9474490748, Loss: 2.505676746368408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18074 , TFLOPS: 70.65533409244863, Tokens per sec: 175091.70120866905, Loss: 2.4794294834136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18075 , TFLOPS: 69.54774882123293, Tokens per sec: 172346.98289600667, Loss: 2.505096435546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18076 , TFLOPS: 69.80016288615163, Tokens per sec: 172972.49275457987, Loss: 2.4861793518066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18077 , TFLOPS: 68.42663130631708, Tokens per sec: 169568.73019275564, Loss: 2.5079994201660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18078 , TFLOPS: 71.52044222074205, Tokens per sec: 177235.5344501077, Loss: 2.5037131309509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18079 , TFLOPS: 69.57263135914724, Tokens per sec: 172408.64456599616, Loss: 2.504587173461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18080 , TFLOPS: 68.14343145391756, Tokens per sec: 168866.9297614705, Loss: 2.482480764389038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18081 , TFLOPS: 71.54371685235424, Tokens per sec: 177293.21155115502, Loss: 2.494873523712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18082 , TFLOPS: 70.63222960275071, Tokens per sec: 175034.44573802798, Loss: 2.485848903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18083 , TFLOPS: 68.87125903366436, Tokens per sec: 170670.56668090975, Loss: 2.4719672203063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18084 , TFLOPS: 70.12496545908162, Tokens per sec: 173777.38931025757, Loss: 2.4954683780670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18085 , TFLOPS: 71.54204198536705, Tokens per sec: 177289.06104625892, Loss: 2.502673387527466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18086 , TFLOPS: 68.87453350450541, Tokens per sec: 170678.6811803666, Loss: 2.4945738315582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18087 , TFLOPS: 67.89735277923239, Tokens per sec: 168257.11969779685, Loss: 2.489471435546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18088 , TFLOPS: 71.45371080174228, Tokens per sec: 177070.1666427536, Loss: 2.484341621398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18089 , TFLOPS: 69.87626317673316, Tokens per sec: 173161.07765777997, Loss: 2.497037172317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18090 , TFLOPS: 67.87397414425229, Tokens per sec: 168199.18486494687, Loss: 2.5050833225250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18091 , TFLOPS: 71.4454621519975, Tokens per sec: 177049.72558002226, Loss: 2.493793249130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18092 , TFLOPS: 71.48005983954576, Tokens per sec: 177135.46246101675, Loss: 2.5001907348632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18093 , TFLOPS: 68.82111369176981, Tokens per sec: 170546.30099972963, Loss: 2.4736480712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18094 , TFLOPS: 68.84124672513985, Tokens per sec: 170596.1928742573, Loss: 2.4903414249420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18095 , TFLOPS: 71.54645649651309, Tokens per sec: 177300.00069117735, Loss: 2.4948863983154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18096 , TFLOPS: 69.8767806520269, Tokens per sec: 173162.36001856244, Loss: 2.4988722801208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18097 , TFLOPS: 67.28484453235632, Tokens per sec: 166739.25678868443, Loss: 2.4768333435058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18098 , TFLOPS: 71.42948775890731, Tokens per sec: 177010.1392742201, Loss: 2.484844207763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18099 , TFLOPS: 70.60920726725273, Tokens per sec: 174977.39385454435, Loss: 2.5035455226898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18100 , TFLOPS: 68.74502431315423, Tokens per sec: 170357.74313758322, Loss: 2.4729480743408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18101 , TFLOPS: 69.83237263552304, Tokens per sec: 173052.31206171904, Loss: 2.4839863777160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18102 , TFLOPS: 70.68249209168965, Tokens per sec: 175159.00172248887, Loss: 2.514477014541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18103 , TFLOPS: 69.82045006610512, Tokens per sec: 173022.76662132205, Loss: 2.4979565143585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18104 , TFLOPS: 69.77746396761866, Tokens per sec: 172916.24233396287, Loss: 2.4947049617767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18105 , TFLOPS: 70.4928132156779, Tokens per sec: 174688.95657287864, Loss: 2.4914205074310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18106 , TFLOPS: 69.75903464624204, Tokens per sec: 172870.5724454343, Loss: 2.5046119689941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18107 , TFLOPS: 68.41131165274346, Tokens per sec: 169530.76640360107, Loss: 2.51674222946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18108 , TFLOPS: 71.53372602488645, Tokens per sec: 177268.45318010933, Loss: 2.490330696105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18109 , TFLOPS: 68.88014969035213, Tokens per sec: 170692.59870756872, Loss: 2.4903600215911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18110 , TFLOPS: 68.8148445119502, Tokens per sec: 170530.76528152946, Loss: 2.5155463218688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18111 , TFLOPS: 70.89083792398115, Tokens per sec: 175675.30564609537, Loss: 2.47576642036438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18112 , TFLOPS: 70.43422563483064, Tokens per sec: 174543.77009358435, Loss: 2.4800021648406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18113 , TFLOPS: 68.88793377133362, Tokens per sec: 170711.88851772816, Loss: 2.487323760986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18114 , TFLOPS: 69.60086000510302, Tokens per sec: 172478.59826031665, Loss: 2.4917454719543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18115 , TFLOPS: 71.48827082261936, Tokens per sec: 177155.8101815876, Loss: 2.4806056022644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18116 , TFLOPS: 68.79074473848061, Tokens per sec: 170471.04338806262, Loss: 2.474761724472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18117 , TFLOPS: 68.05211080046003, Tokens per sec: 168640.6271224023, Loss: 2.484591484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18118 , TFLOPS: 71.47204779297407, Tokens per sec: 177115.6077270123, Loss: 2.490163564682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18119 , TFLOPS: 70.69086746844897, Tokens per sec: 175179.75682872243, Loss: 2.4807674884796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18120 , TFLOPS: 67.85264894085745, Tokens per sec: 168146.33866176938, Loss: 2.4849209785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18121 , TFLOPS: 70.63586930210272, Tokens per sec: 175043.46531396825, Loss: 2.4869742393493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18122 , TFLOPS: 71.45456726424545, Tokens per sec: 177072.2890511836, Loss: 2.480964422225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18123 , TFLOPS: 70.01587140236732, Tokens per sec: 173507.0422200171, Loss: 2.499246597290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18124 , TFLOPS: 68.9068690707305, Tokens per sec: 170758.81227552902, Loss: 2.4827489852905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18125 , TFLOPS: 71.51754822767319, Tokens per sec: 177228.3628164834, Loss: 2.467992067337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18126 , TFLOPS: 69.7456470998498, Tokens per sec: 172837.39663071436, Loss: 2.496142864227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18127 , TFLOPS: 67.77696746894837, Tokens per sec: 167958.79163737633, Loss: 2.4927451610565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18128 , TFLOPS: 71.49944424777497, Tokens per sec: 177183.4991599776, Loss: 2.4919676780700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18129 , TFLOPS: 69.72376421889186, Tokens per sec: 172783.16844110226, Loss: 2.4790568351745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18130 , TFLOPS: 69.2739960163747, Tokens per sec: 171668.59328920706, Loss: 2.469660520553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18131 , TFLOPS: 69.06546270505228, Tokens per sec: 171151.8247719095, Loss: 2.5005030632019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18132 , TFLOPS: 71.52732392142308, Tokens per sec: 177252.58806247715, Loss: 2.4851183891296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18133 , TFLOPS: 68.89413948142477, Tokens per sec: 170727.26695094947, Loss: 2.4852442741394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18134 , TFLOPS: 68.53679355870574, Tokens per sec: 169841.72438954862, Loss: 2.477844715118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18135 , TFLOPS: 71.56757567488884, Tokens per sec: 177352.336341662, Loss: 2.521505355834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18136 , TFLOPS: 69.81807514478812, Tokens per sec: 173016.88130467932, Loss: 2.4969656467437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18137 , TFLOPS: 68.15860599481685, Tokens per sec: 168904.53394543263, Loss: 2.4886484146118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18138 , TFLOPS: 70.50311075494577, Tokens per sec: 174714.47500955203, Loss: 2.480790138244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18139 , TFLOPS: 71.45207017472947, Tokens per sec: 177066.10098828585, Loss: 2.483274459838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18140 , TFLOPS: 68.24557611225002, Tokens per sec: 169120.05547698133, Loss: 2.4965713024139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18141 , TFLOPS: 70.48685235222776, Tokens per sec: 174674.1848966059, Loss: 2.4875810146331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18142 , TFLOPS: 70.47327311113226, Tokens per sec: 174640.53404129436, Loss: 2.49198317527771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18143 , TFLOPS: 69.78062156954077, Tokens per sec: 172924.06721936446, Loss: 2.498861312866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18144 , TFLOPS: 68.97986564994288, Tokens per sec: 170939.70584005438, Loss: 2.4687700271606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18145 , TFLOPS: 71.49538802077404, Tokens per sec: 177173.447382639, Loss: 2.484243154525757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18146 , TFLOPS: 69.72240025206177, Tokens per sec: 172779.78838104394, Loss: 2.5090603828430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18147 , TFLOPS: 67.46635498847255, Tokens per sec: 167189.05969396784, Loss: 2.502061605453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18148 , TFLOPS: 71.50063261706808, Tokens per sec: 177186.44406999557, Loss: 2.472764730453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18149 , TFLOPS: 69.36233974976913, Tokens per sec: 171887.5187924241, Loss: 2.48781418800354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18150 , TFLOPS: 69.03836281130836, Tokens per sec: 171084.66824991233, Loss: 2.478186845779419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18151 , TFLOPS: 69.67525214186576, Tokens per sec: 172662.9501701851, Loss: 2.484267234802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18152 , TFLOPS: 71.51431707622078, Tokens per sec: 177220.3556672436, Loss: 2.5003609657287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18153 , TFLOPS: 68.92899405886347, Tokens per sec: 170813.6404333912, Loss: 2.494823455810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18154 , TFLOPS: 69.03657405999624, Tokens per sec: 171080.23552711366, Loss: 2.4908509254455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18155 , TFLOPS: 71.49740613189721, Tokens per sec: 177178.44848431548, Loss: 2.4813358783721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18156 , TFLOPS: 69.64475990105939, Tokens per sec: 172587.3870384693, Loss: 2.4690566062927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18157 , TFLOPS: 67.28866377514488, Tokens per sec: 166748.72129303883, Loss: 2.4903812408447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18158 , TFLOPS: 71.53830530716385, Tokens per sec: 177279.80114604233, Loss: 2.490614891052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18159 , TFLOPS: 71.47191299418716, Tokens per sec: 177115.27368076902, Loss: 2.4909703731536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18160 , TFLOPS: 68.9007230811652, Tokens per sec: 170743.5818363496, Loss: 2.5150210857391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18161 , TFLOPS: 68.51600511255974, Tokens per sec: 169790.20832996245, Loss: 2.505099058151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18162 , TFLOPS: 71.4989477840211, Tokens per sec: 177182.2688681055, Loss: 2.4690558910369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18163 , TFLOPS: 69.6818978940142, Tokens per sec: 172679.41907609897, Loss: 2.501375675201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18164 , TFLOPS: 67.95834044110316, Tokens per sec: 168408.25384225816, Loss: 2.50144100189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18165 , TFLOPS: 71.51033714701823, Tokens per sec: 177210.4929642543, Loss: 2.496713161468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18166 , TFLOPS: 70.7061418346411, Tokens per sec: 175217.60839075682, Loss: 2.4918150901794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18167 , TFLOPS: 67.78142777550666, Tokens per sec: 167969.84476837108, Loss: 2.4574851989746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18168 , TFLOPS: 69.82726178157051, Tokens per sec: 173039.64680261764, Loss: 2.4904403686523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18169 , TFLOPS: 71.54379727935894, Tokens per sec: 177293.41085813244, Loss: 2.4981179237365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18170 , TFLOPS: 68.27209016233326, Tokens per sec: 169185.76021385135, Loss: 2.4901962280273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18171 , TFLOPS: 68.71634374604352, Tokens per sec: 170286.6695328576, Loss: 2.4721603393554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18172 , TFLOPS: 71.50433318302811, Tokens per sec: 177195.61448009554, Loss: 2.518602132797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18173 , TFLOPS: 69.88298475441624, Tokens per sec: 173177.73446772157, Loss: 2.4648642539978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18174 , TFLOPS: 68.31748684350111, Tokens per sec: 169298.25819064252, Loss: 2.4694833755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18175 , TFLOPS: 70.37355819936191, Tokens per sec: 174393.42950542312, Loss: 2.4991378784179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18176 , TFLOPS: 69.69475323215325, Tokens per sec: 172711.27602013995, Loss: 2.48199725151062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18177 , TFLOPS: 68.9084704400499, Tokens per sec: 170762.78064510794, Loss: 2.4919731616973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18178 , TFLOPS: 70.88922370714955, Tokens per sec: 175671.30543896597, Loss: 2.505964517593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18179 , TFLOPS: 70.50752547344527, Tokens per sec: 174725.4151683152, Loss: 2.459312915802002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18180 , TFLOPS: 69.11564885833616, Tokens per sec: 171276.1915882081, Loss: 2.4836177825927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18181 , TFLOPS: 69.51441697472752, Tokens per sec: 172264.3829085603, Loss: 2.508685827255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18182 , TFLOPS: 71.43728985840445, Tokens per sec: 177029.4737362462, Loss: 2.475541114807129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18183 , TFLOPS: 68.60636245374852, Tokens per sec: 170014.12377511107, Loss: 2.4789011478424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18184 , TFLOPS: 68.88111024747123, Tokens per sec: 170694.97907392395, Loss: 2.482088804244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18185 , TFLOPS: 71.54722531190822, Tokens per sec: 177301.90590041853, Loss: 2.4743661880493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18186 , TFLOPS: 69.71502233973592, Tokens per sec: 172761.50510155634, Loss: 2.497326135635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18187 , TFLOPS: 68.1158776077989, Tokens per sec: 168798.64829548172, Loss: 2.4918737411499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18188 , TFLOPS: 70.5153955550166, Tokens per sec: 174744.91809741, Loss: 2.5001718997955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18189 , TFLOPS: 71.5233332205745, Tokens per sec: 177242.69866616244, Loss: 2.508035659790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18190 , TFLOPS: 67.97161675816923, Tokens per sec: 168441.15401845495, Loss: 2.4980621337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18191 , TFLOPS: 69.26547616172708, Tokens per sec: 171647.48015085078, Loss: 2.499110221862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18192 , TFLOPS: 71.50824691222925, Tokens per sec: 177205.31313218884, Loss: 2.496079444885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18193 , TFLOPS: 69.83123595988208, Tokens per sec: 173049.49525426558, Loss: 2.478114604949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18194 , TFLOPS: 67.68440873672867, Tokens per sec: 167729.42090274981, Loss: 2.4829485416412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18195 , TFLOPS: 71.5288892651177, Tokens per sec: 177256.4671565893, Loss: 2.5014612674713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18196 , TFLOPS: 70.69303787575573, Tokens per sec: 175185.1353371187, Loss: 2.487989664077759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18197 , TFLOPS: 68.32103366502623, Tokens per sec: 169307.04760509645, Loss: 2.48400616645813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18198 , TFLOPS: 69.57194443697871, Tokens per sec: 172406.94229719182, Loss: 2.5023560523986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18199 , TFLOPS: 71.51512522279626, Tokens per sec: 177222.3583434823, Loss: 2.494813919067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18200 , TFLOPS: 69.07994517339253, Tokens per sec: 171187.7139238322, Loss: 2.516071081161499 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/optimizer.pt +[rank0]:[2024-08-27 16:27:40,306] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006412624003132805, 'preprocessing_with_comm': 0.0012437789991963655, 'state_converting': 0.9923197210009675, : 1.0013554219913203}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200, took 4.38s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016200 + eval ppl=9.969962120056152, eval loss=2.299576759338379 +------------------------------------------------------------------ +iteration: 18201 , TFLOPS: 64.22130457565667, Tokens per sec: 159147.46729919701, Loss: 2.506438970565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18202 , TFLOPS: 71.48373235171073, Tokens per sec: 177144.56335072216, Loss: 2.5002307891845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18203 , TFLOPS: 70.59725613358144, Tokens per sec: 174947.7776287808, Loss: 2.4865379333496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18204 , TFLOPS: 67.9047526008615, Tokens per sec: 168275.4572709514, Loss: 2.48121976852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18205 , TFLOPS: 71.47988889799508, Tokens per sec: 177135.03884902343, Loss: 2.4935522079467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18206 , TFLOPS: 70.5242718450621, Tokens per sec: 174766.91452194683, Loss: 2.5007851123809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18207 , TFLOPS: 69.72326479164296, Tokens per sec: 172781.93080536302, Loss: 2.477780818939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18208 , TFLOPS: 68.8961408552077, Tokens per sec: 170732.22657565307, Loss: 2.4952683448791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18209 , TFLOPS: 71.4799187858984, Tokens per sec: 177135.11291454028, Loss: 2.477102279663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18210 , TFLOPS: 69.1743473132456, Tokens per sec: 171421.65282564235, Loss: 2.4935402870178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18211 , TFLOPS: 69.53212503467964, Tokens per sec: 172308.26543182545, Loss: 2.4924709796905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18212 , TFLOPS: 70.35646725680056, Tokens per sec: 174351.07626703332, Loss: 2.5066874027252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18213 , TFLOPS: 70.58839341463099, Tokens per sec: 174925.8148349125, Loss: 2.4820261001586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18214 , TFLOPS: 67.43591749943366, Tokens per sec: 167113.63224316324, Loss: 2.516524076461792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18215 , TFLOPS: 71.53838810007997, Tokens per sec: 177280.00631600886, Loss: 2.501891613006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18216 , TFLOPS: 69.56982780482016, Tokens per sec: 172401.69704953668, Loss: 2.5046257972717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18217 , TFLOPS: 67.78651898085975, Tokens per sec: 167982.46133607862, Loss: 2.5054237842559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18218 , TFLOPS: 70.70193387960103, Tokens per sec: 175207.18061462304, Loss: 2.479295015335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18219 , TFLOPS: 70.38595564641274, Tokens per sec: 174424.15174490662, Loss: 2.481062412261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18220 , TFLOPS: 68.97553012118536, Tokens per sec: 170928.9619221953, Loss: 2.4770052433013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18221 , TFLOPS: 68.84848894092953, Tokens per sec: 170614.13988278006, Loss: 2.458817720413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18222 , TFLOPS: 71.39581460734682, Tokens per sec: 176926.69349524932, Loss: 2.496692180633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18223 , TFLOPS: 68.76383975068437, Tokens per sec: 170404.36986447446, Loss: 2.4807374477386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18224 , TFLOPS: 67.90399387483495, Tokens per sec: 168273.57706427702, Loss: 2.4816746711730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18225 , TFLOPS: 71.50838071810956, Tokens per sec: 177205.64471790017, Loss: 2.497572660446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18226 , TFLOPS: 71.52176219743768, Tokens per sec: 177238.80549776688, Loss: 2.476148843765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18227 , TFLOPS: 67.33490856777391, Tokens per sec: 166863.32098345802, Loss: 2.4851062297821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18228 , TFLOPS: 69.77530674647849, Tokens per sec: 172910.89650234004, Loss: 2.4730288982391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18229 , TFLOPS: 71.43487956609407, Tokens per sec: 177023.5007663855, Loss: 2.480722665786743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18230 , TFLOPS: 69.05881949131441, Tokens per sec: 171135.36215645666, Loss: 2.4951205253601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18231 , TFLOPS: 68.61103920357674, Tokens per sec: 170025.71327637194, Loss: 2.4920878410339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18232 , TFLOPS: 71.548014043107, Tokens per sec: 177303.860463215, Loss: 2.4930052757263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18233 , TFLOPS: 70.57310135126876, Tokens per sec: 174887.9194173411, Loss: 2.4911649227142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18234 , TFLOPS: 66.26323704522916, Tokens per sec: 164207.60089622828, Loss: 2.488617420196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18235 , TFLOPS: 71.48894914567875, Tokens per sec: 177157.4911408522, Loss: 2.498154640197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18236 , TFLOPS: 70.73859590591874, Tokens per sec: 175298.03315449393, Loss: 2.485952615737915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18237 , TFLOPS: 68.9598233934569, Tokens per sec: 170890.03892064528, Loss: 2.4887287616729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18238 , TFLOPS: 69.05081471023271, Tokens per sec: 171115.52542713677, Loss: 2.483280658721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18239 , TFLOPS: 71.48497050485071, Tokens per sec: 177147.63163059685, Loss: 2.4712464809417725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18240 , TFLOPS: 68.95741137327612, Tokens per sec: 170884.06166893136, Loss: 2.495067834854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18241 , TFLOPS: 67.87736159234782, Tokens per sec: 168207.57933448584, Loss: 2.486521005630493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18242 , TFLOPS: 71.47204259213068, Tokens per sec: 177115.59483874947, Loss: 2.4763262271881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18243 , TFLOPS: 70.63808648188753, Tokens per sec: 175048.95972971752, Loss: 2.5106680393218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18244 , TFLOPS: 67.5674450171693, Tokens per sec: 167439.57192106434, Loss: 2.5001027584075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18245 , TFLOPS: 70.43704330809598, Tokens per sec: 174550.7525983282, Loss: 2.5062618255615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18246 , TFLOPS: 71.54016969228361, Tokens per sec: 177284.42129774808, Loss: 2.4956142902374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18247 , TFLOPS: 68.73051993258733, Tokens per sec: 170321.79968474904, Loss: 2.4955029487609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18248 , TFLOPS: 69.12797936426333, Tokens per sec: 171306.74794021298, Loss: 2.4963419437408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18249 , TFLOPS: 70.22624569530609, Tokens per sec: 174028.37289256084, Loss: 2.516249895095825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18250 , TFLOPS: 69.85107460994992, Tokens per sec: 173098.65761454141, Loss: 2.488363742828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18251 , TFLOPS: 69.07832960549243, Tokens per sec: 171183.71036860626, Loss: 2.4594874382019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18252 , TFLOPS: 71.51485772552265, Tokens per sec: 177221.69545577106, Loss: 2.4837403297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18253 , TFLOPS: 69.53148309098452, Tokens per sec: 172306.67462463304, Loss: 2.499603033065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18254 , TFLOPS: 68.15019640432853, Tokens per sec: 168883.6940538097, Loss: 2.4819347858428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18255 , TFLOPS: 70.77001768911155, Tokens per sec: 175375.89979464086, Loss: 2.492560386657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18256 , TFLOPS: 69.73833916054237, Tokens per sec: 172819.28675207452, Loss: 2.495387077331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18257 , TFLOPS: 69.9105647908024, Tokens per sec: 173246.0808360783, Loss: 2.4901719093322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18258 , TFLOPS: 68.56704107660215, Tokens per sec: 169916.68107093513, Loss: 2.491757869720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18259 , TFLOPS: 71.53686537372671, Tokens per sec: 177276.23283236334, Loss: 2.502974033355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18260 , TFLOPS: 68.67077827308303, Tokens per sec: 170173.7532714103, Loss: 2.4816222190856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18261 , TFLOPS: 68.27720941017819, Tokens per sec: 169198.44627394277, Loss: 2.499821901321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18262 , TFLOPS: 71.53304648475381, Tokens per sec: 177266.76920480313, Loss: 2.5089938640594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18263 , TFLOPS: 70.64343949689551, Tokens per sec: 175062.22509059057, Loss: 2.4875853061676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18264 , TFLOPS: 67.95919203850991, Tokens per sec: 168410.36419444327, Loss: 2.4905147552490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18265 , TFLOPS: 70.92729636602165, Tokens per sec: 175765.6536817006, Loss: 2.492250919342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18266 , TFLOPS: 71.50515234557133, Tokens per sec: 177197.64445511586, Loss: 2.50834584236145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18267 , TFLOPS: 68.86928668866429, Tokens per sec: 170665.67899272786, Loss: 2.4894957542419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18268 , TFLOPS: 67.71675989233327, Tokens per sec: 167809.59063011204, Loss: 2.4735121726989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18269 , TFLOPS: 71.5419996035613, Tokens per sec: 177288.95601947553, Loss: 2.49509596824646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18270 , TFLOPS: 69.8299968478465, Tokens per sec: 173046.42459814198, Loss: 2.478938341140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18271 , TFLOPS: 68.786551874957, Tokens per sec: 170460.65301036905, Loss: 2.49060320854187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18272 , TFLOPS: 71.51067422261937, Tokens per sec: 177211.32827472585, Loss: 2.4918341636657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18273 , TFLOPS: 70.64147546111191, Tokens per sec: 175057.3579935624, Loss: 2.4684205055236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18274 , TFLOPS: 68.90292974875494, Tokens per sec: 170749.0502017215, Loss: 2.5186235904693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18275 , TFLOPS: 68.93591548582174, Tokens per sec: 170830.7924918519, Loss: 2.494372606277466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18276 , TFLOPS: 71.45709233320534, Tokens per sec: 177078.5464502255, Loss: 2.4937567710876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18277 , TFLOPS: 69.39657916559821, Tokens per sec: 171972.36783663157, Loss: 2.5105113983154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18278 , TFLOPS: 67.98835253042733, Tokens per sec: 168482.62710570803, Loss: 2.4811408519744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18279 , TFLOPS: 71.51279701619791, Tokens per sec: 177216.58879105846, Loss: 2.489363193511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18280 , TFLOPS: 69.7328529242485, Tokens per sec: 172805.69125417952, Loss: 2.4840142726898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18281 , TFLOPS: 68.10682848942905, Tokens per sec: 168776.22358331858, Loss: 2.4665000438690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18282 , TFLOPS: 70.46692832713107, Tokens per sec: 174624.81096760376, Loss: 2.4905543327331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18283 , TFLOPS: 70.69375467575017, Tokens per sec: 175186.9116464669, Loss: 2.517596483230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18284 , TFLOPS: 69.11755802856504, Tokens per sec: 171280.92272234775, Loss: 2.4849069118499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18285 , TFLOPS: 69.99430594597548, Tokens per sec: 173453.60064344696, Loss: 2.4687724113464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18286 , TFLOPS: 70.54412086790802, Tokens per sec: 174816.10258710937, Loss: 2.484880208969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18287 , TFLOPS: 68.72087719015839, Tokens per sec: 170297.90390677436, Loss: 2.5041027069091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18288 , TFLOPS: 68.9464416015426, Tokens per sec: 170856.8773661556, Loss: 2.5024640560150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18289 , TFLOPS: 71.51018395846384, Tokens per sec: 177210.1133461414, Loss: 2.4735450744628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18290 , TFLOPS: 69.56544747098728, Tokens per sec: 172390.8420997652, Loss: 2.492238998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18291 , TFLOPS: 68.26363826113814, Tokens per sec: 169164.81547163604, Loss: 2.46464467048645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18292 , TFLOPS: 71.49246756636562, Tokens per sec: 177166.2101749003, Loss: 2.4853034019470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18293 , TFLOPS: 70.48467680426506, Tokens per sec: 174668.79364909796, Loss: 2.4996137619018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18294 , TFLOPS: 68.35256395446024, Tokens per sec: 169385.18313566202, Loss: 2.484435796737671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18295 , TFLOPS: 69.11092235479053, Tokens per sec: 171264.47879175472, Loss: 2.471700668334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18296 , TFLOPS: 71.50221768981221, Tokens per sec: 177190.37205486573, Loss: 2.4906575679779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18297 , TFLOPS: 68.95600546562203, Tokens per sec: 170880.57767489104, Loss: 2.4606728553771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18298 , TFLOPS: 68.83909294013658, Tokens per sec: 170590.8555577602, Loss: 2.500640392303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18299 , TFLOPS: 71.5052958747031, Tokens per sec: 177198.00013611538, Loss: 2.506770133972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18300 , TFLOPS: 70.63364767205562, Tokens per sec: 175037.95986997875, Loss: 2.505070686340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18301 , TFLOPS: 66.49311342813854, Tokens per sec: 164777.25989605178, Loss: 2.482473611831665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18302 , TFLOPS: 71.48282950999949, Tokens per sec: 177142.3260095055, Loss: 2.482509136199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18303 , TFLOPS: 70.6092005342052, Tokens per sec: 174977.37716931084, Loss: 2.500483274459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18304 , TFLOPS: 69.0016299719134, Tokens per sec: 170993.6402274353, Loss: 2.4792287349700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18305 , TFLOPS: 69.48851424211482, Tokens per sec: 172200.19308372444, Loss: 2.4923152923583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18306 , TFLOPS: 71.48674335652184, Tokens per sec: 177152.02495233362, Loss: 2.49232816696167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18307 , TFLOPS: 68.9390555683955, Tokens per sec: 170838.57396237834, Loss: 2.495805501937866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18308 , TFLOPS: 68.67357964296987, Tokens per sec: 170180.69537458627, Loss: 2.4812159538269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18309 , TFLOPS: 71.42449073640168, Tokens per sec: 176997.7561019837, Loss: 2.4955902099609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18310 , TFLOPS: 70.60116536030905, Tokens per sec: 174957.46512324872, Loss: 2.4995908737182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18311 , TFLOPS: 68.0817001492971, Tokens per sec: 168713.95278835643, Loss: 2.480074644088745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18312 , TFLOPS: 70.49513883988146, Tokens per sec: 174694.71972584375, Loss: 2.4875431060791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18313 , TFLOPS: 71.49787319229173, Tokens per sec: 177179.60591142406, Loss: 2.472193479537964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18314 , TFLOPS: 68.91375356530673, Tokens per sec: 170775.87281147868, Loss: 2.4941489696502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18315 , TFLOPS: 68.10846366595955, Tokens per sec: 168780.27573089072, Loss: 2.4978573322296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18316 , TFLOPS: 71.53149210238124, Tokens per sec: 177262.91727405455, Loss: 2.4970664978027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18317 , TFLOPS: 69.8231902343032, Tokens per sec: 173029.55705997127, Loss: 2.4799914360046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18318 , TFLOPS: 68.42448973225355, Tokens per sec: 169563.42313631237, Loss: 2.4754128456115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18319 , TFLOPS: 70.45618655044024, Tokens per sec: 174598.19166165948, Loss: 2.469794511795044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18320 , TFLOPS: 70.68995373619079, Tokens per sec: 175177.4924995307, Loss: 2.5050625801086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18321 , TFLOPS: 68.05872795390796, Tokens per sec: 168657.02515758591, Loss: 2.474747896194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18322 , TFLOPS: 70.86248712573459, Tokens per sec: 175605.0492449421, Loss: 2.494499683380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18323 , TFLOPS: 70.45847587089837, Tokens per sec: 174603.8648499442, Loss: 2.486360549926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18324 , TFLOPS: 68.69340918521799, Tokens per sec: 170229.83516468148, Loss: 2.503180980682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18325 , TFLOPS: 69.17953264910682, Tokens per sec: 171434.5026591183, Loss: 2.508516311645508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18326 , TFLOPS: 71.51091672009271, Tokens per sec: 177211.92921017812, Loss: 2.4889094829559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18327 , TFLOPS: 68.85842550019984, Tokens per sec: 170638.76377125425, Loss: 2.4975812435150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18328 , TFLOPS: 68.24724395423321, Tokens per sec: 169124.18857314362, Loss: 2.504298686981201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18329 , TFLOPS: 71.49277017231336, Tokens per sec: 177166.9600657736, Loss: 2.490144729614258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18330 , TFLOPS: 69.66331628039181, Tokens per sec: 172633.37179060897, Loss: 2.4797205924987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18331 , TFLOPS: 68.89486906272477, Tokens per sec: 170729.07493378085, Loss: 2.48468017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18332 , TFLOPS: 70.88496908695522, Tokens per sec: 175660.76202143932, Loss: 2.478635311126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18333 , TFLOPS: 71.5329590365282, Tokens per sec: 177266.55249846683, Loss: 2.4854798316955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18334 , TFLOPS: 68.7767310914524, Tokens per sec: 170436.31602699606, Loss: 2.491797924041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18335 , TFLOPS: 68.85009447416545, Tokens per sec: 170618.11857100273, Loss: 2.4943478107452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18336 , TFLOPS: 71.51722822583778, Tokens per sec: 177227.56981668415, Loss: 2.467338800430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18337 , TFLOPS: 69.82860308140312, Tokens per sec: 173042.97069135922, Loss: 2.489985466003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18338 , TFLOPS: 68.10860095330244, Tokens per sec: 168780.61594404984, Loss: 2.495684862136841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18339 , TFLOPS: 71.4803636049576, Tokens per sec: 177136.2152251698, Loss: 2.4802422523498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18340 , TFLOPS: 70.67617965279331, Tokens per sec: 175143.3588035312, Loss: 2.4749021530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18341 , TFLOPS: 68.82704514999219, Tokens per sec: 170560.99980660874, Loss: 2.4902381896972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18342 , TFLOPS: 69.51106897666972, Tokens per sec: 172256.0862005606, Loss: 2.495121955871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18343 , TFLOPS: 71.50248172249901, Tokens per sec: 177191.02635694924, Loss: 2.4815824031829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18344 , TFLOPS: 68.87161350098917, Tokens per sec: 170671.44508998844, Loss: 2.50203275680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18345 , TFLOPS: 68.0966478233237, Tokens per sec: 168750.99477121656, Loss: 2.4864237308502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18346 , TFLOPS: 71.53101549873801, Tokens per sec: 177261.73619772439, Loss: 2.494889974594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18347 , TFLOPS: 69.8095886086378, Tokens per sec: 172995.85073322966, Loss: 2.482994556427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18348 , TFLOPS: 67.86949100962275, Tokens per sec: 168188.07516347684, Loss: 2.4857006072998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18349 , TFLOPS: 70.44247836460293, Tokens per sec: 174564.22126707333, Loss: 2.494826555252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18350 , TFLOPS: 70.60253838307702, Tokens per sec: 174960.8676249187, Loss: 2.4698994159698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18351 , TFLOPS: 68.80322097839829, Tokens per sec: 170501.96088494992, Loss: 2.49686336517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18352 , TFLOPS: 69.60271795874921, Tokens per sec: 172483.2024741224, Loss: 2.4884297847747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18353 , TFLOPS: 71.5335740505847, Tokens per sec: 177268.07657104824, Loss: 2.490596055984497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18354 , TFLOPS: 69.09991234950036, Tokens per sec: 171237.19478579343, Loss: 2.5088253021240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18355 , TFLOPS: 69.07888634433363, Tokens per sec: 171185.09002878462, Loss: 2.481907606124878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18356 , TFLOPS: 70.44601735018456, Tokens per sec: 174572.99126319605, Loss: 2.499300956726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18357 , TFLOPS: 69.73884401235755, Tokens per sec: 172820.53783048657, Loss: 2.4861509799957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18358 , TFLOPS: 67.99916232497696, Tokens per sec: 168509.4149673992, Loss: 2.4688467979431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18359 , TFLOPS: 71.4688331772802, Tokens per sec: 177107.64155520377, Loss: 2.507134199142456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18360 , TFLOPS: 70.54158288492282, Tokens per sec: 174809.81318569015, Loss: 2.484969139099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18361 , TFLOPS: 69.3960062061118, Tokens per sec: 171970.94797990753, Loss: 2.472476005554199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18362 , TFLOPS: 69.30496598866678, Tokens per sec: 171745.34029217093, Loss: 2.4962124824523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18363 , TFLOPS: 71.45541229957702, Tokens per sec: 177074.38314182355, Loss: 2.4772446155548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18364 , TFLOPS: 67.86571638781932, Tokens per sec: 168178.72123483903, Loss: 2.4711952209472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18365 , TFLOPS: 68.81014687787685, Tokens per sec: 170519.12402680857, Loss: 2.490159511566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18366 , TFLOPS: 71.52541010395136, Tokens per sec: 177247.84541195788, Loss: 2.4775707721710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18367 , TFLOPS: 70.31965987684724, Tokens per sec: 174259.86352484138, Loss: 2.4836223125457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18368 , TFLOPS: 67.77225798296975, Tokens per sec: 167947.12101232857, Loss: 2.4762558937072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18369 , TFLOPS: 71.50613165120609, Tokens per sec: 177200.0712823582, Loss: 2.5063014030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18370 , TFLOPS: 70.62627363857042, Tokens per sec: 175019.6862026856, Loss: 2.5029356479644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18371 , TFLOPS: 68.80437403272792, Tokens per sec: 170504.81828059818, Loss: 2.494619369506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18372 , TFLOPS: 69.84798738033939, Tokens per sec: 173091.00711948008, Loss: 2.513646125793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18373 , TFLOPS: 71.53480849928707, Tokens per sec: 177271.13567091848, Loss: 2.4868743419647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18374 , TFLOPS: 68.61599429339694, Tokens per sec: 170037.99253479458, Loss: 2.4682371616363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18375 , TFLOPS: 68.02631745806111, Tokens per sec: 168576.70837856657, Loss: 2.477898120880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18376 , TFLOPS: 71.51274517394137, Tokens per sec: 177216.46032023587, Loss: 2.4905636310577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18377 , TFLOPS: 70.63211713867187, Tokens per sec: 175034.1670396517, Loss: 2.499265193939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18378 , TFLOPS: 67.28479286661906, Tokens per sec: 166739.12875529606, Loss: 2.48833966255188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18379 , TFLOPS: 71.53755132771661, Tokens per sec: 177277.93270191437, Loss: 2.489365339279175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18380 , TFLOPS: 71.51599441276646, Tokens per sec: 177224.51229197768, Loss: 2.4854695796966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18381 , TFLOPS: 69.19711118001243, Tokens per sec: 171478.06419513744, Loss: 2.5035386085510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18382 , TFLOPS: 67.92907113056238, Tokens per sec: 168335.72126645737, Loss: 2.482160806655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18383 , TFLOPS: 71.51627327947561, Tokens per sec: 177225.20335440265, Loss: 2.4802451133728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18384 , TFLOPS: 69.66045651480427, Tokens per sec: 172626.28497645384, Loss: 2.4788765907287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18385 , TFLOPS: 68.30801756859881, Tokens per sec: 169274.7922843079, Loss: 2.4889276027679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18386 , TFLOPS: 70.55374889294684, Tokens per sec: 174839.96189377012, Loss: 2.481245756149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18387 , TFLOPS: 70.60641321380284, Tokens per sec: 174970.46988230513, Loss: 2.4733381271362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18388 , TFLOPS: 68.17970948812513, Tokens per sec: 168956.83072659222, Loss: 2.4924304485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18389 , TFLOPS: 70.8800280868637, Tokens per sec: 175648.51767891645, Loss: 2.5087966918945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18390 , TFLOPS: 69.57775799394821, Tokens per sec: 172421.34893177843, Loss: 2.4615747928619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18391 , TFLOPS: 69.56210385987846, Tokens per sec: 172382.5562631084, Loss: 2.4882171154022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18392 , TFLOPS: 69.47645559323081, Tokens per sec: 172170.31042342004, Loss: 2.50085186958313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18393 , TFLOPS: 70.57091143665922, Tokens per sec: 174882.49256769964, Loss: 2.477471351623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18394 , TFLOPS: 69.9001829602483, Tokens per sec: 173220.35351631217, Loss: 2.4996719360351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18395 , TFLOPS: 68.45299170556564, Tokens per sec: 169634.05416593124, Loss: 2.491429328918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18396 , TFLOPS: 71.48177752291133, Tokens per sec: 177139.71906961506, Loss: 2.492720603942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18397 , TFLOPS: 69.60262585117098, Tokens per sec: 172482.97422139707, Loss: 2.462985038757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18398 , TFLOPS: 68.04333845386638, Tokens per sec: 168618.8882811895, Loss: 2.4973387718200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18399 , TFLOPS: 70.82962445016271, Tokens per sec: 175523.61191475243, Loss: 2.512282609939575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18400 , TFLOPS: 70.51995569561723, Tokens per sec: 174756.21862815932, Loss: 2.474226713180542 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/optimizer.pt +[rank0]:[2024-08-27 16:53:04,512] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006389916001353413, 'preprocessing_with_comm': 0.0012522650067694485, 'state_converting': 0.9764145619992632, : 0.9854109949956182}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400, took 4.30s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016400 + eval ppl=10.802532196044922, eval loss=2.3797805309295654 +------------------------------------------------------------------ +iteration: 18401 , TFLOPS: 63.90666667913062, Tokens per sec: 158367.7599313799, Loss: 2.4732534885406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18402 , TFLOPS: 69.6796132366902, Tokens per sec: 172673.75744357434, Loss: 2.4979076385498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18403 , TFLOPS: 71.50693169358418, Tokens per sec: 177202.05387550304, Loss: 2.5188870429992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18404 , TFLOPS: 68.41968378548543, Tokens per sec: 169551.5134708022, Loss: 2.5042970180511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18405 , TFLOPS: 69.07630887278142, Tokens per sec: 171178.7027703463, Loss: 2.4858100414276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18406 , TFLOPS: 71.4197518517122, Tokens per sec: 176986.01262369193, Loss: 2.4949252605438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18407 , TFLOPS: 71.47473714935761, Tokens per sec: 177122.2722483878, Loss: 2.4952569007873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18408 , TFLOPS: 67.81966293844408, Tokens per sec: 168064.59571408093, Loss: 2.4953808784484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18409 , TFLOPS: 70.1580205386264, Tokens per sec: 173859.30343868714, Loss: 2.503690242767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18410 , TFLOPS: 71.4787790280148, Tokens per sec: 177132.28846895, Loss: 2.470996379852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18411 , TFLOPS: 67.89277684585673, Tokens per sec: 168245.78003081126, Loss: 2.496145725250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18412 , TFLOPS: 69.23951692088305, Tokens per sec: 171583.15029239282, Loss: 2.469191312789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18413 , TFLOPS: 71.48359715154214, Tokens per sec: 177144.22830981086, Loss: 2.4852283000946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18414 , TFLOPS: 69.82913446734024, Tokens per sec: 173044.28752424842, Loss: 2.4881491661071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18415 , TFLOPS: 67.9324159741649, Tokens per sec: 168344.01015736937, Loss: 2.4832208156585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18416 , TFLOPS: 71.47986823611329, Tokens per sec: 177134.98764660425, Loss: 2.4826765060424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18417 , TFLOPS: 71.5289432761983, Tokens per sec: 177256.60100199684, Loss: 2.4991536140441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18418 , TFLOPS: 68.33473672401969, Tokens per sec: 169341.0052947988, Loss: 2.4924399852752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18419 , TFLOPS: 69.8932850222462, Tokens per sec: 173203.25966607238, Loss: 2.4773480892181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18420 , TFLOPS: 71.51932071498126, Tokens per sec: 177232.7552352884, Loss: 2.499133348464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18421 , TFLOPS: 69.03415862118663, Tokens per sec: 171074.2498036609, Loss: 2.513157844543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18422 , TFLOPS: 69.62677568644806, Tokens per sec: 172542.82017353762, Loss: 2.4944489002227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18423 , TFLOPS: 70.41243087790491, Tokens per sec: 174489.76028503172, Loss: 2.5055038928985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18424 , TFLOPS: 69.77354246316898, Tokens per sec: 172906.52441394722, Loss: 2.477890968322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18425 , TFLOPS: 68.11491688343217, Tokens per sec: 168796.26751466846, Loss: 2.4905200004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18426 , TFLOPS: 70.45999910402925, Tokens per sec: 174607.6395894405, Loss: 2.488055944442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18427 , TFLOPS: 71.46231041922036, Tokens per sec: 177091.47744219037, Loss: 2.4802558422088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18428 , TFLOPS: 69.73571986800279, Tokens per sec: 172812.79585662257, Loss: 2.501812219619751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18429 , TFLOPS: 69.12097844702718, Tokens per sec: 171289.39889608714, Loss: 2.480646848678589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18430 , TFLOPS: 70.4345908072301, Tokens per sec: 174544.67503101728, Loss: 2.485135078430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18431 , TFLOPS: 68.4306074722383, Tokens per sec: 169578.58356991922, Loss: 2.4827182292938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18432 , TFLOPS: 68.83028377621707, Tokens per sec: 170569.02547915812, Loss: 2.4698734283447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18433 , TFLOPS: 71.51175780510466, Tokens per sec: 177214.0135114909, Loss: 2.4885737895965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18434 , TFLOPS: 69.76248595987579, Tokens per sec: 172879.1251808123, Loss: 2.462083339691162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18435 , TFLOPS: 68.59441507065075, Tokens per sec: 169984.5168436819, Loss: 2.4844536781311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18436 , TFLOPS: 70.91334917437852, Tokens per sec: 175731.09100439897, Loss: 2.496624231338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18437 , TFLOPS: 71.54198443556152, Tokens per sec: 177288.91843150117, Loss: 2.4940834045410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18438 , TFLOPS: 67.09188416105764, Tokens per sec: 166261.07973226666, Loss: 2.5183959007263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18439 , TFLOPS: 69.7885648410341, Tokens per sec: 172943.75152114854, Loss: 2.471038341522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18440 , TFLOPS: 71.50091221383668, Tokens per sec: 177187.1369415883, Loss: 2.4967994689941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18441 , TFLOPS: 68.15819363441405, Tokens per sec: 168903.5120709292, Loss: 2.4724154472351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18442 , TFLOPS: 69.67480514258341, Tokens per sec: 172661.84245672243, Loss: 2.4972915649414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18443 , TFLOPS: 71.4426869986411, Tokens per sec: 177042.8484442967, Loss: 2.5002074241638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18444 , TFLOPS: 70.56313626584111, Tokens per sec: 174863.2248379066, Loss: 2.5039467811584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18445 , TFLOPS: 67.30284724626739, Tokens per sec: 166783.86949691875, Loss: 2.499715566635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18446 , TFLOPS: 70.96067408539396, Tokens per sec: 175848.36734716455, Loss: 2.465062379837036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18447 , TFLOPS: 71.54658100859947, Tokens per sec: 177300.30924584356, Loss: 2.474891185760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18448 , TFLOPS: 68.69381950877204, Tokens per sec: 170230.8519916493, Loss: 2.4811038970947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18449 , TFLOPS: 68.85376943616838, Tokens per sec: 170627.2255316765, Loss: 2.4710311889648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18450 , TFLOPS: 71.51539182408224, Tokens per sec: 177223.01901083757, Loss: 2.4611053466796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18451 , TFLOPS: 69.04928836168796, Tokens per sec: 171111.7429673017, Loss: 2.471153497695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18452 , TFLOPS: 67.71926681109197, Tokens per sec: 167815.803050956, Loss: 2.4887051582336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18453 , TFLOPS: 71.51815214748717, Tokens per sec: 177229.85939630968, Loss: 2.481795072555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18454 , TFLOPS: 71.56188994303926, Tokens per sec: 177338.2464718037, Loss: 2.458711862564087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18455 , TFLOPS: 68.13717175020649, Tokens per sec: 168851.41752611115, Loss: 2.50978946685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18456 , TFLOPS: 69.45512205150659, Tokens per sec: 172117.4435569438, Loss: 2.51297664642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18457 , TFLOPS: 71.5277732152528, Tokens per sec: 177253.701462081, Loss: 2.4961934089660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18458 , TFLOPS: 68.15085646743796, Tokens per sec: 168885.3297629068, Loss: 2.502133846282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18459 , TFLOPS: 69.70017969425058, Tokens per sec: 172724.72339099128, Loss: 2.490741491317749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18460 , TFLOPS: 70.4452816635056, Tokens per sec: 174571.16815056297, Loss: 2.485161542892456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18461 , TFLOPS: 68.87974530159585, Tokens per sec: 170691.59658768354, Loss: 2.5017788410186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18462 , TFLOPS: 69.51724667065878, Tokens per sec: 172271.3952067947, Loss: 2.469771146774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18463 , TFLOPS: 70.53844266286735, Tokens per sec: 174802.0313695127, Loss: 2.5024123191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18464 , TFLOPS: 71.5745442555411, Tokens per sec: 177369.60525217085, Loss: 2.4764578342437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18465 , TFLOPS: 67.55265269411386, Tokens per sec: 167402.91491502407, Loss: 2.4681615829467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18466 , TFLOPS: 70.61446400159487, Tokens per sec: 174990.42062132127, Loss: 2.4646058082580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18467 , TFLOPS: 70.41847243748335, Tokens per sec: 174504.73193519982, Loss: 2.496617317199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18468 , TFLOPS: 69.69171094060458, Tokens per sec: 172703.73688654628, Loss: 2.5106992721557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18469 , TFLOPS: 68.84070514219556, Tokens per sec: 170594.8507720611, Loss: 2.4961235523223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18470 , TFLOPS: 71.55221448773386, Tokens per sec: 177314.26962771756, Loss: 2.501499652862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18471 , TFLOPS: 68.94264713265106, Tokens per sec: 170847.4742542471, Loss: 2.4945802688598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18472 , TFLOPS: 68.16684563672288, Tokens per sec: 168924.95268574727, Loss: 2.496014356613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18473 , TFLOPS: 71.52607172903699, Tokens per sec: 177249.48499180409, Loss: 2.4883599281311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18474 , TFLOPS: 70.21253563504634, Tokens per sec: 173994.39785294933, Loss: 2.492617130279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18475 , TFLOPS: 67.96261492091158, Tokens per sec: 168418.84647409624, Loss: 2.476003646850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18476 , TFLOPS: 70.58721574131881, Tokens per sec: 174922.89643071353, Loss: 2.480823040008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18477 , TFLOPS: 71.52695323482138, Tokens per sec: 177251.66946024366, Loss: 2.489778518676758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18478 , TFLOPS: 68.93517009088679, Tokens per sec: 170828.94532109107, Loss: 2.5199055671691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18479 , TFLOPS: 69.17011636947893, Tokens per sec: 171411.1680809116, Loss: 2.502749443054199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18480 , TFLOPS: 71.50205076545862, Tokens per sec: 177189.9583979293, Loss: 2.466959238052368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18481 , TFLOPS: 69.87532130078117, Tokens per sec: 173158.74358541475, Loss: 2.4737296104431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18482 , TFLOPS: 68.0138556439792, Tokens per sec: 168545.82663048944, Loss: 2.505392551422119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18483 , TFLOPS: 71.48709201452913, Tokens per sec: 177152.88896528355, Loss: 2.5109386444091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18484 , TFLOPS: 71.56844322805652, Tokens per sec: 177354.4862339804, Loss: 2.514141798019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18485 , TFLOPS: 67.96256508837578, Tokens per sec: 168418.72298358305, Loss: 2.4757540225982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18486 , TFLOPS: 70.1681782793117, Tokens per sec: 173884.47543907282, Loss: 2.480288505554199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18487 , TFLOPS: 71.50296310270353, Tokens per sec: 177192.21927012442, Loss: 2.4755373001098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18488 , TFLOPS: 69.65040085396622, Tokens per sec: 172601.36594117468, Loss: 2.47603702545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18489 , TFLOPS: 67.5579127076152, Tokens per sec: 167415.9498079187, Loss: 2.5115182399749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18490 , TFLOPS: 71.47174563255783, Tokens per sec: 177114.85894021517, Loss: 2.4841184616088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18491 , TFLOPS: 69.92445749133886, Tokens per sec: 173280.5084784149, Loss: 2.5055153369903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18492 , TFLOPS: 68.01533690827733, Tokens per sec: 168549.49736659494, Loss: 2.4813523292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18493 , TFLOPS: 70.19181946430201, Tokens per sec: 173943.06089977032, Loss: 2.500833511352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18494 , TFLOPS: 71.5611323419095, Tokens per sec: 177336.3690527475, Loss: 2.506864547729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18495 , TFLOPS: 68.93064308825636, Tokens per sec: 170817.72690988236, Loss: 2.4949822425842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18496 , TFLOPS: 69.82229001953057, Tokens per sec: 173027.32622859787, Loss: 2.4929025173187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18497 , TFLOPS: 70.465475349692, Tokens per sec: 174621.21032945107, Loss: 2.4861347675323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18498 , TFLOPS: 69.11480942691125, Tokens per sec: 171274.11138466618, Loss: 2.474855899810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18499 , TFLOPS: 68.9219069039819, Tokens per sec: 170796.07768288013, Loss: 2.482847213745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18500 , TFLOPS: 69.93828625682968, Tokens per sec: 173314.7776826655, Loss: 2.491680383682251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18501 , TFLOPS: 70.66826115420824, Tokens per sec: 175123.73589174278, Loss: 2.4878530502319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18502 , TFLOPS: 68.97097298197141, Tokens per sec: 170917.66882921284, Loss: 2.506706953048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18503 , TFLOPS: 70.90878366805832, Tokens per sec: 175719.77717680548, Loss: 2.501951217651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18504 , TFLOPS: 70.42472121863447, Tokens per sec: 174520.21710325233, Loss: 2.4912524223327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18505 , TFLOPS: 68.11400901019319, Tokens per sec: 168794.01770477177, Loss: 2.4920196533203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18506 , TFLOPS: 69.73336768073025, Tokens per sec: 172806.96687744645, Loss: 2.4971117973327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18507 , TFLOPS: 71.48997165964732, Tokens per sec: 177160.02504310553, Loss: 2.482042074203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18508 , TFLOPS: 68.65149951256399, Tokens per sec: 170125.97837911334, Loss: 2.4945907592773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18509 , TFLOPS: 68.96409418178663, Tokens per sec: 170900.62240459255, Loss: 2.4664652347564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18510 , TFLOPS: 71.50810290773072, Tokens per sec: 177204.95627317808, Loss: 2.510979175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18511 , TFLOPS: 69.84042118728767, Tokens per sec: 173072.257259613, Loss: 2.494701862335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18512 , TFLOPS: 68.0952762343276, Tokens per sec: 168747.59582259122, Loss: 2.4903578758239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18513 , TFLOPS: 70.80717643968958, Tokens per sec: 175467.98327194754, Loss: 2.4676764011383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18514 , TFLOPS: 71.5212195881339, Tokens per sec: 177237.46085213879, Loss: 2.485196590423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18515 , TFLOPS: 68.72057871986398, Tokens per sec: 170297.16426450558, Loss: 2.4879748821258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18516 , TFLOPS: 69.65501733943387, Tokens per sec: 172612.8061007114, Loss: 2.5022213459014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18517 , TFLOPS: 71.57802205361291, Tokens per sec: 177378.2236189039, Loss: 2.4952375888824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18518 , TFLOPS: 69.07179484567463, Tokens per sec: 171167.51651391454, Loss: 2.4846434593200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18519 , TFLOPS: 67.95333478064448, Tokens per sec: 168395.84926422252, Loss: 2.479696273803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18520 , TFLOPS: 71.4808802891404, Tokens per sec: 177137.49562549224, Loss: 2.4937331676483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18521 , TFLOPS: 71.48558842033358, Tokens per sec: 177149.16289323242, Loss: 2.4961893558502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18522 , TFLOPS: 67.95127285079812, Tokens per sec: 168390.73957492292, Loss: 2.5054001808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18523 , TFLOPS: 70.54065515020756, Tokens per sec: 174807.5141568673, Loss: 2.4752695560455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18524 , TFLOPS: 71.45215924812794, Tokens per sec: 177066.32172197953, Loss: 2.4929652214050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18525 , TFLOPS: 69.01573695641417, Tokens per sec: 171028.59888904085, Loss: 2.49548602104187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18526 , TFLOPS: 68.28859157253292, Tokens per sec: 169226.6525261065, Loss: 2.5195066928863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18527 , TFLOPS: 71.55537022683642, Tokens per sec: 177322.08989684706, Loss: 2.4677894115448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18528 , TFLOPS: 69.80902024131629, Tokens per sec: 172994.44225639614, Loss: 2.4923436641693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18529 , TFLOPS: 68.87241673488383, Tokens per sec: 170673.43559206292, Loss: 2.471625566482544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18530 , TFLOPS: 70.4053104434803, Tokens per sec: 174472.11506414693, Loss: 2.5213217735290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18531 , TFLOPS: 70.67382066262368, Tokens per sec: 175137.51296602667, Loss: 2.493269205093384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18532 , TFLOPS: 69.1645603233214, Tokens per sec: 171397.3995864842, Loss: 2.494896650314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18533 , TFLOPS: 70.12147234611835, Tokens per sec: 173768.7329915364, Loss: 2.4865241050720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18534 , TFLOPS: 70.40891453614665, Tokens per sec: 174481.0464027977, Loss: 2.4975287914276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18535 , TFLOPS: 69.00529817607645, Tokens per sec: 171002.7304414374, Loss: 2.477442502975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18536 , TFLOPS: 69.49385499874568, Tokens per sec: 172213.42806698862, Loss: 2.4875059127807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18537 , TFLOPS: 70.47531475951337, Tokens per sec: 174645.5934708889, Loss: 2.487818717956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18538 , TFLOPS: 69.81462987309979, Tokens per sec: 173008.34354190764, Loss: 2.5060946941375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18539 , TFLOPS: 68.58529552385639, Tokens per sec: 169961.91760212448, Loss: 2.4615397453308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18540 , TFLOPS: 71.5099581754536, Tokens per sec: 177209.55383096996, Loss: 2.509636640548706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18541 , TFLOPS: 70.5591623205728, Tokens per sec: 174853.37696376548, Loss: 2.475865364074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18542 , TFLOPS: 67.23743089719382, Tokens per sec: 166621.7605777087, Loss: 2.5083799362182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18543 , TFLOPS: 70.60864138001382, Tokens per sec: 174975.99152362873, Loss: 2.505589485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18544 , TFLOPS: 71.48987644903679, Tokens per sec: 177159.78910072433, Loss: 2.4878580570220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18545 , TFLOPS: 67.88415675626803, Tokens per sec: 168224.41849922918, Loss: 2.498490333557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18546 , TFLOPS: 68.96997966552694, Tokens per sec: 170915.20728164056, Loss: 2.4701087474823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18547 , TFLOPS: 71.5176996609657, Tokens per sec: 177228.738084864, Loss: 2.463559150695801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18548 , TFLOPS: 70.4918559313427, Tokens per sec: 174686.58431684246, Loss: 2.504650115966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18549 , TFLOPS: 67.9816995341214, Tokens per sec: 168466.14024797388, Loss: 2.4883391857147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18550 , TFLOPS: 71.51767859061499, Tokens per sec: 177228.6858702138, Loss: 2.490226984024048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18551 , TFLOPS: 71.55700618096841, Tokens per sec: 177326.14397140132, Loss: 2.488173484802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18552 , TFLOPS: 68.2353614724573, Tokens per sec: 169094.74247433952, Loss: 2.4853157997131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18553 , TFLOPS: 69.71890062742517, Tokens per sec: 172771.11592567893, Loss: 2.5254642963409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18554 , TFLOPS: 71.54027804851336, Tokens per sec: 177284.6898164218, Loss: 2.5017356872558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18555 , TFLOPS: 69.01500234832255, Tokens per sec: 171026.77844926607, Loss: 2.5023300647735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18556 , TFLOPS: 68.6062946832661, Tokens per sec: 170013.95583239011, Loss: 2.478031873703003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18557 , TFLOPS: 71.52153356814428, Tokens per sec: 177238.23892919224, Loss: 2.493272304534912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18558 , TFLOPS: 69.79008340436326, Tokens per sec: 172947.51468836173, Loss: 2.5034337043762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18559 , TFLOPS: 67.1737228684145, Tokens per sec: 166463.8850643748, Loss: 2.5201327800750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18560 , TFLOPS: 71.55191229696717, Tokens per sec: 177313.5207657086, Loss: 2.4959816932678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18561 , TFLOPS: 71.48028584892509, Tokens per sec: 177136.02253715572, Loss: 2.4768831729888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18562 , TFLOPS: 69.02960704307587, Tokens per sec: 171062.9704917047, Loss: 2.480954885482788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18563 , TFLOPS: 68.6923656659769, Tokens per sec: 170227.24920904892, Loss: 2.4802420139312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18564 , TFLOPS: 71.55263717216968, Tokens per sec: 177315.31708631202, Loss: 2.4801056385040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18565 , TFLOPS: 68.92977878053316, Tokens per sec: 170815.58506013293, Loss: 2.4986870288848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18566 , TFLOPS: 68.31239754295729, Tokens per sec: 169285.6463432622, Loss: 2.493527412414551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18567 , TFLOPS: 70.5416092488379, Tokens per sec: 174809.87851837595, Loss: 2.483707904815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18568 , TFLOPS: 71.55195263068705, Tokens per sec: 177313.6207171096, Loss: 2.494647264480591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18569 , TFLOPS: 68.3155152886852, Tokens per sec: 169293.37246062403, Loss: 2.4517154693603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18570 , TFLOPS: 70.91101499430381, Tokens per sec: 175725.3066490427, Loss: 2.4897232055664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18571 , TFLOPS: 70.43743200326466, Tokens per sec: 174551.71582777446, Loss: 2.4771084785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18572 , TFLOPS: 68.07208029541741, Tokens per sec: 168690.11373072816, Loss: 2.490060329437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18573 , TFLOPS: 69.60061012256102, Tokens per sec: 172477.97902385076, Loss: 2.4771957397460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18574 , TFLOPS: 70.49049684893583, Tokens per sec: 174683.21636092296, Loss: 2.5000722408294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18575 , TFLOPS: 69.01664075867649, Tokens per sec: 171030.83861060892, Loss: 2.498478412628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18576 , TFLOPS: 69.0809961929684, Tokens per sec: 171190.31846612057, Loss: 2.489056348800659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18577 , TFLOPS: 71.49114993533048, Tokens per sec: 177162.94494004617, Loss: 2.530318260192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18578 , TFLOPS: 68.77207457136811, Tokens per sec: 170424.77665726916, Loss: 2.489753007888794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18579 , TFLOPS: 68.83068808548802, Tokens per sec: 170570.02740206989, Loss: 2.5118257999420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18580 , TFLOPS: 70.8804956186687, Tokens per sec: 175649.67627423213, Loss: 2.4872422218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18581 , TFLOPS: 71.43365626456328, Tokens per sec: 177020.46929043438, Loss: 2.512097120285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18582 , TFLOPS: 68.03458853806066, Tokens per sec: 168597.2050259401, Loss: 2.497393846511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18583 , TFLOPS: 69.61076880938782, Tokens per sec: 172503.1533688793, Loss: 2.487307548522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18584 , TFLOPS: 71.51290010241274, Tokens per sec: 177216.8442500547, Loss: 2.5045230388641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18585 , TFLOPS: 68.91705648836519, Tokens per sec: 170784.0578186645, Loss: 2.484771728515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18586 , TFLOPS: 67.89879856519444, Tokens per sec: 168260.70251467725, Loss: 2.460391044616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18587 , TFLOPS: 71.52401733041826, Tokens per sec: 177244.39396571615, Loss: 2.481926441192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18588 , TFLOPS: 71.51949415424104, Tokens per sec: 177233.18503688002, Loss: 2.4781055450439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18589 , TFLOPS: 66.20968606063983, Tokens per sec: 164074.89565728797, Loss: 2.4958386421203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18590 , TFLOPS: 70.51585364912192, Tokens per sec: 174746.05330506622, Loss: 2.49432373046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18591 , TFLOPS: 71.50611519422036, Tokens per sec: 177200.03050013466, Loss: 2.5159387588500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18592 , TFLOPS: 67.97389336113645, Tokens per sec: 168446.7956913962, Loss: 2.4880542755126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18593 , TFLOPS: 69.59967667868766, Tokens per sec: 172475.66584710524, Loss: 2.465482473373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18594 , TFLOPS: 71.51718746733803, Tokens per sec: 177227.46881263185, Loss: 2.482347011566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18595 , TFLOPS: 68.88481093683826, Tokens per sec: 170704.14978984007, Loss: 2.4812204837799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18596 , TFLOPS: 69.32837490705481, Tokens per sec: 171803.3501706417, Loss: 2.4901325702667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18597 , TFLOPS: 71.51540020905993, Tokens per sec: 177223.03978973595, Loss: 2.492968797683716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18598 , TFLOPS: 70.32607965433424, Tokens per sec: 174275.77244065102, Loss: 2.48225736618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18599 , TFLOPS: 68.34425716823979, Tokens per sec: 169364.59800433973, Loss: 2.4954469203948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18600 , TFLOPS: 69.55839575005616, Tokens per sec: 172373.3671584867, Loss: 2.5070858001708984 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/optimizer.pt +[rank0]:[2024-08-27 17:18:28,446] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006349023984512314, 'preprocessing_with_comm': 0.001212214003317058, 'state_converting': 0.9920621319906786, : 1.0010274040105287}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016600 + eval ppl=9.695931434631348, eval loss=2.2717063426971436 +------------------------------------------------------------------ +iteration: 18601 , TFLOPS: 66.89322258260485, Tokens per sec: 165768.77445648116, Loss: 2.495925188064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18602 , TFLOPS: 69.09810279474105, Tokens per sec: 171232.71050976097, Loss: 2.462923049926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18603 , TFLOPS: 69.16838525089524, Tokens per sec: 171406.87817836273, Loss: 2.481050968170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18604 , TFLOPS: 69.9993066045731, Tokens per sec: 173465.99282631988, Loss: 2.4937732219696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18605 , TFLOPS: 69.73636410019954, Tokens per sec: 172814.39233497417, Loss: 2.496007204055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18606 , TFLOPS: 68.19325947518837, Tokens per sec: 168990.40908718927, Loss: 2.5010275840759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18607 , TFLOPS: 71.37823441568601, Tokens per sec: 176883.12784369587, Loss: 2.4855599403381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18608 , TFLOPS: 69.83692225141179, Tokens per sec: 173063.58651107294, Loss: 2.4784297943115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18609 , TFLOPS: 68.49069115546905, Tokens per sec: 169727.47755572916, Loss: 2.505354166030884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18610 , TFLOPS: 70.88879841217502, Tokens per sec: 175670.251511169, Loss: 2.514705181121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18611 , TFLOPS: 70.39962573898363, Tokens per sec: 174458.0277401823, Loss: 2.506282329559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18612 , TFLOPS: 69.71148689512907, Tokens per sec: 172752.7438803592, Loss: 2.49100923538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18613 , TFLOPS: 68.86709582730495, Tokens per sec: 170660.2497969361, Loss: 2.502105712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18614 , TFLOPS: 71.4988151529693, Tokens per sec: 177181.94019374863, Loss: 2.470790147781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18615 , TFLOPS: 69.50809133875158, Tokens per sec: 172248.70728578558, Loss: 2.4945144653320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18616 , TFLOPS: 67.51539810633318, Tokens per sec: 167310.59394257193, Loss: 2.504272937774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18617 , TFLOPS: 71.4975854028775, Tokens per sec: 177178.89273755302, Loss: 2.5048696994781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18618 , TFLOPS: 71.52722926855672, Tokens per sec: 177252.35350224748, Loss: 2.481050491333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18619 , TFLOPS: 67.70165055985711, Tokens per sec: 167772.14803980573, Loss: 2.481842041015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18620 , TFLOPS: 70.62651928595032, Tokens per sec: 175020.29494395325, Loss: 2.477489709854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18621 , TFLOPS: 71.52898667671279, Tokens per sec: 177256.70855325268, Loss: 2.5009922981262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18622 , TFLOPS: 67.80570675460177, Tokens per sec: 168030.0107531191, Loss: 2.501864433288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18623 , TFLOPS: 69.77893455563385, Tokens per sec: 172919.8866130632, Loss: 2.494508981704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18624 , TFLOPS: 71.48124715952753, Tokens per sec: 177138.40477072567, Loss: 2.491037130355835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18625 , TFLOPS: 70.66227649625056, Tokens per sec: 175108.90525005912, Loss: 2.4897217750549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18626 , TFLOPS: 67.15194204701919, Tokens per sec: 166409.9097896017, Loss: 2.4709956645965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18627 , TFLOPS: 71.49257703323188, Tokens per sec: 177166.4814458537, Loss: 2.4931983947753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18628 , TFLOPS: 71.56307973167793, Tokens per sec: 177341.19489911635, Loss: 2.503632068634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18629 , TFLOPS: 67.28335467532384, Tokens per sec: 166735.56475883664, Loss: 2.5018105506896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18630 , TFLOPS: 69.63698084934475, Tokens per sec: 172568.10969138713, Loss: 2.4928550720214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18631 , TFLOPS: 71.55810913856648, Tokens per sec: 177328.87722183013, Loss: 2.497976303100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18632 , TFLOPS: 69.25831250939089, Tokens per sec: 171629.72783122028, Loss: 2.4777910709381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18633 , TFLOPS: 68.8596430709746, Tokens per sec: 170641.7810457606, Loss: 2.512669563293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18634 , TFLOPS: 70.47008219153989, Tokens per sec: 174632.62659103566, Loss: 2.4819893836975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18635 , TFLOPS: 70.62457038833135, Tokens per sec: 175015.46536096468, Loss: 2.489555597305298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18636 , TFLOPS: 67.31280369435866, Tokens per sec: 166808.5426720822, Loss: 2.4880285263061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18637 , TFLOPS: 70.38024628903402, Tokens per sec: 174410.0033283845, Loss: 2.489015817642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18638 , TFLOPS: 71.51715611826778, Tokens per sec: 177227.39112618237, Loss: 2.465317964553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18639 , TFLOPS: 68.6369081137353, Tokens per sec: 170089.81928544055, Loss: 2.5046114921569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18640 , TFLOPS: 70.52938593224631, Tokens per sec: 174779.58779335252, Loss: 2.484917640686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18641 , TFLOPS: 70.38128623811949, Tokens per sec: 174412.58043677814, Loss: 2.4910717010498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18642 , TFLOPS: 68.96360606313318, Tokens per sec: 170899.4127927986, Loss: 2.4861488342285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18643 , TFLOPS: 69.1757258306609, Tokens per sec: 171425.06894365363, Loss: 2.489964485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18644 , TFLOPS: 71.4800557090586, Tokens per sec: 177135.45222521457, Loss: 2.4757978916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18645 , TFLOPS: 70.42596662255825, Tokens per sec: 174523.30335136806, Loss: 2.4998888969421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18646 , TFLOPS: 67.27105080766397, Tokens per sec: 166705.07441939256, Loss: 2.477888345718384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18647 , TFLOPS: 71.51285690527541, Tokens per sec: 177216.7372027899, Loss: 2.4781041145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18648 , TFLOPS: 71.53294764132168, Tokens per sec: 177266.52425988982, Loss: 2.493847608566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18649 , TFLOPS: 67.17840349203561, Tokens per sec: 166475.4841653212, Loss: 2.5055737495422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18650 , TFLOPS: 69.90508709857214, Tokens per sec: 173232.50651131407, Loss: 2.4954335689544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18651 , TFLOPS: 71.52453294562453, Tokens per sec: 177245.67171699702, Loss: 2.4883785247802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18652 , TFLOPS: 69.46994838211128, Tokens per sec: 172154.18483743738, Loss: 2.5029687881469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18653 , TFLOPS: 68.20436946999662, Tokens per sec: 169017.94087819103, Loss: 2.505849838256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18654 , TFLOPS: 71.52626737775336, Tokens per sec: 177249.96983087572, Loss: 2.4711427688598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18655 , TFLOPS: 70.66798408849154, Tokens per sec: 175123.04929237548, Loss: 2.4863696098327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18656 , TFLOPS: 67.31068893671986, Tokens per sec: 166803.30206969345, Loss: 2.4835567474365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18657 , TFLOPS: 71.50940099696413, Tokens per sec: 177208.1730812948, Loss: 2.471529960632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18658 , TFLOPS: 71.47150931496105, Tokens per sec: 177114.2733191778, Loss: 2.480027437210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18659 , TFLOPS: 68.54547669002837, Tokens per sec: 169863.24214549197, Loss: 2.4786536693573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18660 , TFLOPS: 69.91742821876386, Tokens per sec: 173263.0891666358, Loss: 2.490647077560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18661 , TFLOPS: 71.5050001449757, Tokens per sec: 177197.26728527359, Loss: 2.487124443054199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18662 , TFLOPS: 69.807679739759, Tokens per sec: 172991.12034586875, Loss: 2.5048813819885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18663 , TFLOPS: 67.3008512180844, Tokens per sec: 166778.92311920066, Loss: 2.5029492378234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18664 , TFLOPS: 71.5106660701046, Tokens per sec: 177211.3080718962, Loss: 2.4977869987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18665 , TFLOPS: 71.47597564722714, Tokens per sec: 177125.3413825403, Loss: 2.5077102184295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18666 , TFLOPS: 68.0989559167662, Tokens per sec: 168756.71448102454, Loss: 2.5008535385131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18667 , TFLOPS: 69.83018146755238, Tokens per sec: 173046.8821061107, Loss: 2.48915696144104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18668 , TFLOPS: 71.4840421899811, Tokens per sec: 177145.3311640876, Loss: 2.502574920654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18669 , TFLOPS: 68.02161983838427, Tokens per sec: 168565.0671595218, Loss: 2.4828760623931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18670 , TFLOPS: 69.36850124152662, Tokens per sec: 171902.78764774348, Loss: 2.4957785606384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18671 , TFLOPS: 70.52237440883253, Tokens per sec: 174762.2124659501, Loss: 2.4854273796081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18672 , TFLOPS: 70.58095885848041, Tokens per sec: 174907.39118578745, Loss: 2.4824624061584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18673 , TFLOPS: 68.15034194853745, Tokens per sec: 168884.0547283924, Loss: 2.5064220428466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18674 , TFLOPS: 70.41221169240609, Tokens per sec: 174489.21711922035, Loss: 2.4866583347320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18675 , TFLOPS: 70.54496014773132, Tokens per sec: 174818.18241496582, Loss: 2.4825844764709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18676 , TFLOPS: 68.05493167675797, Tokens per sec: 168647.61756461379, Loss: 2.4724338054656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18677 , TFLOPS: 70.86862307211706, Tokens per sec: 175620.2547960082, Loss: 2.4865832328796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18678 , TFLOPS: 70.47163741593742, Tokens per sec: 174636.48060841486, Loss: 2.5112173557281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18679 , TFLOPS: 69.5469719375581, Tokens per sec: 172345.05769267966, Loss: 2.4813952445983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18680 , TFLOPS: 69.54114994130624, Tokens per sec: 172330.6301446217, Loss: 2.4922189712524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18681 , TFLOPS: 71.52831295681212, Tokens per sec: 177255.03900112244, Loss: 2.50628924369812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18682 , TFLOPS: 68.62725114315543, Tokens per sec: 170065.88824854052, Loss: 2.4908812046051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18683 , TFLOPS: 67.9790046976012, Tokens per sec: 168459.46214621604, Loss: 2.4807536602020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18684 , TFLOPS: 71.49460589491312, Tokens per sec: 177171.50918859735, Loss: 2.525409460067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18685 , TFLOPS: 70.48270191908144, Tokens per sec: 174663.89966606136, Loss: 2.497378349304199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18686 , TFLOPS: 67.89641895873342, Tokens per sec: 168254.80558773156, Loss: 2.4673798084259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18687 , TFLOPS: 70.54254366306368, Tokens per sec: 174812.19409976166, Loss: 2.5007100105285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18688 , TFLOPS: 71.48646925665675, Tokens per sec: 177151.34570267334, Loss: 2.5037379264831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18689 , TFLOPS: 68.76085409222007, Tokens per sec: 170396.9710739025, Loss: 2.497051954269409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18690 , TFLOPS: 69.17746583138845, Tokens per sec: 171429.3808571334, Loss: 2.488637685775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18691 , TFLOPS: 71.55860756540918, Tokens per sec: 177330.1123784526, Loss: 2.487419366836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18692 , TFLOPS: 70.54351950683217, Tokens per sec: 174814.61234811807, Loss: 2.4739882946014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18693 , TFLOPS: 67.0130694221024, Tokens per sec: 166065.76812697537, Loss: 2.4957287311553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18694 , TFLOPS: 71.50863282951511, Tokens per sec: 177206.26947773548, Loss: 2.4833314418792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18695 , TFLOPS: 71.51455108795007, Tokens per sec: 177220.93557408729, Loss: 2.492950916290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18696 , TFLOPS: 67.33244405175701, Tokens per sec: 166857.2136412793, Loss: 2.4962716102600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18697 , TFLOPS: 70.08868164891189, Tokens per sec: 173687.47403166528, Loss: 2.473073959350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18698 , TFLOPS: 71.46520234532635, Tokens per sec: 177098.64395365294, Loss: 2.499884605407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18699 , TFLOPS: 69.59701471913063, Tokens per sec: 172469.06921808384, Loss: 2.5085620880126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18700 , TFLOPS: 68.44947370197644, Tokens per sec: 169625.33616549714, Loss: 2.5012998580932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18701 , TFLOPS: 71.52785225333741, Tokens per sec: 177253.89732716145, Loss: 2.4767203330993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18702 , TFLOPS: 70.71414688775185, Tokens per sec: 175237.4457941947, Loss: 2.49080753326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18703 , TFLOPS: 67.45935992329139, Tokens per sec: 167171.7251518789, Loss: 2.4883129596710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18704 , TFLOPS: 70.5856702990691, Tokens per sec: 174919.0666545742, Loss: 2.472076177597046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18705 , TFLOPS: 71.46907767777421, Tokens per sec: 177108.24745436187, Loss: 2.4864797592163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18706 , TFLOPS: 67.91254996539627, Tokens per sec: 168294.77999921917, Loss: 2.4988934993743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18707 , TFLOPS: 70.63886969674964, Tokens per sec: 175050.90062242406, Loss: 2.478087902069092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18708 , TFLOPS: 70.3303812499065, Tokens per sec: 174286.43226833895, Loss: 2.5070056915283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18709 , TFLOPS: 68.90623798361723, Tokens per sec: 170757.2483721424, Loss: 2.4938840866088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18710 , TFLOPS: 68.86285425007668, Tokens per sec: 170649.73870131612, Loss: 2.4914958477020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18711 , TFLOPS: 70.52417520602462, Tokens per sec: 174766.6750397663, Loss: 2.4905357360839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18712 , TFLOPS: 71.52585079310222, Tokens per sec: 177248.9374882196, Loss: 2.490485429763794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18713 , TFLOPS: 67.40542635568147, Tokens per sec: 167038.07183006857, Loss: 2.4840831756591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18714 , TFLOPS: 71.5060868781039, Tokens per sec: 177199.9603296788, Loss: 2.4789905548095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18715 , TFLOPS: 70.45040109083631, Tokens per sec: 174583.85465543997, Loss: 2.495711326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18716 , TFLOPS: 68.00857209377543, Tokens per sec: 168532.73341105337, Loss: 2.506883144378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18717 , TFLOPS: 69.7295707034806, Tokens per sec: 172797.5575495502, Loss: 2.471736431121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18718 , TFLOPS: 71.50189446444676, Tokens per sec: 177189.5710668039, Loss: 2.4650321006774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18719 , TFLOPS: 69.48284707372623, Tokens per sec: 172186.1492161674, Loss: 2.4822752475738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18720 , TFLOPS: 69.10684195062274, Tokens per sec: 171254.3671007333, Loss: 2.49252986907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18721 , TFLOPS: 71.53873340525496, Tokens per sec: 177280.862020271, Loss: 2.4705452919006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18722 , TFLOPS: 70.74644167154412, Tokens per sec: 175317.47582599602, Loss: 2.5045583248138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18723 , TFLOPS: 66.30896293145022, Tokens per sec: 164320.91467940627, Loss: 2.491598606109619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18724 , TFLOPS: 71.50394918568637, Tokens per sec: 177194.6628923819, Loss: 2.5241501331329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18725 , TFLOPS: 71.49854968916716, Tokens per sec: 177181.28234520368, Loss: 2.489131212234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18726 , TFLOPS: 68.92222047642733, Tokens per sec: 170796.85474994275, Loss: 2.483820915222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18727 , TFLOPS: 69.81288024503739, Tokens per sec: 173004.00777083152, Loss: 2.4860787391662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18728 , TFLOPS: 71.51885410848357, Tokens per sec: 177231.59893298615, Loss: 2.4872355461120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18729 , TFLOPS: 69.80492791273257, Tokens per sec: 172984.30101535848, Loss: 2.480238437652588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18730 , TFLOPS: 67.42424888412849, Tokens per sec: 167084.71612903298, Loss: 2.4836206436157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18731 , TFLOPS: 71.52253337130527, Tokens per sec: 177240.71655156257, Loss: 2.5055646896362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18732 , TFLOPS: 71.44490355797883, Tokens per sec: 177048.34132250978, Loss: 2.475123643875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18733 , TFLOPS: 67.73696451849376, Tokens per sec: 167859.65991946103, Loss: 2.494629383087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18734 , TFLOPS: 70.9275105314957, Tokens per sec: 175766.1844073376, Loss: 2.4833152294158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18735 , TFLOPS: 71.4710862664675, Tokens per sec: 177113.22495840816, Loss: 2.482426404953003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18736 , TFLOPS: 68.11745873276273, Tokens per sec: 168802.56649731743, Loss: 2.488091468811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18737 , TFLOPS: 68.63351172314124, Tokens per sec: 170081.40265540517, Loss: 2.483356475830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18738 , TFLOPS: 71.49409058578134, Tokens per sec: 177170.2321958028, Loss: 2.494119644165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18739 , TFLOPS: 70.43530182259545, Tokens per sec: 174546.43700541742, Loss: 2.490206003189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18740 , TFLOPS: 68.85642638891692, Tokens per sec: 170633.80975327487, Loss: 2.5005240440368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18741 , TFLOPS: 69.63833369931862, Tokens per sec: 172571.4622026511, Loss: 2.503751277923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18742 , TFLOPS: 70.73282890583135, Tokens per sec: 175283.7418929899, Loss: 2.4950110912323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18743 , TFLOPS: 67.97883568034763, Tokens per sec: 168459.0433028429, Loss: 2.4854135513305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18744 , TFLOPS: 70.83962947359359, Tokens per sec: 175548.40546495846, Loss: 2.489760398864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18745 , TFLOPS: 70.41781224455615, Tokens per sec: 174503.09590439996, Loss: 2.487778902053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18746 , TFLOPS: 69.02683422175846, Tokens per sec: 171056.0991350287, Loss: 2.502263069152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18747 , TFLOPS: 69.62837746029285, Tokens per sec: 172546.78954557498, Loss: 2.487518310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18748 , TFLOPS: 70.52610407053382, Tokens per sec: 174771.45497850206, Loss: 2.492427349090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18749 , TFLOPS: 69.81260635743536, Tokens per sec: 173003.3290471825, Loss: 2.4922170639038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18750 , TFLOPS: 67.98475468230649, Tokens per sec: 168473.71124172886, Loss: 2.492969512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18751 , TFLOPS: 71.50834431595689, Tokens per sec: 177205.55450935583, Loss: 2.4808034896850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18752 , TFLOPS: 70.5473389890734, Tokens per sec: 174824.0774458625, Loss: 2.491666793823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18753 , TFLOPS: 67.89980350962752, Tokens per sec: 168263.19287768618, Loss: 2.480194568634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18754 , TFLOPS: 70.56676437013743, Tokens per sec: 174872.21568002156, Loss: 2.496792793273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18755 , TFLOPS: 71.46729991410498, Tokens per sec: 177103.8419601527, Loss: 2.4993209838867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18756 , TFLOPS: 67.07784995116958, Tokens per sec: 166226.30141416818, Loss: 2.4833900928497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18757 , TFLOPS: 69.73129661434349, Tokens per sec: 172801.83454679322, Loss: 2.514256477355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18758 , TFLOPS: 71.52769971422079, Tokens per sec: 177253.5193184269, Loss: 2.489816665649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18759 , TFLOPS: 70.48929610519926, Tokens per sec: 174680.2407856709, Loss: 2.4802803993225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18760 , TFLOPS: 68.73437118402991, Tokens per sec: 170331.3435100741, Loss: 2.499817132949829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18761 , TFLOPS: 71.51273448096521, Tokens per sec: 177216.43382186306, Loss: 2.4916865825653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18762 , TFLOPS: 71.56004435078857, Tokens per sec: 177333.67289089772, Loss: 2.504275321960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18763 , TFLOPS: 66.48514085285434, Tokens per sec: 164757.50297624583, Loss: 2.485114574432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18764 , TFLOPS: 70.68052225086414, Tokens per sec: 175154.1202399274, Loss: 2.4817399978637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18765 , TFLOPS: 71.52406982681914, Tokens per sec: 177244.5240575805, Loss: 2.4804866313934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18766 , TFLOPS: 68.86156125008338, Tokens per sec: 170646.53450489705, Loss: 2.4926860332489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18767 , TFLOPS: 68.6033847829739, Tokens per sec: 170006.74477891406, Loss: 2.4952473640441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18768 , TFLOPS: 71.49771085547144, Tokens per sec: 177179.2036229005, Loss: 2.4903128147125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18769 , TFLOPS: 70.6575670638002, Tokens per sec: 175097.23475766034, Loss: 2.4959328174591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18770 , TFLOPS: 67.28710935510578, Tokens per sec: 166744.86926894845, Loss: 2.494225025177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18771 , TFLOPS: 71.5150907848929, Tokens per sec: 177222.27300256418, Loss: 2.4921133518218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18772 , TFLOPS: 71.48949236562018, Tokens per sec: 177158.8372997077, Loss: 2.50528883934021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18773 , TFLOPS: 68.67127014681304, Tokens per sec: 170174.97218869763, Loss: 2.4587032794952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18774 , TFLOPS: 68.92324361542467, Tokens per sec: 170799.3902010862, Loss: 2.492276191711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18775 , TFLOPS: 71.55466408635534, Tokens per sec: 177320.34000294714, Loss: 2.494075059890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18776 , TFLOPS: 68.96601382739561, Tokens per sec: 170905.37949787802, Loss: 2.485980272293091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18777 , TFLOPS: 69.16231297797982, Tokens per sec: 171391.83041716224, Loss: 2.489811658859253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18778 , TFLOPS: 70.40582819101644, Tokens per sec: 174473.3980995761, Loss: 2.4734108448028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18779 , TFLOPS: 71.51291044796282, Tokens per sec: 177216.86988746747, Loss: 2.497788667678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18780 , TFLOPS: 67.9678181950677, Tokens per sec: 168431.7407606431, Loss: 2.484309434890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18781 , TFLOPS: 71.48794506880004, Tokens per sec: 177155.00292773856, Loss: 2.4921014308929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18782 , TFLOPS: 70.4912431098113, Tokens per sec: 174685.06567757978, Loss: 2.4738731384277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18783 , TFLOPS: 68.16831192813801, Tokens per sec: 168928.58631739925, Loss: 2.504605293273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18784 , TFLOPS: 70.12872583523314, Tokens per sec: 173786.70793660064, Loss: 2.4850118160247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18785 , TFLOPS: 70.50284075716674, Tokens per sec: 174713.8059253129, Loss: 2.4697766304016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18786 , TFLOPS: 69.79699961420727, Tokens per sec: 172964.6538182386, Loss: 2.4714667797088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18787 , TFLOPS: 69.0527280726397, Tokens per sec: 171120.26694995552, Loss: 2.495046377182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18788 , TFLOPS: 71.5020990195777, Tokens per sec: 177190.07797695202, Loss: 2.493908166885376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18789 , TFLOPS: 69.59300698965704, Tokens per sec: 172459.13762296046, Loss: 2.497657537460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18790 , TFLOPS: 67.34550165310252, Tokens per sec: 166889.571815827, Loss: 2.4738402366638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18791 , TFLOPS: 71.5284139522254, Tokens per sec: 177255.28927888218, Loss: 2.4749786853790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18792 , TFLOPS: 71.49268811264004, Tokens per sec: 177166.75671286348, Loss: 2.4767494201660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18793 , TFLOPS: 68.67593708733608, Tokens per sec: 170186.53738141965, Loss: 2.503087282180786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18794 , TFLOPS: 69.21464808288309, Tokens per sec: 171521.52257230028, Loss: 2.4924232959747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18795 , TFLOPS: 71.49732165728894, Tokens per sec: 177178.23914693051, Loss: 2.497541666030884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18796 , TFLOPS: 69.87001278959465, Tokens per sec: 173145.5885099138, Loss: 2.45308256149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18797 , TFLOPS: 67.76204605847003, Tokens per sec: 167921.81473848195, Loss: 2.460221290588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18798 , TFLOPS: 71.53356528441414, Tokens per sec: 177268.05484751193, Loss: 2.4862537384033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18799 , TFLOPS: 71.51114362968124, Tokens per sec: 177212.49151713462, Loss: 2.4614710807800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18800 , TFLOPS: 68.49499743109158, Tokens per sec: 169738.14898109718, Loss: 2.4886980056762695 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/optimizer.pt +[rank0]:[2024-08-27 17:43:51,329] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00637814900255762, 'preprocessing_with_comm': 0.001629246020456776, 'state_converting': 0.9792737499810755, : 0.9886549840157386}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800, took 4.39s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0016800 + eval ppl=10.250767707824707, eval loss=2.327352523803711 +------------------------------------------------------------------ +iteration: 18801 , TFLOPS: 65.5402402503125, Tokens per sec: 162415.935193753, Loss: 2.4756851196289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18802 , TFLOPS: 71.48341970715315, Tokens per sec: 177143.7885830678, Loss: 2.474226951599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18803 , TFLOPS: 68.3009900769517, Tokens per sec: 169257.37738586424, Loss: 2.481346845626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18804 , TFLOPS: 68.34813619277227, Tokens per sec: 169374.210654441, Loss: 2.4865660667419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18805 , TFLOPS: 71.49150039317098, Tokens per sec: 177163.81341318108, Loss: 2.4872469902038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18806 , TFLOPS: 70.70185757212265, Tokens per sec: 175206.99151628563, Loss: 2.4823949337005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18807 , TFLOPS: 67.40179666298717, Tokens per sec: 167029.07705172844, Loss: 2.4747331142425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18808 , TFLOPS: 71.5012700968556, Tokens per sec: 177188.02381513326, Loss: 2.494986057281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18809 , TFLOPS: 70.57108677821606, Tokens per sec: 174882.92708339295, Loss: 2.469531297683716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18810 , TFLOPS: 67.32432595411423, Tokens per sec: 166837.0961010382, Loss: 2.5068061351776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18811 , TFLOPS: 70.36794696116185, Tokens per sec: 174379.52423903445, Loss: 2.481433153152466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18812 , TFLOPS: 71.47840482982087, Tokens per sec: 177131.36116460414, Loss: 2.4994983673095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18813 , TFLOPS: 68.01617519349136, Tokens per sec: 168551.57472970005, Loss: 2.489400625228882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18814 , TFLOPS: 70.61499000074448, Tokens per sec: 174991.7241051576, Loss: 2.4727959632873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18815 , TFLOPS: 70.31268613184075, Tokens per sec: 174242.58181649388, Loss: 2.51340913772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18816 , TFLOPS: 68.94496604797604, Tokens per sec: 170853.220781872, Loss: 2.4937641620635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18817 , TFLOPS: 69.73171629906925, Tokens per sec: 172802.87457177576, Loss: 2.4824743270874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18818 , TFLOPS: 71.51782753162595, Tokens per sec: 177229.0549624462, Loss: 2.504401922225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18819 , TFLOPS: 69.61620342835535, Tokens per sec: 172516.62095335353, Loss: 2.485574960708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18820 , TFLOPS: 68.77860282909302, Tokens per sec: 170440.95439905842, Loss: 2.4849469661712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18821 , TFLOPS: 70.95356754786522, Tokens per sec: 175830.75656431858, Loss: 2.4968535900115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18822 , TFLOPS: 70.42852934614804, Tokens per sec: 174529.65406273087, Loss: 2.5045506954193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18823 , TFLOPS: 68.44684545119034, Tokens per sec: 169618.82307052347, Loss: 2.4772610664367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18824 , TFLOPS: 70.54816176259533, Tokens per sec: 174826.11636928582, Loss: 2.5042858123779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18825 , TFLOPS: 71.5022259463548, Tokens per sec: 177190.3925154878, Loss: 2.498763084411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18826 , TFLOPS: 69.63904368785154, Tokens per sec: 172573.22163244736, Loss: 2.4977762699127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18827 , TFLOPS: 68.24395610800906, Tokens per sec: 169116.04092801403, Loss: 2.5034611225128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18828 , TFLOPS: 71.52046420067893, Tokens per sec: 177235.58891881254, Loss: 2.5009372234344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18829 , TFLOPS: 69.59932218025148, Tokens per sec: 172474.78736092913, Loss: 2.5157711505889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18830 , TFLOPS: 67.30193853592117, Tokens per sec: 166781.6176125781, Loss: 2.4853103160858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18831 , TFLOPS: 71.51288700433214, Tokens per sec: 177216.81179156812, Loss: 2.4968419075012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18832 , TFLOPS: 71.48265830408464, Tokens per sec: 177141.90174238858, Loss: 2.499298095703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18833 , TFLOPS: 68.68511440295447, Tokens per sec: 170209.2797804868, Loss: 2.51495361328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18834 , TFLOPS: 70.10893763310008, Tokens per sec: 173737.6705918645, Loss: 2.483543872833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18835 , TFLOPS: 71.49256969628199, Tokens per sec: 177166.4632640836, Loss: 2.4927258491516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18836 , TFLOPS: 69.80006256230102, Tokens per sec: 172972.24414102652, Loss: 2.4935436248779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18837 , TFLOPS: 68.8171102470044, Tokens per sec: 170536.38002258507, Loss: 2.4995906352996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18838 , TFLOPS: 71.4968980814568, Tokens per sec: 177177.1894793578, Loss: 2.507460117340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18839 , TFLOPS: 71.55491252074283, Tokens per sec: 177320.95565072662, Loss: 2.473811388015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18840 , TFLOPS: 68.22568254263456, Tokens per sec: 169070.7570200166, Loss: 2.490497350692749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18841 , TFLOPS: 69.76329439807849, Tokens per sec: 172881.1285797354, Loss: 2.4963958263397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18842 , TFLOPS: 71.4885170585938, Tokens per sec: 177156.42038145737, Loss: 2.4922173023223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18843 , TFLOPS: 68.9730358447161, Tokens per sec: 170922.78083033726, Loss: 2.481792688369751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18844 , TFLOPS: 68.99274001693806, Tokens per sec: 170971.60993969702, Loss: 2.4814071655273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18845 , TFLOPS: 71.50400874094326, Tokens per sec: 177194.81047686905, Loss: 2.4790146350860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18846 , TFLOPS: 70.73101126705018, Tokens per sec: 175279.2375838607, Loss: 2.4926416873931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18847 , TFLOPS: 67.94357228778449, Tokens per sec: 168371.6567314881, Loss: 2.494100570678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18848 , TFLOPS: 70.44408596526826, Tokens per sec: 174568.20507861383, Loss: 2.4968678951263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18849 , TFLOPS: 70.63256955122802, Tokens per sec: 175035.28816780317, Loss: 2.4824929237365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18850 , TFLOPS: 68.06446744034115, Tokens per sec: 168671.24823722398, Loss: 2.483099937438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18851 , TFLOPS: 70.87884867099686, Tokens per sec: 175645.59495647502, Loss: 2.5023789405822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18852 , TFLOPS: 70.52631215587296, Tokens per sec: 174771.97063689493, Loss: 2.496830701828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18853 , TFLOPS: 69.750054870979, Tokens per sec: 172848.31957312857, Loss: 2.4926719665527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18854 , TFLOPS: 69.27186050415231, Tokens per sec: 171663.3012546739, Loss: 2.464815378189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18855 , TFLOPS: 71.46395422711589, Tokens per sec: 177095.55097923585, Loss: 2.49747896194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18856 , TFLOPS: 69.63944533476248, Tokens per sec: 172574.21695773778, Loss: 2.4957008361816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18857 , TFLOPS: 68.26652906698047, Tokens per sec: 169171.9792069619, Loss: 2.4737250804901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18858 , TFLOPS: 71.52426645025437, Tokens per sec: 177245.01131211285, Loss: 2.4747941493988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18859 , TFLOPS: 70.53074606373703, Tokens per sec: 174782.95834901737, Loss: 2.483201503753662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18860 , TFLOPS: 67.76211169448028, Tokens per sec: 167921.97739174575, Loss: 2.4802231788635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18861 , TFLOPS: 70.58020894670578, Tokens per sec: 174905.5328218003, Loss: 2.488213539123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18862 , TFLOPS: 71.53217268606724, Tokens per sec: 177264.60383540092, Loss: 2.5149471759796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18863 , TFLOPS: 67.86341644906511, Tokens per sec: 168173.02173324692, Loss: 2.5085368156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18864 , TFLOPS: 69.756132395117, Tokens per sec: 172863.3803474371, Loss: 2.47725772857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18865 , TFLOPS: 71.52255968139683, Tokens per sec: 177240.78175086775, Loss: 2.489382266998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18866 , TFLOPS: 69.71723896921921, Tokens per sec: 172766.9981535978, Loss: 2.4943270683288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18867 , TFLOPS: 68.18122599472501, Tokens per sec: 168960.58879700376, Loss: 2.525953769683838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18868 , TFLOPS: 71.52589029537442, Tokens per sec: 177249.0353792017, Loss: 2.476539373397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18869 , TFLOPS: 71.5756687713631, Tokens per sec: 177372.3919262534, Loss: 2.4891443252563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18870 , TFLOPS: 68.3505833946017, Tokens per sec: 169380.27509015586, Loss: 2.50181245803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18871 , TFLOPS: 70.09066092349377, Tokens per sec: 173692.3788921143, Loss: 2.4776313304901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18872 , TFLOPS: 71.444520555308, Tokens per sec: 177047.39219970023, Loss: 2.4671554565429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18873 , TFLOPS: 70.50243895623534, Tokens per sec: 174712.81021834287, Loss: 2.5059280395507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18874 , TFLOPS: 67.2443978013916, Tokens per sec: 166639.02533377864, Loss: 2.4844632148742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18875 , TFLOPS: 71.54102026254041, Tokens per sec: 177286.52910454245, Loss: 2.4803128242492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18876 , TFLOPS: 70.74391727967664, Tokens per sec: 175311.22010486372, Loss: 2.4957754611968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18877 , TFLOPS: 67.97494163561042, Tokens per sec: 168449.3934310197, Loss: 2.4786176681518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18878 , TFLOPS: 70.54376333801633, Tokens per sec: 174815.2165886526, Loss: 2.4801394939422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18879 , TFLOPS: 71.51764299044135, Tokens per sec: 177228.59764906185, Loss: 2.4838807582855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18880 , TFLOPS: 67.99946730380857, Tokens per sec: 168510.17073854012, Loss: 2.5099732875823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18881 , TFLOPS: 69.52654461066635, Tokens per sec: 172294.43652638607, Loss: 2.4904356002807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18882 , TFLOPS: 71.55241802528174, Tokens per sec: 177314.77401618267, Loss: 2.4953980445861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18883 , TFLOPS: 68.92436614162594, Tokens per sec: 170802.17194466945, Loss: 2.493736743927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18884 , TFLOPS: 69.84031254791971, Tokens per sec: 173071.98803929158, Loss: 2.500812530517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18885 , TFLOPS: 70.60203728374297, Tokens per sec: 174959.62584556828, Loss: 2.4856739044189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18886 , TFLOPS: 70.60838037601589, Tokens per sec: 174975.34472696987, Loss: 2.48128604888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18887 , TFLOPS: 68.91924659118102, Tokens per sec: 170789.48513470188, Loss: 2.494020462036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18888 , TFLOPS: 70.84310476336483, Tokens per sec: 175557.01761584636, Loss: 2.485511302947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18889 , TFLOPS: 70.4556732750604, Tokens per sec: 174596.91970872632, Loss: 2.4996628761291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18890 , TFLOPS: 68.19119979701784, Tokens per sec: 168985.3049777904, Loss: 2.498847007751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18891 , TFLOPS: 69.71262669803731, Tokens per sec: 172755.56843752545, Loss: 2.481736183166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18892 , TFLOPS: 71.49821935191001, Tokens per sec: 177180.4637330908, Loss: 2.4930472373962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18893 , TFLOPS: 69.56450502571634, Tokens per sec: 172388.50661656488, Loss: 2.4694669246673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18894 , TFLOPS: 67.9788435882908, Tokens per sec: 168459.06289959722, Loss: 2.500096082687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18895 , TFLOPS: 71.53477290146726, Tokens per sec: 177271.04745559956, Loss: 2.5017518997192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18896 , TFLOPS: 69.7077204403016, Tokens per sec: 172743.41019038745, Loss: 2.4732654094696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18897 , TFLOPS: 67.88288646088111, Tokens per sec: 168221.27056732582, Loss: 2.5090677738189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18898 , TFLOPS: 71.54130011730473, Tokens per sec: 177287.22261547702, Loss: 2.4714813232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18899 , TFLOPS: 71.49126519041126, Tokens per sec: 177163.23055483273, Loss: 2.484628677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18900 , TFLOPS: 66.68504948275165, Tokens per sec: 165252.89858288493, Loss: 2.4808740615844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18901 , TFLOPS: 70.60048449885088, Tokens per sec: 174955.7778735523, Loss: 2.4816625118255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18902 , TFLOPS: 71.55731713834003, Tokens per sec: 177326.91455802298, Loss: 2.4996113777160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18903 , TFLOPS: 69.62302433056423, Tokens per sec: 172533.5239004116, Loss: 2.4991414546966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18904 , TFLOPS: 69.02868719677464, Tokens per sec: 171060.69101124082, Loss: 2.4937903881073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18905 , TFLOPS: 71.49406479874371, Tokens per sec: 177170.16829268288, Loss: 2.485992670059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18906 , TFLOPS: 71.49599023390343, Tokens per sec: 177174.93973311287, Loss: 2.486942768096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18907 , TFLOPS: 66.762272621939, Tokens per sec: 165444.2660286222, Loss: 2.4734318256378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18908 , TFLOPS: 70.59874190754624, Tokens per sec: 174951.45954033712, Loss: 2.460254192352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18909 , TFLOPS: 71.47531872188326, Tokens per sec: 177123.71344917183, Loss: 2.4787724018096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18910 , TFLOPS: 68.75982270380747, Tokens per sec: 170394.41517979928, Loss: 2.515523910522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18911 , TFLOPS: 69.04196759392146, Tokens per sec: 171093.60129832715, Loss: 2.4809682369232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18912 , TFLOPS: 71.5083491257433, Tokens per sec: 177205.56642853637, Loss: 2.4995625019073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18913 , TFLOPS: 70.5218328145019, Tokens per sec: 174760.8703355373, Loss: 2.4722588062286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18914 , TFLOPS: 67.97773949947563, Tokens per sec: 168456.32684588782, Loss: 2.4957404136657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18915 , TFLOPS: 70.4766666145528, Tokens per sec: 174648.94351659573, Loss: 2.5109617710113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18916 , TFLOPS: 70.70887637843896, Tokens per sec: 175224.38489152282, Loss: 2.482896327972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18917 , TFLOPS: 67.89454296594641, Tokens per sec: 168250.15667094768, Loss: 2.49578857421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18918 , TFLOPS: 70.428369758295, Tokens per sec: 174529.25858645097, Loss: 2.493802070617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18919 , TFLOPS: 71.50287581486002, Tokens per sec: 177192.00296123265, Loss: 2.5155649185180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18920 , TFLOPS: 69.06812504670296, Tokens per sec: 171158.422347801, Loss: 2.489558219909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18921 , TFLOPS: 69.83923893231842, Tokens per sec: 173069.32750156306, Loss: 2.4711973667144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18922 , TFLOPS: 70.44318544368137, Tokens per sec: 174565.97348692085, Loss: 2.4729437828063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18923 , TFLOPS: 70.7529642549261, Tokens per sec: 175333.63950613845, Loss: 2.4936163425445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18924 , TFLOPS: 68.23602050416721, Tokens per sec: 169096.37562751502, Loss: 2.492333173751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18925 , TFLOPS: 71.49641611666208, Tokens per sec: 177175.99511750365, Loss: 2.4612812995910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18926 , TFLOPS: 70.51272346512789, Tokens per sec: 174738.29636431078, Loss: 2.4873597621917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18927 , TFLOPS: 68.05097837836446, Tokens per sec: 168637.8208557029, Loss: 2.4844324588775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18928 , TFLOPS: 70.57026553024795, Tokens per sec: 174880.89194046005, Loss: 2.477576732635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18929 , TFLOPS: 71.4787925000688, Tokens per sec: 177132.32185418386, Loss: 2.4959840774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18930 , TFLOPS: 67.88521485984054, Tokens per sec: 168227.04059644163, Loss: 2.453411340713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18931 , TFLOPS: 68.95256566691995, Tokens per sec: 170872.05347478215, Loss: 2.5031204223632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18932 , TFLOPS: 71.54570761420965, Tokens per sec: 177298.14487833323, Loss: 2.4879724979400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18933 , TFLOPS: 69.60021115581496, Tokens per sec: 172476.9903403042, Loss: 2.4862489700317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18934 , TFLOPS: 68.03446861433744, Tokens per sec: 168596.90784174332, Loss: 2.50571870803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18935 , TFLOPS: 71.51201163099145, Tokens per sec: 177214.6425199996, Loss: 2.4927730560302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18936 , TFLOPS: 70.66308965453574, Tokens per sec: 175110.92034586662, Loss: 2.4792520999908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18937 , TFLOPS: 68.64512945212505, Tokens per sec: 170110.19266762555, Loss: 2.495166778564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18938 , TFLOPS: 70.77311109913562, Tokens per sec: 175383.56560544774, Loss: 2.5175201892852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18939 , TFLOPS: 71.54257579395832, Tokens per sec: 177290.38388275195, Loss: 2.482149600982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18940 , TFLOPS: 69.64439470062376, Tokens per sec: 172586.4820315597, Loss: 2.4754512310028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18941 , TFLOPS: 69.05351393917326, Tokens per sec: 171122.21441379603, Loss: 2.490048885345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18942 , TFLOPS: 71.4804677459009, Tokens per sec: 177136.47329789936, Loss: 2.477783203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18943 , TFLOPS: 70.7374259460182, Tokens per sec: 175295.13386497865, Loss: 2.4931342601776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18944 , TFLOPS: 67.19951001595884, Tokens per sec: 166527.78845667848, Loss: 2.5066189765930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18945 , TFLOPS: 71.51521382567348, Tokens per sec: 177222.5779111725, Loss: 2.4911954402923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18946 , TFLOPS: 71.50285906005455, Tokens per sec: 177191.96144097898, Loss: 2.488882064819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18947 , TFLOPS: 68.09589066201926, Tokens per sec: 168749.11844209593, Loss: 2.5029678344726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18948 , TFLOPS: 70.2999875609266, Tokens per sec: 174211.11335890557, Loss: 2.4976861476898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18949 , TFLOPS: 71.54592584009224, Tokens per sec: 177298.68566610984, Loss: 2.490234851837158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18950 , TFLOPS: 69.73769470818758, Tokens per sec: 172817.6897281471, Loss: 2.4852519035339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18951 , TFLOPS: 68.35653005152376, Tokens per sec: 169395.01156108707, Loss: 2.468846321105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18952 , TFLOPS: 70.50041377950893, Tokens per sec: 174707.7916073232, Loss: 2.463247060775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18953 , TFLOPS: 70.62847978581893, Tokens per sec: 175025.1532785946, Loss: 2.515273094177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18954 , TFLOPS: 68.9742325721131, Tokens per sec: 170925.74645265742, Loss: 2.4626686573028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18955 , TFLOPS: 69.73482425864776, Tokens per sec: 172810.5764379813, Loss: 2.5010147094726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18956 , TFLOPS: 71.5465212374875, Tokens per sec: 177300.16112644374, Loss: 2.480283260345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18957 , TFLOPS: 68.29157396401426, Tokens per sec: 169234.0432207348, Loss: 2.5315723419189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18958 , TFLOPS: 70.44609479370547, Tokens per sec: 174573.18317677197, Loss: 2.4842114448547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18959 , TFLOPS: 70.53330664622415, Tokens per sec: 174789.3037544919, Loss: 2.481532573699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18960 , TFLOPS: 70.61262886819246, Tokens per sec: 174985.87295859348, Loss: 2.4776253700256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18961 , TFLOPS: 68.2511182826485, Tokens per sec: 169133.78958574924, Loss: 2.491504430770874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18962 , TFLOPS: 71.53650556849475, Tokens per sec: 177275.34119536268, Loss: 2.4676427841186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18963 , TFLOPS: 69.49378314647058, Tokens per sec: 172213.25000913575, Loss: 2.481121301651001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18964 , TFLOPS: 67.99345696872189, Tokens per sec: 168495.2764661022, Loss: 2.495244026184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18965 , TFLOPS: 71.49788741481841, Tokens per sec: 177179.6411564119, Loss: 2.50679874420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18966 , TFLOPS: 71.52924921565076, Tokens per sec: 177257.35915366205, Loss: 2.4811010360717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18967 , TFLOPS: 67.11434669688339, Tokens per sec: 166316.74437049546, Loss: 2.4830188751220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18968 , TFLOPS: 70.58953303863966, Tokens per sec: 174928.63894874597, Loss: 2.496506929397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18969 , TFLOPS: 71.5011423140621, Tokens per sec: 177187.70715529448, Loss: 2.4957690238952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18970 , TFLOPS: 70.28539971655945, Tokens per sec: 174174.96307358006, Loss: 2.4824674129486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18971 , TFLOPS: 67.55346157280701, Tokens per sec: 167404.91940553102, Loss: 2.505258083343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18972 , TFLOPS: 71.51488868600406, Tokens per sec: 177221.77217925456, Loss: 2.499767303466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18973 , TFLOPS: 71.49734596970445, Tokens per sec: 177178.29939577446, Loss: 2.4838883876800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18974 , TFLOPS: 67.36865064817106, Tokens per sec: 166946.93757567916, Loss: 2.489577293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18975 , TFLOPS: 70.57633865684608, Tokens per sec: 174895.94181718034, Loss: 2.521834373474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18976 , TFLOPS: 71.5357522647105, Tokens per sec: 177273.47442560206, Loss: 2.4761972427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18977 , TFLOPS: 67.94514386283991, Tokens per sec: 168375.55126759873, Loss: 2.4998104572296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18978 , TFLOPS: 69.92283369017038, Tokens per sec: 173276.484520243, Loss: 2.49773907661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18979 , TFLOPS: 71.52717455414893, Tokens per sec: 177252.2179139177, Loss: 2.500126838684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18980 , TFLOPS: 70.6882608997868, Tokens per sec: 175173.2974644403, Loss: 2.5011353492736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18981 , TFLOPS: 67.12575550908734, Tokens per sec: 166345.01666392127, Loss: 2.4732179641723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18982 , TFLOPS: 71.53774529924799, Tokens per sec: 177278.4133847368, Loss: 2.470625877380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18983 , TFLOPS: 70.60287427036698, Tokens per sec: 174961.6999906243, Loss: 2.500281572341919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18984 , TFLOPS: 68.08222309665504, Tokens per sec: 168715.2487095168, Loss: 2.500354051589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18985 , TFLOPS: 69.36743971916388, Tokens per sec: 171900.15707839228, Loss: 2.4881043434143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18986 , TFLOPS: 71.51877282935358, Tokens per sec: 177231.3975143484, Loss: 2.478299140930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18987 , TFLOPS: 69.02943541677176, Tokens per sec: 171062.54518281692, Loss: 2.486090660095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18988 , TFLOPS: 69.91633583307332, Tokens per sec: 173260.38211455842, Loss: 2.4738857746124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18989 , TFLOPS: 70.25033434622311, Tokens per sec: 174088.0672231172, Loss: 2.4972848892211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18990 , TFLOPS: 70.52975958026825, Tokens per sec: 174780.51373431162, Loss: 2.474149703979492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18991 , TFLOPS: 67.2881811347724, Tokens per sec: 166747.5252570286, Loss: 2.51246976852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18992 , TFLOPS: 70.47636749246725, Tokens per sec: 174648.20225911672, Loss: 2.4805195331573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18993 , TFLOPS: 71.55439939766896, Tokens per sec: 177319.68407522442, Loss: 2.485400676727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18994 , TFLOPS: 67.37341520825925, Tokens per sec: 166958.74468043732, Loss: 2.473447561264038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18995 , TFLOPS: 70.66815430060949, Tokens per sec: 175123.4710967542, Loss: 2.5156803131103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18996 , TFLOPS: 70.42587849678763, Tokens per sec: 174523.0849660006, Loss: 2.4814648628234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18997 , TFLOPS: 68.89213097190292, Tokens per sec: 170722.28964309872, Loss: 2.506824254989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18998 , TFLOPS: 69.67889793632581, Tokens per sec: 172671.98485047455, Loss: 2.482419490814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18999 , TFLOPS: 71.46638668280622, Tokens per sec: 177101.57887239358, Loss: 2.469175100326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19000 , TFLOPS: 69.77655360193002, Tokens per sec: 172913.9863475014, Loss: 2.490478277206421 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/optimizer.pt +[rank0]:[2024-08-27 18:09:13,197] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006417414988391101, 'preprocessing_with_comm': 0.0012378609972074628, 'state_converting': 0.9777057860046625, : 0.9867570020142011}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017000 + eval ppl=10.021748542785645, eval loss=2.304757595062256 +------------------------------------------------------------------ +iteration: 19001 , TFLOPS: 61.93174453631281, Tokens per sec: 153473.68530584362, Loss: 2.494976043701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19002 , TFLOPS: 71.49942804510196, Tokens per sec: 177183.45900796898, Loss: 2.476771354675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19003 , TFLOPS: 68.58986068123787, Tokens per sec: 169973.23056500938, Loss: 2.4867565631866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19004 , TFLOPS: 68.35326799900528, Tokens per sec: 169386.92783560156, Loss: 2.5092334747314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19005 , TFLOPS: 70.58053257753329, Tokens per sec: 174906.33481464177, Loss: 2.514665126800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19006 , TFLOPS: 71.48243130501574, Tokens per sec: 177141.33921368985, Loss: 2.5095884799957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19007 , TFLOPS: 68.30360487340405, Tokens per sec: 169263.85713951677, Loss: 2.5009567737579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19008 , TFLOPS: 69.8962576347375, Tokens per sec: 173210.62612728612, Loss: 2.488295793533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19009 , TFLOPS: 71.47649430464887, Tokens per sec: 177126.62667276614, Loss: 2.498744249343872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19010 , TFLOPS: 70.58003939348052, Tokens per sec: 174905.11265023026, Loss: 2.499211072921753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19011 , TFLOPS: 66.34727859897133, Tokens per sec: 164415.86512434276, Loss: 2.4766173362731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19012 , TFLOPS: 71.4825240250855, Tokens per sec: 177141.56898423666, Loss: 2.5007126331329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19013 , TFLOPS: 70.60446841039786, Tokens per sec: 174965.65044522978, Loss: 2.488032102584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19014 , TFLOPS: 66.46317958541276, Tokens per sec: 164703.08053629228, Loss: 2.484079360961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19015 , TFLOPS: 70.80975884733182, Tokens per sec: 175474.38276256065, Loss: 2.5050652027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19016 , TFLOPS: 71.43103996638094, Tokens per sec: 177013.9858153295, Loss: 2.4893264770507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19017 , TFLOPS: 68.55827710831892, Tokens per sec: 169894.96299209847, Loss: 2.5059683322906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19018 , TFLOPS: 67.60847238274035, Tokens per sec: 167541.24225248682, Loss: 2.4625155925750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19019 , TFLOPS: 71.49841116979566, Tokens per sec: 177180.9390789419, Loss: 2.4727671146392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19020 , TFLOPS: 70.70236365814563, Tokens per sec: 175208.24565320055, Loss: 2.4941134452819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19021 , TFLOPS: 67.60157057884173, Tokens per sec: 167524.13882212958, Loss: 2.512256383895874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19022 , TFLOPS: 70.39869221437398, Tokens per sec: 174455.71436336334, Loss: 2.4984586238861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19023 , TFLOPS: 70.63387134789936, Tokens per sec: 175038.5141633595, Loss: 2.503525733947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19024 , TFLOPS: 68.89917591462769, Tokens per sec: 170739.74778723522, Loss: 2.47979736328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19025 , TFLOPS: 70.60980069888431, Tokens per sec: 174978.8644434993, Loss: 2.484997510910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19026 , TFLOPS: 70.40653455304698, Tokens per sec: 174475.14854249998, Loss: 2.495347738265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19027 , TFLOPS: 69.8123901908552, Tokens per sec: 173002.79336258414, Loss: 2.4639761447906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19028 , TFLOPS: 68.90257473481961, Tokens per sec: 170748.17043808175, Loss: 2.489758253097534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19029 , TFLOPS: 70.41264645899747, Tokens per sec: 174490.29451872729, Loss: 2.5097272396087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19030 , TFLOPS: 70.5474367254985, Tokens per sec: 174824.31964749045, Loss: 2.4527740478515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19031 , TFLOPS: 67.44800740924985, Tokens per sec: 167143.5923715013, Loss: 2.5018808841705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19032 , TFLOPS: 71.47656625273434, Tokens per sec: 177126.8049680477, Loss: 2.465416193008423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19033 , TFLOPS: 70.4419556824476, Tokens per sec: 174562.9260031142, Loss: 2.4865198135375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19034 , TFLOPS: 67.53122409900375, Tokens per sec: 167349.8124957866, Loss: 2.4757535457611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19035 , TFLOPS: 70.57175020612303, Tokens per sec: 174884.57113082905, Loss: 2.5054707527160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19036 , TFLOPS: 71.49438298132182, Tokens per sec: 177170.95678416212, Loss: 2.4905846118927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19037 , TFLOPS: 68.74886200475198, Tokens per sec: 170367.2533601191, Loss: 2.4950828552246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19038 , TFLOPS: 68.41014338678492, Tokens per sec: 169527.87131186068, Loss: 2.506563425064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19039 , TFLOPS: 71.49228536668106, Tokens per sec: 177165.75866401102, Loss: 2.5227174758911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19040 , TFLOPS: 69.53577695342928, Tokens per sec: 172317.31528877912, Loss: 2.5000460147857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19041 , TFLOPS: 67.30657765369358, Tokens per sec: 166793.1138574591, Loss: 2.472137451171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19042 , TFLOPS: 71.50350178440351, Tokens per sec: 177193.55418271763, Loss: 2.485621213912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19043 , TFLOPS: 70.61276523884676, Tokens per sec: 174986.2109000973, Loss: 2.4887263774871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19044 , TFLOPS: 68.8006915207311, Tokens per sec: 170495.6926102079, Loss: 2.4969778060913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19045 , TFLOPS: 69.94385652021427, Tokens per sec: 173328.58140894765, Loss: 2.484267234802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19046 , TFLOPS: 71.4322483841385, Tokens per sec: 177016.980407651, Loss: 2.485614538192749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19047 , TFLOPS: 69.72462405431561, Tokens per sec: 172785.29920800097, Loss: 2.48795485496521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19048 , TFLOPS: 68.35180823042887, Tokens per sec: 169383.31036826244, Loss: 2.4653398990631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19049 , TFLOPS: 71.52743587212579, Tokens per sec: 177252.8654886509, Loss: 2.4812772274017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19050 , TFLOPS: 70.66714473744283, Tokens per sec: 175120.96928801463, Loss: 2.4727563858032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19051 , TFLOPS: 67.20641831135282, Tokens per sec: 166544.90797367634, Loss: 2.5021774768829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19052 , TFLOPS: 70.86942137812635, Tokens per sec: 175622.23308623987, Loss: 2.4672110080718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19053 , TFLOPS: 71.45580488974882, Tokens per sec: 177075.35602351665, Loss: 2.4832308292388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19054 , TFLOPS: 67.14327664533133, Tokens per sec: 166388.43596964638, Loss: 2.5017919540405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19055 , TFLOPS: 69.5982218525596, Tokens per sec: 172472.0606276976, Loss: 2.472684144973755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19056 , TFLOPS: 71.5251852104782, Tokens per sec: 177247.2881011573, Loss: 2.5170769691467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19057 , TFLOPS: 70.68082475976745, Tokens per sec: 175154.86989031392, Loss: 2.504415273666382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19058 , TFLOPS: 67.65133748731066, Tokens per sec: 167647.46670359312, Loss: 2.489201545715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19059 , TFLOPS: 70.55807916612893, Tokens per sec: 174850.69278773433, Loss: 2.4692814350128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19060 , TFLOPS: 70.57491345164026, Tokens per sec: 174892.41000168098, Loss: 2.475234031677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19061 , TFLOPS: 67.19780923505243, Tokens per sec: 166523.57373423592, Loss: 2.462028980255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19062 , TFLOPS: 71.52020944563782, Tokens per sec: 177234.9576077571, Loss: 2.50146746635437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19063 , TFLOPS: 69.64954373725583, Tokens per sec: 172599.24191154714, Loss: 2.481696367263794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19064 , TFLOPS: 70.66574995908478, Tokens per sec: 175117.51287359587, Loss: 2.501227617263794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19065 , TFLOPS: 69.17337991599499, Tokens per sec: 171419.2555086877, Loss: 2.505577325820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19066 , TFLOPS: 70.53904062035882, Tokens per sec: 174803.513174047, Loss: 2.4962921142578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19067 , TFLOPS: 69.71633907893474, Tokens per sec: 172764.76812634178, Loss: 2.498532772064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19068 , TFLOPS: 68.73159612668643, Tokens per sec: 170324.46661227915, Loss: 2.492889881134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19069 , TFLOPS: 71.48131872142073, Tokens per sec: 177138.5821089802, Loss: 2.467514753341675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19070 , TFLOPS: 69.69811804010018, Tokens per sec: 172719.61438489662, Loss: 2.481361150741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19071 , TFLOPS: 68.25938029986142, Tokens per sec: 169154.26377453917, Loss: 2.4969146251678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19072 , TFLOPS: 70.62715333363958, Tokens per sec: 175021.86618397338, Loss: 2.5030601024627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19073 , TFLOPS: 71.52075060919692, Tokens per sec: 177236.29867067063, Loss: 2.4764089584350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19074 , TFLOPS: 66.73282529394504, Tokens per sec: 165371.29230596218, Loss: 2.493495225906372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19075 , TFLOPS: 70.00026480143634, Tokens per sec: 173468.3673437011, Loss: 2.492805004119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19076 , TFLOPS: 71.5082130116521, Tokens per sec: 177205.22912282418, Loss: 2.5052590370178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19077 , TFLOPS: 70.71513759116165, Tokens per sec: 175239.90086637915, Loss: 2.4993884563446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19078 , TFLOPS: 66.297143108063, Tokens per sec: 164291.62385499166, Loss: 2.50239896774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19079 , TFLOPS: 71.39577684665164, Tokens per sec: 176926.599920087, Loss: 2.5198123455047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19080 , TFLOPS: 70.53152900467009, Tokens per sec: 174784.89856289755, Loss: 2.4958925247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19081 , TFLOPS: 66.47378510785992, Tokens per sec: 164729.3621892111, Loss: 2.4999539852142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19082 , TFLOPS: 71.55912478234642, Tokens per sec: 177331.39409899924, Loss: 2.4639315605163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19083 , TFLOPS: 71.51063408975327, Tokens per sec: 177211.22882106263, Loss: 2.5200891494750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19084 , TFLOPS: 69.95323383234233, Tokens per sec: 173351.81942139717, Loss: 2.525578022003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19085 , TFLOPS: 67.9292242207653, Tokens per sec: 168336.10064084452, Loss: 2.4930708408355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19086 , TFLOPS: 71.49267976192728, Tokens per sec: 177166.7360188774, Loss: 2.479348659515381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19087 , TFLOPS: 70.56540444825842, Tokens per sec: 174868.8456437976, Loss: 2.4840731620788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19088 , TFLOPS: 67.41806737089145, Tokens per sec: 167069.3976582817, Loss: 2.5081875324249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19089 , TFLOPS: 71.4987953973877, Tokens per sec: 177181.89123724116, Loss: 2.4730000495910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19090 , TFLOPS: 70.71000895040382, Tokens per sec: 175227.19152961482, Loss: 2.4796857833862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19091 , TFLOPS: 68.0018069391893, Tokens per sec: 168515.96861274564, Loss: 2.4667136669158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19092 , TFLOPS: 69.63902180461882, Tokens per sec: 172573.16740338606, Loss: 2.4957571029663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19093 , TFLOPS: 71.54700561849084, Tokens per sec: 177301.36147592904, Loss: 2.4853103160858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19094 , TFLOPS: 68.91620533165252, Tokens per sec: 170781.94855856788, Loss: 2.4691720008850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19095 , TFLOPS: 69.1328708451572, Tokens per sec: 171318.8695687091, Loss: 2.493825912475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19096 , TFLOPS: 70.33781077562723, Tokens per sec: 174304.84345150477, Loss: 2.494868278503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19097 , TFLOPS: 70.6928184411216, Tokens per sec: 175184.5915539226, Loss: 2.480705738067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19098 , TFLOPS: 67.2543983803172, Tokens per sec: 166663.80787001096, Loss: 2.481588363647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19099 , TFLOPS: 71.48611152206738, Tokens per sec: 177150.45919695316, Loss: 2.4905991554260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19100 , TFLOPS: 70.53372088319284, Tokens per sec: 174790.3302793323, Loss: 2.4844141006469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19101 , TFLOPS: 67.6985453867603, Tokens per sec: 167764.45307880876, Loss: 2.491168737411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19102 , TFLOPS: 70.86141686984996, Tokens per sec: 175602.39703296157, Loss: 2.512810707092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19103 , TFLOPS: 69.34462263094954, Tokens per sec: 171843.6138202874, Loss: 2.491724729537964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19104 , TFLOPS: 70.66970446989082, Tokens per sec: 175127.31258699857, Loss: 2.4937937259674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19105 , TFLOPS: 68.21443927048671, Tokens per sec: 169042.8949530856, Loss: 2.508734703063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19106 , TFLOPS: 71.54059782197315, Tokens per sec: 177285.48225028111, Loss: 2.4979641437530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19107 , TFLOPS: 69.6257337879098, Tokens per sec: 172540.2382341852, Loss: 2.5005733966827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19108 , TFLOPS: 68.15089912019516, Tokens per sec: 168885.43546113776, Loss: 2.476611375808716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19109 , TFLOPS: 71.52858213989063, Tokens per sec: 177255.706066444, Loss: 2.4899003505706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19110 , TFLOPS: 70.61689135960137, Tokens per sec: 174996.43588185695, Loss: 2.476485252380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19111 , TFLOPS: 67.83327655578323, Tokens per sec: 168098.3317575111, Loss: 2.4813923835754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19112 , TFLOPS: 69.8251795353829, Tokens per sec: 173034.48676718638, Loss: 2.477783679962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19113 , TFLOPS: 71.52269514456461, Tokens per sec: 177241.11744352002, Loss: 2.4790878295898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19114 , TFLOPS: 69.6079619702285, Tokens per sec: 172496.19771224365, Loss: 2.4855213165283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19115 , TFLOPS: 68.41952117324026, Tokens per sec: 169551.11049974544, Loss: 2.4752273559570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19116 , TFLOPS: 71.51330969767807, Tokens per sec: 177217.85927224284, Loss: 2.490121603012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19117 , TFLOPS: 70.65123861696661, Tokens per sec: 175081.55216927076, Loss: 2.4909260272979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19118 , TFLOPS: 67.94133386348463, Tokens per sec: 168366.10966949025, Loss: 2.481942653656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19119 , TFLOPS: 70.81335341671975, Tokens per sec: 175483.29050147848, Loss: 2.5110440254211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19120 , TFLOPS: 71.50576953771487, Tokens per sec: 177199.17392523683, Loss: 2.4920663833618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19121 , TFLOPS: 67.43362795272347, Tokens per sec: 167107.95849420098, Loss: 2.474196672439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19122 , TFLOPS: 69.70153898342893, Tokens per sec: 172728.0918593133, Loss: 2.4796321392059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19123 , TFLOPS: 71.4650287832533, Tokens per sec: 177098.2138477165, Loss: 2.4772331714630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19124 , TFLOPS: 70.71513350132243, Tokens per sec: 175239.890731307, Loss: 2.50175404548645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19125 , TFLOPS: 67.1213032968893, Tokens per sec: 166333.98359164284, Loss: 2.5009677410125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19126 , TFLOPS: 71.48209214094062, Tokens per sec: 177140.4987277497, Loss: 2.4836695194244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19127 , TFLOPS: 70.59179012679212, Tokens per sec: 174934.23226182652, Loss: 2.4855282306671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19128 , TFLOPS: 68.14559430075086, Tokens per sec: 168872.28953418066, Loss: 2.5023248195648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19129 , TFLOPS: 70.43369265523734, Tokens per sec: 174542.4493114396, Loss: 2.4862594604492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19130 , TFLOPS: 71.5175723699642, Tokens per sec: 177228.42264374002, Loss: 2.5033888816833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19131 , TFLOPS: 69.09567524844137, Tokens per sec: 171226.69478261404, Loss: 2.4912266731262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19132 , TFLOPS: 68.9125834320954, Tokens per sec: 170772.97309248016, Loss: 2.503505229949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19133 , TFLOPS: 70.57490925561503, Tokens per sec: 174892.39960346822, Loss: 2.5131702423095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19134 , TFLOPS: 70.6019471255883, Tokens per sec: 174959.40242372925, Loss: 2.4937143325805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19135 , TFLOPS: 67.67519807143697, Tokens per sec: 167706.59585951848, Loss: 2.480192184448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19136 , TFLOPS: 71.50186675001203, Tokens per sec: 177189.50238738168, Loss: 2.5033767223358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19137 , TFLOPS: 69.63331451922514, Tokens per sec: 172559.02412147116, Loss: 2.4554710388183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19138 , TFLOPS: 68.02555796208732, Tokens per sec: 168574.82626387812, Loss: 2.477638006210327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19139 , TFLOPS: 70.24627325207295, Tokens per sec: 174078.0033844466, Loss: 2.5124318599700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19140 , TFLOPS: 70.52887966983937, Tokens per sec: 174778.3332193387, Loss: 2.4943623542785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19141 , TFLOPS: 68.16251717143362, Tokens per sec: 168914.22627194066, Loss: 2.478640556335449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19142 , TFLOPS: 70.11101840799545, Tokens per sec: 173742.82698127147, Loss: 2.4983437061309814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19143 , TFLOPS: 71.55271409923427, Tokens per sec: 177315.50772005226, Loss: 2.470562696456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19144 , TFLOPS: 69.72697645059851, Tokens per sec: 172791.12870512786, Loss: 2.4815452098846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19145 , TFLOPS: 67.97524301136164, Tokens per sec: 168450.1402733305, Loss: 2.494830369949341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19146 , TFLOPS: 71.53743354533206, Tokens per sec: 177277.6408241907, Loss: 2.4991962909698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19147 , TFLOPS: 70.60773938637935, Tokens per sec: 174973.75628403967, Loss: 2.503645420074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19148 , TFLOPS: 67.21041715699559, Tokens per sec: 166554.8175536883, Loss: 2.4984803199768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19149 , TFLOPS: 71.49774593363944, Tokens per sec: 177179.290550465, Loss: 2.468693971633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19150 , TFLOPS: 71.51866464426114, Tokens per sec: 177231.12941977175, Loss: 2.493802309036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19151 , TFLOPS: 69.00785151598699, Tokens per sec: 171009.05789900932, Loss: 2.468486785888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19152 , TFLOPS: 67.81040001133748, Tokens per sec: 168041.6411603151, Loss: 2.4815022945404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19153 , TFLOPS: 71.53160377641757, Tokens per sec: 177263.19401461846, Loss: 2.475550889968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19154 , TFLOPS: 70.69804127921935, Tokens per sec: 175197.5343220719, Loss: 2.485318422317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19155 , TFLOPS: 67.24868097358659, Tokens per sec: 166649.6395063082, Loss: 2.4675309658050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19156 , TFLOPS: 71.51105912288911, Tokens per sec: 177212.2820999945, Loss: 2.489837884902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19157 , TFLOPS: 70.58790945671589, Tokens per sec: 174924.61553388656, Loss: 2.482478618621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19158 , TFLOPS: 67.6983836737272, Tokens per sec: 167764.05233609854, Loss: 2.4858975410461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19159 , TFLOPS: 69.76667403769983, Tokens per sec: 172889.50369901516, Loss: 2.4962375164031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19160 , TFLOPS: 71.51654298008762, Tokens per sec: 177225.87170222917, Loss: 2.495929002761841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19161 , TFLOPS: 69.16526331329382, Tokens per sec: 171399.1416730766, Loss: 2.5111122131347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19162 , TFLOPS: 68.06369660034996, Tokens per sec: 168669.33801081075, Loss: 2.4808623790740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19163 , TFLOPS: 71.55654643269311, Tokens per sec: 177325.00466452967, Loss: 2.489365577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19164 , TFLOPS: 70.62445597970367, Tokens per sec: 175015.18184378213, Loss: 2.4961891174316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19165 , TFLOPS: 67.6791081314794, Tokens per sec: 167716.28541903227, Loss: 2.4920854568481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19166 , TFLOPS: 70.4963831496892, Tokens per sec: 174697.80326261936, Loss: 2.4890825748443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19167 , TFLOPS: 71.47589928106046, Tokens per sec: 177125.15213876683, Loss: 2.484147548675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19168 , TFLOPS: 68.31603196985095, Tokens per sec: 169294.65285346986, Loss: 2.484098434448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19169 , TFLOPS: 70.83742897669842, Tokens per sec: 175542.95239124747, Loss: 2.475154399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19170 , TFLOPS: 70.4095671186005, Tokens per sec: 174482.66357400626, Loss: 2.4776618480682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19171 , TFLOPS: 69.78341914419396, Tokens per sec: 172930.99991753334, Loss: 2.4760477542877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19172 , TFLOPS: 68.34690791559517, Tokens per sec: 169371.16684829028, Loss: 2.460716724395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19173 , TFLOPS: 71.47510803801983, Tokens per sec: 177123.19135134952, Loss: 2.504559278488159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19174 , TFLOPS: 69.79530860611057, Tokens per sec: 172960.46331389504, Loss: 2.4678592681884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19175 , TFLOPS: 67.15970591999566, Tokens per sec: 166429.1495220986, Loss: 2.5001938343048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19176 , TFLOPS: 71.51843889457706, Tokens per sec: 177230.56998718626, Loss: 2.4755377769470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19177 , TFLOPS: 69.59932946315104, Tokens per sec: 172474.80540875654, Loss: 2.4753916263580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19178 , TFLOPS: 68.976280532506, Tokens per sec: 170930.82152411249, Loss: 2.516606092453003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19179 , TFLOPS: 70.0340764492995, Tokens per sec: 173552.1563317665, Loss: 2.492595672607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19180 , TFLOPS: 71.50649070657508, Tokens per sec: 177200.96106111576, Loss: 2.4875993728637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19181 , TFLOPS: 68.93484377141824, Tokens per sec: 170828.13666550096, Loss: 2.485473871231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19182 , TFLOPS: 68.38966968134174, Tokens per sec: 169477.13521440764, Loss: 2.4920969009399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19183 , TFLOPS: 71.46342270061642, Tokens per sec: 177094.23379801767, Loss: 2.508387565612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19184 , TFLOPS: 70.58965475072468, Tokens per sec: 174928.94056470034, Loss: 2.487515449523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19185 , TFLOPS: 67.83713134329435, Tokens per sec: 168107.88434560355, Loss: 2.489183187484741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19186 , TFLOPS: 70.9236770892491, Tokens per sec: 175756.68471516197, Loss: 2.451420307159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19187 , TFLOPS: 71.49977544589198, Tokens per sec: 177184.31990539603, Loss: 2.491032123565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19188 , TFLOPS: 68.10507030149483, Tokens per sec: 168771.86659993717, Loss: 2.488149642944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19189 , TFLOPS: 70.63437435967955, Tokens per sec: 175039.7606819621, Loss: 2.512742042541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19190 , TFLOPS: 71.50302707922374, Tokens per sec: 177192.37781098907, Loss: 2.506249189376831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19191 , TFLOPS: 70.57535903566963, Tokens per sec: 174893.5142079909, Loss: 2.4994418621063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19192 , TFLOPS: 67.3515094202805, Tokens per sec: 166904.45972470418, Loss: 2.482574701309204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19193 , TFLOPS: 71.499014283378, Tokens per sec: 177182.43366083739, Loss: 2.49042010307312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19194 , TFLOPS: 70.66708968027966, Tokens per sec: 175120.83285029925, Loss: 2.487414598464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19195 , TFLOPS: 67.40119557205004, Tokens per sec: 167027.5874821706, Loss: 2.4759912490844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19196 , TFLOPS: 70.51322661259542, Tokens per sec: 174739.54321916157, Loss: 2.484445810317993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19197 , TFLOPS: 70.5760510594413, Tokens per sec: 174895.22911912983, Loss: 2.4941577911376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19198 , TFLOPS: 69.54494878866416, Tokens per sec: 172340.0441068522, Loss: 2.4899299144744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19199 , TFLOPS: 67.99173466642226, Tokens per sec: 168491.0084112765, Loss: 2.485048770904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19200 , TFLOPS: 71.51109145164075, Tokens per sec: 177212.3622142024, Loss: 2.484569549560547 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/optimizer.pt +[rank0]:[2024-08-27 18:34:38,348] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006432427006075159, 'preprocessing_with_comm': 0.001223125000251457, 'state_converting': 0.9891935669875238, : 0.99824630501098}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200, took 4.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017200 + eval ppl=10.013998031616211, eval loss=2.3039839267730713 +------------------------------------------------------------------ +iteration: 19201 , TFLOPS: 64.9327072361805, Tokens per sec: 160910.40145944754, Loss: 2.493368625640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19202 , TFLOPS: 68.46603569464027, Tokens per sec: 169666.3786077722, Loss: 2.4960458278656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19203 , TFLOPS: 70.07840811446803, Tokens per sec: 173662.01508159016, Loss: 2.5138823986053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19204 , TFLOPS: 70.57838782443835, Tokens per sec: 174901.01988020842, Loss: 2.504706382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19205 , TFLOPS: 69.7248705858564, Tokens per sec: 172785.91014031673, Loss: 2.501258373260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19206 , TFLOPS: 70.82548703864572, Tokens per sec: 175513.3589532398, Loss: 2.483156204223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19207 , TFLOPS: 70.37049474498122, Tokens per sec: 174385.83792800037, Loss: 2.5066189765930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19208 , TFLOPS: 68.803211786877, Tokens per sec: 170501.93810734764, Loss: 2.4922237396240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19209 , TFLOPS: 69.88201536687966, Tokens per sec: 173175.33221861903, Loss: 2.4831440448760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19210 , TFLOPS: 70.2975461254911, Tokens per sec: 174205.06321295007, Loss: 2.4855024814605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19211 , TFLOPS: 70.61941063129659, Tokens per sec: 175002.67891463838, Loss: 2.4811956882476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19212 , TFLOPS: 67.9300103733122, Tokens per sec: 168338.04881345757, Loss: 2.50788950920105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19213 , TFLOPS: 71.50239445494196, Tokens per sec: 177190.81009832953, Loss: 2.484896183013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19214 , TFLOPS: 69.55588100879216, Tokens per sec: 172367.13535261285, Loss: 2.4887044429779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19215 , TFLOPS: 68.80179996417259, Tokens per sec: 170498.439455161, Loss: 2.485717535018921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19216 , TFLOPS: 71.52347296728387, Tokens per sec: 177243.0449739026, Loss: 2.4757065773010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19217 , TFLOPS: 70.62279845945052, Tokens per sec: 175011.07432600387, Loss: 2.484661102294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19218 , TFLOPS: 68.04425694995183, Tokens per sec: 168621.16441566992, Loss: 2.49275803565979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19219 , TFLOPS: 69.65137346610926, Tokens per sec: 172603.7761812072, Loss: 2.4889378547668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19220 , TFLOPS: 71.50900936312034, Tokens per sec: 177207.20256948782, Loss: 2.4752109050750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19221 , TFLOPS: 69.69766658185058, Tokens per sec: 172718.4956216219, Loss: 2.487051486968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19222 , TFLOPS: 68.65678308660539, Tokens per sec: 170139.07165762162, Loss: 2.495375871658325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19223 , TFLOPS: 71.5340736240288, Tokens per sec: 177269.3145690753, Loss: 2.476799249649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19224 , TFLOPS: 70.64208565713976, Tokens per sec: 175058.87012653807, Loss: 2.500249147415161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19225 , TFLOPS: 68.06462370520694, Tokens per sec: 168671.6354787754, Loss: 2.478206157684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19226 , TFLOPS: 70.69011598382681, Tokens per sec: 175177.89456704582, Loss: 2.5017800331115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19227 , TFLOPS: 71.467999025208, Tokens per sec: 177105.57443447944, Loss: 2.5083909034729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19228 , TFLOPS: 68.79588013295415, Tokens per sec: 170483.76946127845, Loss: 2.5079104900360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19229 , TFLOPS: 68.6677299569371, Tokens per sec: 170166.1992082009, Loss: 2.497872829437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19230 , TFLOPS: 71.5081190468737, Tokens per sec: 177204.99626775223, Loss: 2.4760098457336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19231 , TFLOPS: 70.67280700334294, Tokens per sec: 175135.0010066652, Loss: 2.4954333305358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19232 , TFLOPS: 68.01415096050484, Tokens per sec: 168546.55845737178, Loss: 2.4921791553497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19233 , TFLOPS: 70.3482524364557, Tokens per sec: 174330.7190372816, Loss: 2.4882678985595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19234 , TFLOPS: 70.53451911367813, Tokens per sec: 174792.30838240715, Loss: 2.495450019836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19235 , TFLOPS: 68.77976170636471, Tokens per sec: 170443.82622459857, Loss: 2.496070384979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19236 , TFLOPS: 71.52895627412795, Tokens per sec: 177256.63321229836, Loss: 2.4642343521118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19237 , TFLOPS: 70.40325017415464, Tokens per sec: 174467.00948980052, Loss: 2.4919986724853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19238 , TFLOPS: 70.1442508777166, Tokens per sec: 173825.18070209923, Loss: 2.4951252937316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19239 , TFLOPS: 69.19914770558292, Tokens per sec: 171483.11092984295, Loss: 2.5015711784362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19240 , TFLOPS: 70.48862467324902, Tokens per sec: 174678.57690333386, Loss: 2.4785966873168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19241 , TFLOPS: 70.63790843214545, Tokens per sec: 175048.51850284284, Loss: 2.4739835262298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19242 , TFLOPS: 69.00731006103574, Tokens per sec: 171007.71611399393, Loss: 2.4705393314361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19243 , TFLOPS: 71.52971345513656, Tokens per sec: 177258.50959024805, Loss: 2.484980583190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19244 , TFLOPS: 69.53806568734623, Tokens per sec: 172322.9870235502, Loss: 2.4829604625701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19245 , TFLOPS: 69.01938805376224, Tokens per sec: 171037.64671047093, Loss: 2.4619076251983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19246 , TFLOPS: 70.18065376178868, Tokens per sec: 173915.39105893858, Loss: 2.4915530681610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19247 , TFLOPS: 70.37654524739995, Tokens per sec: 174400.8317395119, Loss: 2.5031633377075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19248 , TFLOPS: 69.61119033935022, Tokens per sec: 172504.19796656145, Loss: 2.4745852947235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19249 , TFLOPS: 69.04992045048895, Tokens per sec: 171113.3093529809, Loss: 2.470263957977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19250 , TFLOPS: 71.5069871300471, Tokens per sec: 177202.191253165, Loss: 2.47341251373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19251 , TFLOPS: 69.7627227118816, Tokens per sec: 172879.71187836307, Loss: 2.494903087615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19252 , TFLOPS: 68.20248265083276, Tokens per sec: 169013.26513245274, Loss: 2.5068793296813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19253 , TFLOPS: 71.44795275280877, Tokens per sec: 177055.89756319489, Loss: 2.479043483734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19254 , TFLOPS: 71.54936032321547, Tokens per sec: 177307.19669362876, Loss: 2.5026731491088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19255 , TFLOPS: 68.0799332347073, Tokens per sec: 168709.5741793612, Loss: 2.492967367172241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19256 , TFLOPS: 70.7047982051995, Tokens per sec: 175214.27872898753, Loss: 2.482297658920288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19257 , TFLOPS: 71.5388582193403, Tokens per sec: 177281.17132332386, Loss: 2.4815943241119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19258 , TFLOPS: 70.61669836519968, Tokens per sec: 174995.95762046945, Loss: 2.47745418548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19259 , TFLOPS: 68.27032615185969, Tokens per sec: 169181.38880157605, Loss: 2.511723279953003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19260 , TFLOPS: 71.47192642490174, Tokens per sec: 177115.3069635593, Loss: 2.497509479522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19261 , TFLOPS: 70.63434711592951, Tokens per sec: 175039.69316894838, Loss: 2.5014963150024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19262 , TFLOPS: 68.5245575409654, Tokens per sec: 169811.40218967947, Loss: 2.503167152404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19263 , TFLOPS: 71.50815816273783, Tokens per sec: 177205.0932011725, Loss: 2.490097999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19264 , TFLOPS: 70.67431539429506, Tokens per sec: 175138.73896560754, Loss: 2.4900753498077393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19265 , TFLOPS: 69.83636962424215, Tokens per sec: 173062.2170400697, Loss: 2.4635283946990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19266 , TFLOPS: 69.09276456023491, Tokens per sec: 171219.48177659966, Loss: 2.49259352684021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19267 , TFLOPS: 71.5531196340677, Tokens per sec: 177316.51268004288, Loss: 2.5133180618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19268 , TFLOPS: 69.65171975354022, Tokens per sec: 172604.63431960784, Loss: 2.4762606620788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19269 , TFLOPS: 68.71329340779366, Tokens per sec: 170279.11045865194, Loss: 2.4922475814819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19270 , TFLOPS: 70.41605163032962, Tokens per sec: 174498.73290839893, Loss: 2.50714111328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19271 , TFLOPS: 70.70194283992534, Tokens per sec: 175207.2028192937, Loss: 2.48710036277771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19272 , TFLOPS: 68.8858757452, Tokens per sec: 170706.78850225877, Loss: 2.4749908447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19273 , TFLOPS: 70.85071490390048, Tokens per sec: 175575.8763824199, Loss: 2.4740850925445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19274 , TFLOPS: 70.5010112987147, Tokens per sec: 174709.27232573726, Loss: 2.4932005405426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19275 , TFLOPS: 68.90259341345005, Tokens per sec: 170748.21672578558, Loss: 2.5005431175231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19276 , TFLOPS: 70.49618209312456, Tokens per sec: 174697.30502230395, Loss: 2.5003902912139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19277 , TFLOPS: 70.15331740498196, Tokens per sec: 173847.64855542328, Loss: 2.4989817142486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19278 , TFLOPS: 70.66954938115813, Tokens per sec: 175126.92826003453, Loss: 2.50536847114563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19279 , TFLOPS: 68.17814686780103, Tokens per sec: 168952.9583812923, Loss: 2.5011587142944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19280 , TFLOPS: 71.47053304149235, Tokens per sec: 177111.85400597696, Loss: 2.4727063179016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19281 , TFLOPS: 69.65811558809808, Tokens per sec: 172620.48390220158, Loss: 2.487197160720825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19282 , TFLOPS: 68.70797448990747, Tokens per sec: 170265.92959420278, Loss: 2.4716291427612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19283 , TFLOPS: 71.51497060350233, Tokens per sec: 177221.97517983927, Loss: 2.491947650909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19284 , TFLOPS: 70.451000809138, Tokens per sec: 174585.34082345598, Loss: 2.5078988075256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19285 , TFLOPS: 69.24065032524372, Tokens per sec: 171585.95899325336, Loss: 2.4905200004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19286 , TFLOPS: 69.73412531247965, Tokens per sec: 172808.84437238146, Loss: 2.5121564865112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19287 , TFLOPS: 71.47931580146881, Tokens per sec: 177133.61865269952, Loss: 2.490203619003296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19288 , TFLOPS: 69.66890152009204, Tokens per sec: 172647.2126298511, Loss: 2.5004382133483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19289 , TFLOPS: 68.35467096077687, Tokens per sec: 169390.40452942054, Loss: 2.4912753105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19290 , TFLOPS: 71.52653238032698, Tokens per sec: 177250.6265364456, Loss: 2.495553731918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19291 , TFLOPS: 70.57502007985914, Tokens per sec: 174892.67423815347, Loss: 2.478811740875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19292 , TFLOPS: 67.70522104182174, Tokens per sec: 167780.99608743514, Loss: 2.4990928173065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19293 , TFLOPS: 70.49567293750442, Tokens per sec: 174696.04327858918, Loss: 2.4946439266204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19294 , TFLOPS: 71.5119136763955, Tokens per sec: 177214.39977772022, Loss: 2.504054069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19295 , TFLOPS: 68.96243682351646, Tokens per sec: 170896.51528822596, Loss: 2.470668315887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19296 , TFLOPS: 70.49187276009928, Tokens per sec: 174686.62602035515, Loss: 2.490752696990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19297 , TFLOPS: 71.57007080188168, Tokens per sec: 177358.51954121122, Loss: 2.4781713485717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19298 , TFLOPS: 70.65917212433011, Tokens per sec: 175101.2122744655, Loss: 2.4780871868133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19299 , TFLOPS: 66.93193784071862, Tokens per sec: 165864.71513092486, Loss: 2.4860172271728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19300 , TFLOPS: 71.44906459712587, Tokens per sec: 177058.65283589246, Loss: 2.49350643157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19301 , TFLOPS: 70.7213723940551, Tokens per sec: 175255.35139476936, Loss: 2.482935905456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19302 , TFLOPS: 68.86264128284137, Tokens per sec: 170649.21094504677, Loss: 2.506300449371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19303 , TFLOPS: 70.45830017891159, Tokens per sec: 174603.4294658469, Loss: 2.516752004623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19304 , TFLOPS: 71.51239770473707, Tokens per sec: 177215.59925327075, Loss: 2.482983350753784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19305 , TFLOPS: 69.67143525940348, Tokens per sec: 172653.49151497928, Loss: 2.477433204650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19306 , TFLOPS: 69.14361913534088, Tokens per sec: 171345.5050158062, Loss: 2.4794869422912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19307 , TFLOPS: 70.48580740405372, Tokens per sec: 174671.59539992007, Loss: 2.4919273853302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19308 , TFLOPS: 70.71342631185252, Tokens per sec: 175235.66012773817, Loss: 2.4900896549224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19309 , TFLOPS: 67.96784593348758, Tokens per sec: 168431.80949950317, Loss: 2.5015697479248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19310 , TFLOPS: 71.50732789198132, Tokens per sec: 177203.03569877602, Loss: 2.5019497871398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19311 , TFLOPS: 69.50222725596012, Tokens per sec: 172234.1754426463, Loss: 2.490722417831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19312 , TFLOPS: 68.89609966743645, Tokens per sec: 170732.1245078187, Loss: 2.488220691680908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19313 , TFLOPS: 70.60776959464364, Tokens per sec: 174973.83114344632, Loss: 2.499948024749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19314 , TFLOPS: 70.52114167450819, Tokens per sec: 174759.15761449747, Loss: 2.4884250164031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19315 , TFLOPS: 69.84378141750187, Tokens per sec: 173080.5842802437, Loss: 2.498929500579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19316 , TFLOPS: 69.93838808729848, Tokens per sec: 173315.03002978468, Loss: 2.513065814971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19317 , TFLOPS: 71.48644597503332, Tokens per sec: 177151.2880082458, Loss: 2.4828433990478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19318 , TFLOPS: 69.50849447903063, Tokens per sec: 172249.7063118066, Loss: 2.4866936206817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19319 , TFLOPS: 68.04903180530475, Tokens per sec: 168632.9970332282, Loss: 2.473844528198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19320 , TFLOPS: 71.5088265961874, Tokens per sec: 177206.7496528946, Loss: 2.4758076667785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19321 , TFLOPS: 70.50980812111648, Tokens per sec: 174731.07182069885, Loss: 2.4750611782073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19322 , TFLOPS: 69.30399350145542, Tokens per sec: 171742.93036173287, Loss: 2.5098936557769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19323 , TFLOPS: 70.81105106405974, Tokens per sec: 175477.5850179619, Loss: 2.4930691719055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19324 , TFLOPS: 71.52302708032619, Tokens per sec: 177241.94001690295, Loss: 2.476020574569702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19325 , TFLOPS: 68.87435543563802, Tokens per sec: 170678.23990609733, Loss: 2.4846293926239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19326 , TFLOPS: 68.1880984808527, Tokens per sec: 168977.619574695, Loss: 2.4871065616607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19327 , TFLOPS: 71.48475945564665, Tokens per sec: 177147.10862742024, Loss: 2.4870617389678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19328 , TFLOPS: 70.69893302498194, Tokens per sec: 175199.7441663055, Loss: 2.501434326171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19329 , TFLOPS: 67.09758088048535, Tokens per sec: 166275.196830554, Loss: 2.502817153930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19330 , TFLOPS: 71.53531428544082, Tokens per sec: 177272.3890647244, Loss: 2.4833216667175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19331 , TFLOPS: 70.58122780824452, Tokens per sec: 174908.05767293015, Loss: 2.4805219173431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19332 , TFLOPS: 69.74821426622626, Tokens per sec: 172843.7583517919, Loss: 2.4926793575286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19333 , TFLOPS: 69.75395295671383, Tokens per sec: 172857.9794589891, Loss: 2.486987590789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19334 , TFLOPS: 71.56119615175317, Tokens per sec: 177336.52718056933, Loss: 2.493802785873413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19335 , TFLOPS: 69.887942377526, Tokens per sec: 173190.0200039148, Loss: 2.5061352252960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19336 , TFLOPS: 68.58176951424781, Tokens per sec: 169953.17976189242, Loss: 2.480008125305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19337 , TFLOPS: 71.52131572302238, Tokens per sec: 177237.69908498257, Loss: 2.48997163772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19338 , TFLOPS: 70.59225396737124, Tokens per sec: 174935.38170987787, Loss: 2.4828033447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19339 , TFLOPS: 68.89221318563793, Tokens per sec: 170722.49337779064, Loss: 2.4789071083068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19340 , TFLOPS: 69.78491836019113, Tokens per sec: 172934.71513992661, Loss: 2.4802160263061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19341 , TFLOPS: 71.50352569517838, Tokens per sec: 177193.61343625176, Loss: 2.4804258346557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19342 , TFLOPS: 69.09217961910625, Tokens per sec: 171218.03222804627, Loss: 2.498901605606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19343 , TFLOPS: 70.48553756719093, Tokens per sec: 174670.9267144491, Loss: 2.4913976192474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19344 , TFLOPS: 70.5197141987584, Tokens per sec: 174755.62017234004, Loss: 2.478461980819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19345 , TFLOPS: 70.50312548590279, Tokens per sec: 174714.51151448628, Loss: 2.4920129776000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19346 , TFLOPS: 67.95155273645742, Tokens per sec: 168391.4331624186, Loss: 2.4904537200927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19347 , TFLOPS: 71.5084597037854, Tokens per sec: 177205.8404531059, Loss: 2.4820282459259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19348 , TFLOPS: 69.73285600122986, Tokens per sec: 172805.6988792783, Loss: 2.4713003635406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19349 , TFLOPS: 68.1299386517211, Tokens per sec: 168833.4931112728, Loss: 2.47586727142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19350 , TFLOPS: 71.39041743247496, Tokens per sec: 176913.31870136896, Loss: 2.4965438842773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19351 , TFLOPS: 69.47549767914282, Tokens per sec: 172167.93660678703, Loss: 2.5181634426116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19352 , TFLOPS: 70.47894084712621, Tokens per sec: 174654.57931544006, Loss: 2.4627537727355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19353 , TFLOPS: 69.0893108879586, Tokens per sec: 171210.9231962456, Loss: 2.5042223930358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19354 , TFLOPS: 71.51708821337085, Tokens per sec: 177227.22285036743, Loss: 2.4936468601226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19355 , TFLOPS: 68.9400926101598, Tokens per sec: 170841.14386610984, Loss: 2.4829587936401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19356 , TFLOPS: 68.76299373191635, Tokens per sec: 170402.27333677065, Loss: 2.4988179206848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19357 , TFLOPS: 71.53053331518772, Tokens per sec: 177260.5412937699, Loss: 2.4787936210632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19358 , TFLOPS: 69.55873616060197, Tokens per sec: 172374.21073331853, Loss: 2.4718899726867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19359 , TFLOPS: 68.93642459349246, Tokens per sec: 170832.0541167428, Loss: 2.5028936862945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19360 , TFLOPS: 70.62373366946284, Tokens per sec: 175013.3918794363, Loss: 2.4780030250549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19361 , TFLOPS: 71.49209577319367, Tokens per sec: 177165.2888304638, Loss: 2.4916574954986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19362 , TFLOPS: 68.92805139703212, Tokens per sec: 170811.30441353034, Loss: 2.4756102561950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19363 , TFLOPS: 69.68551137356741, Tokens per sec: 172688.3736764895, Loss: 2.4871933460235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19364 , TFLOPS: 71.49212283327131, Tokens per sec: 177165.3558883171, Loss: 2.4924237728118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19365 , TFLOPS: 70.61173264885981, Tokens per sec: 174983.65202836046, Loss: 2.488846778869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19366 , TFLOPS: 67.66524628467891, Tokens per sec: 167681.93423565218, Loss: 2.476325273513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19367 , TFLOPS: 71.52407019408837, Tokens per sec: 177244.52496771413, Loss: 2.493696928024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19368 , TFLOPS: 70.643545662902, Tokens per sec: 175062.4881816498, Loss: 2.5023350715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19369 , TFLOPS: 69.00102546642408, Tokens per sec: 170992.14219624098, Loss: 2.5168440341949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19370 , TFLOPS: 69.90555920982503, Tokens per sec: 173233.67645500612, Loss: 2.477642059326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19371 , TFLOPS: 71.50796084530218, Tokens per sec: 177204.60422683068, Loss: 2.486668825149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19372 , TFLOPS: 69.8681333560157, Tokens per sec: 173140.93106646842, Loss: 2.4656660556793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19373 , TFLOPS: 67.89720842713349, Tokens per sec: 168256.76197739405, Loss: 2.509875774383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19374 , TFLOPS: 71.51326799707341, Tokens per sec: 177217.7559335508, Loss: 2.4932141304016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19375 , TFLOPS: 70.68027712404614, Tokens per sec: 175153.51278866947, Loss: 2.5060341358184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19376 , TFLOPS: 69.01709634818123, Tokens per sec: 171031.96761158895, Loss: 2.488675594329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19377 , TFLOPS: 70.52310537492862, Tokens per sec: 174764.0238804588, Loss: 2.506868600845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19378 , TFLOPS: 70.57092230128643, Tokens per sec: 174882.51949144268, Loss: 2.4888603687286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19379 , TFLOPS: 68.9464782794229, Tokens per sec: 170856.96825798336, Loss: 2.5030057430267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19380 , TFLOPS: 70.55748127168786, Tokens per sec: 174849.2111394458, Loss: 2.496804714202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19381 , TFLOPS: 70.42308323366605, Tokens per sec: 174516.1579960617, Loss: 2.4935216903686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19382 , TFLOPS: 69.88247478802957, Tokens per sec: 173176.47071483795, Loss: 2.49765682220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19383 , TFLOPS: 69.16081590106917, Tokens per sec: 171388.12049566102, Loss: 2.4759819507598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19384 , TFLOPS: 71.51588259431121, Tokens per sec: 177224.23519352774, Loss: 2.505720615386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19385 , TFLOPS: 69.67813832202607, Tokens per sec: 172670.10244256136, Loss: 2.510349988937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19386 , TFLOPS: 67.89840965757445, Tokens per sec: 168259.73875875323, Loss: 2.4842429161071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19387 , TFLOPS: 71.45438705144515, Tokens per sec: 177071.84246401236, Loss: 2.4853458404541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19388 , TFLOPS: 70.41507301670569, Tokens per sec: 174496.3077960357, Loss: 2.4947376251220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19389 , TFLOPS: 69.23250895747744, Tokens per sec: 171565.78378708282, Loss: 2.4956727027893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19390 , TFLOPS: 70.11378208126814, Tokens per sec: 173749.675668085, Loss: 2.4769785404205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19391 , TFLOPS: 71.48642285776378, Tokens per sec: 177151.23072110533, Loss: 2.4815313816070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19392 , TFLOPS: 69.73267934455428, Tokens per sec: 172805.26110457588, Loss: 2.480896472930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19393 , TFLOPS: 68.18792385605536, Tokens per sec: 168977.18683521068, Loss: 2.4877028465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19394 , TFLOPS: 71.52717838128048, Tokens per sec: 177252.22739797123, Loss: 2.47415828704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19395 , TFLOPS: 69.59425917590096, Tokens per sec: 172462.2406784126, Loss: 2.495858669281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19396 , TFLOPS: 68.76371802858779, Tokens per sec: 170404.06822371035, Loss: 2.4893763065338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19397 , TFLOPS: 71.50457691240199, Tokens per sec: 177196.218468333, Loss: 2.486776828765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19398 , TFLOPS: 70.70167353546842, Tokens per sec: 175206.53545318317, Loss: 2.500002861022949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19399 , TFLOPS: 69.51510760798155, Tokens per sec: 172266.09437384328, Loss: 2.480933427810669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19400 , TFLOPS: 70.04701135965085, Tokens per sec: 173584.21046451514, Loss: 2.507518768310547 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/optimizer.pt +[rank0]:[2024-08-27 18:59:57,859] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006376083998475224, 'preprocessing_with_comm': 0.0013826210051774979, 'state_converting': 0.9922046489955392, : 1.0013572860043496}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017400 + eval ppl=10.401763916015625, eval loss=2.341975450515747 +------------------------------------------------------------------ +iteration: 19401 , TFLOPS: 66.17505841539399, Tokens per sec: 163989.08453781877, Loss: 2.4833831787109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19402 , TFLOPS: 69.6799451410901, Tokens per sec: 172674.57993923957, Loss: 2.486895799636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19403 , TFLOPS: 68.35404967422353, Tokens per sec: 169388.864912901, Loss: 2.480008125305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19404 , TFLOPS: 71.52564823714208, Tokens per sec: 177248.435532237, Loss: 2.500293731689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19405 , TFLOPS: 70.726123508751, Tokens per sec: 175267.12518036432, Loss: 2.490525722503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19406 , TFLOPS: 68.007034648624, Tokens per sec: 168528.92345260456, Loss: 2.492798089981079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19407 , TFLOPS: 70.39543506512572, Tokens per sec: 174447.64278872154, Loss: 2.483713388442993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19408 , TFLOPS: 71.42785319726637, Tokens per sec: 177006.08865041204, Loss: 2.478318214416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19409 , TFLOPS: 69.72991611574005, Tokens per sec: 172798.41351917945, Loss: 2.4629979133605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19410 , TFLOPS: 68.91420298049655, Tokens per sec: 170776.98651182622, Loss: 2.515199661254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19411 , TFLOPS: 71.4760414767451, Tokens per sec: 177125.5045153375, Loss: 2.4910035133361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19412 , TFLOPS: 70.6813679214759, Tokens per sec: 175156.21590486163, Loss: 2.495034694671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19413 , TFLOPS: 68.41189256556244, Tokens per sec: 169532.20596955932, Loss: 2.5014443397521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19414 , TFLOPS: 70.38493141985965, Tokens per sec: 174421.61359868018, Loss: 2.4929327964782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19415 , TFLOPS: 70.5456416397597, Tokens per sec: 174819.8712272847, Loss: 2.5047197341918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19416 , TFLOPS: 67.95786872361361, Tokens per sec: 168407.0848743551, Loss: 2.474661350250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19417 , TFLOPS: 71.49283807936561, Tokens per sec: 177167.12834692968, Loss: 2.4971513748168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19418 , TFLOPS: 70.3783884503666, Tokens per sec: 174405.3993995088, Loss: 2.4900119304656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19419 , TFLOPS: 70.0983630919112, Tokens per sec: 173711.46571391658, Loss: 2.469916820526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19420 , TFLOPS: 68.97885506478457, Tokens per sec: 170937.20149870694, Loss: 2.4872665405273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19421 , TFLOPS: 70.40023465657487, Tokens per sec: 174459.53670505117, Loss: 2.4953088760375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19422 , TFLOPS: 69.73158822370208, Tokens per sec: 172802.55718690722, Loss: 2.487070083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19423 , TFLOPS: 68.07558827042048, Tokens per sec: 168698.8068792211, Loss: 2.4921212196350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19424 , TFLOPS: 71.52594365453098, Tokens per sec: 177249.16760906958, Loss: 2.482083320617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19425 , TFLOPS: 69.62976638636393, Tokens per sec: 172550.23145738197, Loss: 2.4870357513427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19426 , TFLOPS: 68.77289653229427, Tokens per sec: 170426.81356699063, Loss: 2.490008592605591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19427 , TFLOPS: 70.78421070634607, Tokens per sec: 175411.07165483793, Loss: 2.4889512062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19428 , TFLOPS: 71.55474034115305, Tokens per sec: 177320.528970736, Loss: 2.4950833320617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19429 , TFLOPS: 68.7299273059176, Tokens per sec: 170320.33109057823, Loss: 2.5012245178222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19430 , TFLOPS: 69.74402718259314, Tokens per sec: 172833.38229730373, Loss: 2.516608238220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19431 , TFLOPS: 71.54089355615255, Tokens per sec: 177286.21511215545, Loss: 2.5034499168395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19432 , TFLOPS: 69.5838575655908, Tokens per sec: 172436.46434223207, Loss: 2.4976487159729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19433 , TFLOPS: 67.85854737089501, Tokens per sec: 168160.9556211682, Loss: 2.4825241565704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19434 , TFLOPS: 71.4409082823594, Tokens per sec: 177038.44058940888, Loss: 2.4782512187957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19435 , TFLOPS: 71.5254208682131, Tokens per sec: 177247.87208698425, Loss: 2.4705705642700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19436 , TFLOPS: 67.93393408663432, Tokens per sec: 168347.77220730248, Loss: 2.482590675354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19437 , TFLOPS: 70.97240892643642, Tokens per sec: 175877.44757596718, Loss: 2.485257387161255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19438 , TFLOPS: 71.4736863271476, Tokens per sec: 177119.66819519378, Loss: 2.488114595413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19439 , TFLOPS: 69.84261678501346, Tokens per sec: 173077.69819264257, Loss: 2.4940528869628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19440 , TFLOPS: 68.72959290750285, Tokens per sec: 170319.50241446943, Loss: 2.4770328998565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19441 , TFLOPS: 71.52398165644185, Tokens per sec: 177244.30556167278, Loss: 2.4915409088134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19442 , TFLOPS: 70.56823924870517, Tokens per sec: 174875.8705915837, Loss: 2.489670991897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19443 , TFLOPS: 68.29853658369422, Tokens per sec: 169251.2973592976, Loss: 2.4881956577301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19444 , TFLOPS: 70.38590360012401, Tokens per sec: 174424.02276846976, Loss: 2.4942522048950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19445 , TFLOPS: 70.70889664427365, Tokens per sec: 175224.43511249364, Loss: 2.4494869709014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19446 , TFLOPS: 69.58190923372251, Tokens per sec: 172431.63616123595, Loss: 2.4768433570861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19447 , TFLOPS: 68.9346069418494, Tokens per sec: 170827.54977574042, Loss: 2.486783027648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19448 , TFLOPS: 71.46113494617498, Tokens per sec: 177088.56449049484, Loss: 2.5082192420959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19449 , TFLOPS: 69.72119738309289, Tokens per sec: 172776.80753923208, Loss: 2.4847664833068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19450 , TFLOPS: 69.30695102885232, Tokens per sec: 171750.25944042095, Loss: 2.4921624660491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19451 , TFLOPS: 70.48567671270736, Tokens per sec: 174671.27153236698, Loss: 2.4974234104156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19452 , TFLOPS: 70.71403264200289, Tokens per sec: 175237.16268064355, Loss: 2.4941835403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19453 , TFLOPS: 68.22237450634303, Tokens per sec: 169062.55934167595, Loss: 2.4972574710845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19454 , TFLOPS: 71.53128761080839, Tokens per sec: 177262.4105214104, Loss: 2.4858837127685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19455 , TFLOPS: 70.48832208986187, Tokens per sec: 174677.8270683682, Loss: 2.4751439094543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19456 , TFLOPS: 67.88498051305113, Tokens per sec: 168226.459859282, Loss: 2.4980578422546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19457 , TFLOPS: 70.60164018887245, Tokens per sec: 174958.64180073552, Loss: 2.4753799438476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19458 , TFLOPS: 71.52432493810727, Tokens per sec: 177245.1562514553, Loss: 2.465916156768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19459 , TFLOPS: 68.74411934550746, Tokens per sec: 170355.50052806412, Loss: 2.467811107635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19460 , TFLOPS: 69.05648007550155, Tokens per sec: 171129.56482636448, Loss: 2.497891902923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19461 , TFLOPS: 71.48600730624757, Tokens per sec: 177150.2009386714, Loss: 2.489551544189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19462 , TFLOPS: 69.65288010844486, Tokens per sec: 172607.50980688518, Loss: 2.478623390197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19463 , TFLOPS: 67.80251070210205, Tokens per sec: 168022.09058295056, Loss: 2.4993953704833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19464 , TFLOPS: 71.51988216375365, Tokens per sec: 177234.14656719504, Loss: 2.475715398788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19465 , TFLOPS: 70.61090489815433, Tokens per sec: 174981.60077092802, Loss: 2.4866390228271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19466 , TFLOPS: 68.80371130034453, Tokens per sec: 170503.1759567461, Loss: 2.489633798599243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19467 , TFLOPS: 69.76808899789704, Tokens per sec: 172893.01012625455, Loss: 2.488224506378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19468 , TFLOPS: 71.5240753732598, Tokens per sec: 177244.53780227146, Loss: 2.4994263648986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19469 , TFLOPS: 68.86553739732977, Tokens per sec: 170656.38783578237, Loss: 2.491905927658081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19470 , TFLOPS: 69.60552969153372, Tokens per sec: 172490.17025770035, Loss: 2.48508620262146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19471 , TFLOPS: 71.53960914285989, Tokens per sec: 177283.03219452646, Loss: 2.4627790451049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19472 , TFLOPS: 70.6000432014846, Tokens per sec: 174954.68429006575, Loss: 2.4830923080444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19473 , TFLOPS: 67.9355751808934, Tokens per sec: 168351.83901965793, Loss: 2.491856813430786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19474 , TFLOPS: 71.52120380103726, Tokens per sec: 177237.42172997422, Loss: 2.482515811920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19475 , TFLOPS: 71.4528858453049, Tokens per sec: 177068.12230982532, Loss: 2.490962505340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19476 , TFLOPS: 68.91366650126658, Tokens per sec: 170775.65705719625, Loss: 2.4961657524108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19477 , TFLOPS: 69.80265173732072, Tokens per sec: 172978.6604019465, Loss: 2.4759557247161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19478 , TFLOPS: 71.4733709723852, Tokens per sec: 177118.88671135332, Loss: 2.481607437133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19479 , TFLOPS: 70.63682078512524, Tokens per sec: 175045.82319371257, Loss: 2.4984307289123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19480 , TFLOPS: 68.38907334213873, Tokens per sec: 169475.65742017038, Loss: 2.4857139587402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19481 , TFLOPS: 70.43286933575529, Tokens per sec: 174540.40903506693, Loss: 2.468334913253784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19482 , TFLOPS: 70.64844844835683, Tokens per sec: 175074.63782409328, Loss: 2.4774534702301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19483 , TFLOPS: 67.68714344689468, Tokens per sec: 167736.1978157944, Loss: 2.4647631645202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19484 , TFLOPS: 70.44666420877779, Tokens per sec: 174574.59425004743, Loss: 2.51759672164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19485 , TFLOPS: 71.48484683438129, Tokens per sec: 177147.32516155022, Loss: 2.5053253173828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19486 , TFLOPS: 70.02373191368403, Tokens per sec: 173526.5214329667, Loss: 2.4664061069488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19487 , TFLOPS: 68.892292772542, Tokens per sec: 170722.69060290616, Loss: 2.4687185287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19488 , TFLOPS: 70.41035543393306, Tokens per sec: 174484.61710624045, Loss: 2.487082004547119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19489 , TFLOPS: 69.7735614791767, Tokens per sec: 172906.57153770915, Loss: 2.478928804397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19490 , TFLOPS: 69.24674106286155, Tokens per sec: 171601.05251202048, Loss: 2.5005381107330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19491 , TFLOPS: 71.48654388908794, Tokens per sec: 177151.53065005929, Loss: 2.4794862270355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19492 , TFLOPS: 69.56936796045235, Tokens per sec: 172400.5575045372, Loss: 2.4844589233398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19493 , TFLOPS: 68.81145225750707, Tokens per sec: 170522.35890133173, Loss: 2.491361618041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19494 , TFLOPS: 70.87303121954568, Tokens per sec: 175631.17867093382, Loss: 2.503202199935913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19495 , TFLOPS: 70.41252338987096, Tokens per sec: 174489.9895398746, Loss: 2.480903148651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19496 , TFLOPS: 69.81480562913275, Tokens per sec: 173008.7790847184, Loss: 2.521824836730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19497 , TFLOPS: 69.20921548596183, Tokens per sec: 171508.05999867944, Loss: 2.495931386947632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19498 , TFLOPS: 71.48898780623843, Tokens per sec: 177157.58694597788, Loss: 2.491283893585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19499 , TFLOPS: 70.50123325276871, Tokens per sec: 174709.82235233334, Loss: 2.487372636795044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19500 , TFLOPS: 68.06613506152294, Tokens per sec: 168675.3807862161, Loss: 2.4984424114227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19501 , TFLOPS: 71.46000003105311, Tokens per sec: 177085.75204580164, Loss: 2.4949262142181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19502 , TFLOPS: 71.51143142199174, Tokens per sec: 177213.20469818302, Loss: 2.51128888130188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19503 , TFLOPS: 67.28887370326535, Tokens per sec: 166749.24151804685, Loss: 2.5025389194488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19504 , TFLOPS: 70.955489656169, Tokens per sec: 175835.51976043303, Loss: 2.478029251098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19505 , TFLOPS: 71.52290802211859, Tokens per sec: 177241.64497754918, Loss: 2.495534896850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19506 , TFLOPS: 69.46378389413772, Tokens per sec: 172138.90855716448, Loss: 2.487919569015503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19507 , TFLOPS: 68.78444420952803, Tokens per sec: 170455.42998325755, Loss: 2.4588255882263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19508 , TFLOPS: 71.50233889997226, Tokens per sec: 177190.67242699474, Loss: 2.4906301498413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19509 , TFLOPS: 69.78752159968111, Tokens per sec: 172941.1662541533, Loss: 2.4895706176757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19510 , TFLOPS: 69.31957861898397, Tokens per sec: 171781.55199981105, Loss: 2.4942545890808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19511 , TFLOPS: 71.52984925479265, Tokens per sec: 177258.8461167554, Loss: 2.484456777572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19512 , TFLOPS: 70.57252597721012, Tokens per sec: 174886.4935770411, Loss: 2.491969585418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19513 , TFLOPS: 69.67200726601271, Tokens per sec: 172654.90901036866, Loss: 2.479344606399536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19514 , TFLOPS: 68.92203311673218, Tokens per sec: 170796.3904519788, Loss: 2.489051103591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19515 , TFLOPS: 71.53342106203289, Tokens per sec: 177267.6974485638, Loss: 2.5011374950408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19516 , TFLOPS: 69.78801444125791, Tokens per sec: 172942.3875698716, Loss: 2.504577875137329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19517 , TFLOPS: 68.7257758245582, Tokens per sec: 170310.0432624462, Loss: 2.505250930786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19518 , TFLOPS: 70.4444191656785, Tokens per sec: 174569.03078593547, Loss: 2.504680871963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19519 , TFLOPS: 70.61600581655608, Tokens per sec: 174994.24140864008, Loss: 2.4945549964904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19520 , TFLOPS: 68.06203804966033, Tokens per sec: 168665.22793949742, Loss: 2.4862520694732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19521 , TFLOPS: 69.2707422378257, Tokens per sec: 171660.53006752933, Loss: 2.4923923015594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19522 , TFLOPS: 71.51408261170342, Tokens per sec: 177219.7746383412, Loss: 2.4975733757019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19523 , TFLOPS: 68.94948344698827, Tokens per sec: 170864.41539425688, Loss: 2.4751203060150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19524 , TFLOPS: 69.84030301232896, Tokens per sec: 173071.96440904724, Loss: 2.4828457832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19525 , TFLOPS: 70.3181350658286, Tokens per sec: 174256.0848751654, Loss: 2.481316566467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19526 , TFLOPS: 69.71618194969439, Tokens per sec: 172764.37874277498, Loss: 2.496706247329712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19527 , TFLOPS: 68.55597795318036, Tokens per sec: 169889.26543239225, Loss: 2.4795026779174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19528 , TFLOPS: 71.49720245309643, Tokens per sec: 177177.9437458101, Loss: 2.511505126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19529 , TFLOPS: 69.14032665852073, Tokens per sec: 171337.34589554957, Loss: 2.4897546768188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19530 , TFLOPS: 69.04996980143346, Tokens per sec: 171113.43165005784, Loss: 2.4964077472686768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19531 , TFLOPS: 71.53908633062534, Tokens per sec: 177281.7366082167, Loss: 2.487830400466919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19532 , TFLOPS: 71.51826439788935, Tokens per sec: 177230.1375651717, Loss: 2.494119882583618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19533 , TFLOPS: 68.84351001863214, Tokens per sec: 170601.80156485358, Loss: 2.485445499420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19534 , TFLOPS: 69.72528119386142, Tokens per sec: 172786.9276721854, Loss: 2.514869451522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19535 , TFLOPS: 71.50499564425125, Tokens per sec: 177197.2561319826, Loss: 2.4823639392852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19536 , TFLOPS: 68.5851326566389, Tokens per sec: 169961.51399921824, Loss: 2.4623467922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19537 , TFLOPS: 68.94951141803493, Tokens per sec: 170864.48470959178, Loss: 2.4926223754882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19538 , TFLOPS: 71.45283489002863, Tokens per sec: 177067.99603703752, Loss: 2.4943668842315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19539 , TFLOPS: 70.6027219226275, Tokens per sec: 174961.32245614333, Loss: 2.503762722015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19540 , TFLOPS: 67.9080617685376, Tokens per sec: 168283.6577529877, Loss: 2.464890956878662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19541 , TFLOPS: 71.55335664645177, Tokens per sec: 177317.10002283967, Loss: 2.4693443775177 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19542 , TFLOPS: 71.49344619611654, Tokens per sec: 177168.63532722805, Loss: 2.4856128692626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19543 , TFLOPS: 68.02842767674464, Tokens per sec: 168581.9377329241, Loss: 2.4824748039245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19544 , TFLOPS: 69.65502430433855, Tokens per sec: 172612.82336051244, Loss: 2.5005996227264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19545 , TFLOPS: 71.51788632751229, Tokens per sec: 177229.20066512944, Loss: 2.516333818435669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19546 , TFLOPS: 70.62728376810753, Tokens per sec: 175022.1894149535, Loss: 2.4894397258758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19547 , TFLOPS: 67.31534001935265, Tokens per sec: 166814.82796481656, Loss: 2.4945714473724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19548 , TFLOPS: 71.4862309016549, Tokens per sec: 177150.75503272182, Loss: 2.4830503463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19549 , TFLOPS: 70.67522676921705, Tokens per sec: 175140.99745306044, Loss: 2.4834671020507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19550 , TFLOPS: 68.81022738433998, Tokens per sec: 170519.32353069272, Loss: 2.4905147552490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19551 , TFLOPS: 70.46862775244892, Tokens per sec: 174629.02233074856, Loss: 2.5085949897766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19552 , TFLOPS: 71.55790075190284, Tokens per sec: 177328.36081672195, Loss: 2.5136189460754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19553 , TFLOPS: 69.69329173161069, Tokens per sec: 172707.65426079792, Loss: 2.4784677028656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19554 , TFLOPS: 69.07091020673252, Tokens per sec: 171165.32428116497, Loss: 2.493309259414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19555 , TFLOPS: 70.53254698768094, Tokens per sec: 174787.42123693856, Loss: 2.5137882232666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19556 , TFLOPS: 69.81068335013293, Tokens per sec: 172998.56362325037, Loss: 2.502969980239868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19557 , TFLOPS: 68.99942309296068, Tokens per sec: 170988.1713382827, Loss: 2.4731943607330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19558 , TFLOPS: 70.29343289787234, Tokens per sec: 174194.87018180135, Loss: 2.479884624481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19559 , TFLOPS: 70.59319409156714, Tokens per sec: 174937.71144119854, Loss: 2.474630355834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19560 , TFLOPS: 68.73151126220151, Tokens per sec: 170324.25630873692, Loss: 2.4712860584259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19561 , TFLOPS: 70.87934147555517, Tokens per sec: 175646.8161804574, Loss: 2.4849865436553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19562 , TFLOPS: 70.49206396514548, Tokens per sec: 174687.09984752256, Loss: 2.4701693058013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19563 , TFLOPS: 69.80958251506394, Tokens per sec: 172995.8356326824, Loss: 2.489926338195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19564 , TFLOPS: 69.2024575822896, Tokens per sec: 171491.3131689351, Loss: 2.47751522064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19565 , TFLOPS: 71.5112882258078, Tokens per sec: 177212.84984226484, Loss: 2.470505475997925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19566 , TFLOPS: 69.53939185342902, Tokens per sec: 172326.2734091926, Loss: 2.4884588718414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19567 , TFLOPS: 67.34837865007724, Tokens per sec: 166896.7013312577, Loss: 2.512830972671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19568 , TFLOPS: 71.51286860276133, Tokens per sec: 177216.76619044854, Loss: 2.490182399749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19569 , TFLOPS: 70.5575253912787, Tokens per sec: 174849.3204726521, Loss: 2.4979910850524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19570 , TFLOPS: 69.13397633229204, Tokens per sec: 171321.6090876087, Loss: 2.494837760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19571 , TFLOPS: 70.9643079348348, Tokens per sec: 175857.37242638052, Loss: 2.4813241958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19572 , TFLOPS: 71.49964063944779, Tokens per sec: 177183.98584017716, Loss: 2.4927139282226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19573 , TFLOPS: 68.73931907033727, Tokens per sec: 170343.6049173984, Loss: 2.495776891708374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19574 , TFLOPS: 68.98431817897529, Tokens per sec: 170950.73969748325, Loss: 2.4706578254699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19575 , TFLOPS: 71.48435623916987, Tokens per sec: 177146.10941257278, Loss: 2.4826982021331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19576 , TFLOPS: 69.33639891687784, Tokens per sec: 171823.23455090082, Loss: 2.5020804405212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19577 , TFLOPS: 68.5974312300088, Tokens per sec: 169991.99121882854, Loss: 2.500609874725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19578 , TFLOPS: 71.55573622777955, Tokens per sec: 177322.9968875024, Loss: 2.500499963760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19579 , TFLOPS: 70.69851119965438, Tokens per sec: 175198.6988366759, Loss: 2.475843906402588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19580 , TFLOPS: 68.84388336643664, Tokens per sec: 170602.72676184078, Loss: 2.4976890087127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19581 , TFLOPS: 70.08479429719961, Tokens per sec: 173677.84074589514, Loss: 2.473759651184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19582 , TFLOPS: 71.52170530817016, Tokens per sec: 177238.66451989507, Loss: 2.4798033237457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19583 , TFLOPS: 69.74677966766814, Tokens per sec: 172840.20325853082, Loss: 2.490478277206421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19584 , TFLOPS: 67.75730480941439, Tokens per sec: 167910.06540103036, Loss: 2.478485107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19585 , TFLOPS: 71.51623597071466, Tokens per sec: 177225.1108991831, Loss: 2.489964008331299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19586 , TFLOPS: 70.718379023822, Tokens per sec: 175247.93349358477, Loss: 2.5060036182403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19587 , TFLOPS: 68.00977891022339, Tokens per sec: 168535.72403515325, Loss: 2.4909474849700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19588 , TFLOPS: 70.26863989322793, Tokens per sec: 174133.43038511756, Loss: 2.496164083480835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19589 , TFLOPS: 71.52377824155259, Tokens per sec: 177243.80147716924, Loss: 2.4788451194763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19590 , TFLOPS: 68.82573644997711, Tokens per sec: 170557.7567038062, Loss: 2.4783403873443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19591 , TFLOPS: 70.55420102591174, Tokens per sec: 174841.08232906368, Loss: 2.4927077293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19592 , TFLOPS: 70.48092180770524, Tokens per sec: 174659.4883539757, Loss: 2.496962308883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19593 , TFLOPS: 69.38742788781855, Tokens per sec: 171949.6899621971, Loss: 2.4762356281280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19594 , TFLOPS: 68.92734418409455, Tokens per sec: 170809.55186196513, Loss: 2.4743869304656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19595 , TFLOPS: 69.86544352925789, Tokens per sec: 173134.26537945372, Loss: 2.4915528297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19596 , TFLOPS: 70.5868430742371, Tokens per sec: 174921.9729206324, Loss: 2.488420009613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19597 , TFLOPS: 68.45379894083689, Tokens per sec: 169636.05458385774, Loss: 2.474064826965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19598 , TFLOPS: 71.49836956384343, Tokens per sec: 177180.835974809, Loss: 2.4769909381866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19599 , TFLOPS: 69.7289814139092, Tokens per sec: 172796.09722507666, Loss: 2.4870760440826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19600 , TFLOPS: 70.56582141368877, Tokens per sec: 174869.87893006654, Loss: 2.494969367980957 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/optimizer.pt +[rank0]:[2024-08-27 19:25:17,841] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006379778002155945, 'preprocessing_with_comm': 0.001261313009308651, 'state_converting': 1.003008422005223, : 1.0120324459858239}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600, took 4.45s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017600 + eval ppl=9.895880699157715, eval loss=2.292118549346924 +------------------------------------------------------------------ +iteration: 19601 , TFLOPS: 64.23200032006989, Tokens per sec: 159173.97253208648, Loss: 2.4908077716827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19602 , TFLOPS: 70.0801011801073, Tokens per sec: 173666.21068475084, Loss: 2.493403673171997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19603 , TFLOPS: 70.58670732134846, Tokens per sec: 174921.63651001992, Loss: 2.4916694164276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19604 , TFLOPS: 68.41641865594158, Tokens per sec: 169543.42212010885, Loss: 2.500472068786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19605 , TFLOPS: 70.88686007906662, Tokens per sec: 175665.44810820132, Loss: 2.4836857318878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19606 , TFLOPS: 70.28391382428924, Tokens per sec: 174171.28086884992, Loss: 2.511744499206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19607 , TFLOPS: 68.8026736097745, Tokens per sec: 170500.60444520254, Loss: 2.4683239459991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19608 , TFLOPS: 70.67819321079526, Tokens per sec: 175148.34862207176, Loss: 2.4753470420837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19609 , TFLOPS: 71.52947357441259, Tokens per sec: 177257.91513938905, Loss: 2.483045816421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19610 , TFLOPS: 67.2751673775202, Tokens per sec: 166715.27573297394, Loss: 2.4783873558044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19611 , TFLOPS: 69.1889485812626, Tokens per sec: 171457.8363762509, Loss: 2.5005180835723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19612 , TFLOPS: 71.48077168808047, Tokens per sec: 177137.22650010232, Loss: 2.4847323894500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19613 , TFLOPS: 70.64020912472931, Tokens per sec: 175054.21987250782, Loss: 2.4738705158233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19614 , TFLOPS: 67.60515487361037, Tokens per sec: 167533.02109940894, Loss: 2.4972615242004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19615 , TFLOPS: 71.46034138704232, Tokens per sec: 177086.59796354634, Loss: 2.4997782707214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19616 , TFLOPS: 70.61270978644362, Tokens per sec: 174986.07348293372, Loss: 2.472954511642456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19617 , TFLOPS: 66.32937612274495, Tokens per sec: 164371.50081613322, Loss: 2.4984140396118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19618 , TFLOPS: 70.6805285070443, Tokens per sec: 175154.13574343102, Loss: 2.5132768154144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19619 , TFLOPS: 71.5154011655278, Tokens per sec: 177223.04215996873, Loss: 2.4909908771514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19620 , TFLOPS: 68.88447407907532, Tokens per sec: 170703.31501919552, Loss: 2.5142765045166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19621 , TFLOPS: 68.00276478128544, Tokens per sec: 168518.34225097488, Loss: 2.4839868545532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19622 , TFLOPS: 71.55905277408121, Tokens per sec: 177331.21565458569, Loss: 2.496898889541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19623 , TFLOPS: 69.92802506304093, Tokens per sec: 173289.34931409315, Loss: 2.500476837158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19624 , TFLOPS: 69.69750931646286, Tokens per sec: 172718.1059006669, Loss: 2.4997527599334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19625 , TFLOPS: 69.82911287887673, Tokens per sec: 173044.23402565767, Loss: 2.498959541320801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19626 , TFLOPS: 70.52232698000358, Tokens per sec: 174762.09493208723, Loss: 2.467109441757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19627 , TFLOPS: 68.7530152491114, Tokens per sec: 170377.54555715935, Loss: 2.482149124145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19628 , TFLOPS: 70.57292349755961, Tokens per sec: 174887.47867625763, Loss: 2.4824252128601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19629 , TFLOPS: 69.6164562791261, Tokens per sec: 172517.24754541728, Loss: 2.491696834564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19630 , TFLOPS: 69.80752420494179, Tokens per sec: 172990.73491345812, Loss: 2.5031826496124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19631 , TFLOPS: 69.14295569371855, Tokens per sec: 171343.86093438193, Loss: 2.4918980598449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19632 , TFLOPS: 70.45394936655744, Tokens per sec: 174592.64767355178, Loss: 2.498664140701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19633 , TFLOPS: 70.60740651316716, Tokens per sec: 174972.93138755087, Loss: 2.4911022186279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19634 , TFLOPS: 68.07009666752018, Tokens per sec: 168685.19808228416, Loss: 2.498527765274048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19635 , TFLOPS: 71.52603253535744, Tokens per sec: 177249.3878655486, Loss: 2.471177816390991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19636 , TFLOPS: 69.58293592964468, Tokens per sec: 172434.18042683098, Loss: 2.485109806060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19637 , TFLOPS: 69.1245618629726, Tokens per sec: 171298.2789955457, Loss: 2.5020010471343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19638 , TFLOPS: 69.98301630110404, Tokens per sec: 173425.62365979847, Loss: 2.489284038543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19639 , TFLOPS: 70.45208318160837, Tokens per sec: 174588.02306167074, Loss: 2.492008924484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19640 , TFLOPS: 68.82649617625097, Tokens per sec: 170559.63938920354, Loss: 2.462739944458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19641 , TFLOPS: 68.86318339894535, Tokens per sec: 170650.55436847146, Loss: 2.464872360229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19642 , TFLOPS: 71.51130139133294, Tokens per sec: 177212.8824678864, Loss: 2.485153913497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19643 , TFLOPS: 69.50913720252302, Tokens per sec: 172251.2990514226, Loss: 2.500459909439087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19644 , TFLOPS: 69.20786583166942, Tokens per sec: 171504.71540666613, Loss: 2.49894642829895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19645 , TFLOPS: 70.63138180953673, Tokens per sec: 175032.3448130516, Loss: 2.5032522678375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19646 , TFLOPS: 70.61259817528136, Tokens per sec: 174985.79689817867, Loss: 2.485999584197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19647 , TFLOPS: 67.71302899981265, Tokens per sec: 167800.34506745418, Loss: 2.4802489280700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19648 , TFLOPS: 70.56173185993143, Tokens per sec: 174859.74456535434, Loss: 2.4931819438934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19649 , TFLOPS: 71.51778962235659, Tokens per sec: 177228.96101910068, Loss: 2.4980549812316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19650 , TFLOPS: 70.14728522238894, Tokens per sec: 173832.70014245796, Loss: 2.483264207839966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19651 , TFLOPS: 67.43854040538326, Tokens per sec: 167120.13209304336, Loss: 2.515205144882202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19652 , TFLOPS: 71.51525450460807, Tokens per sec: 177222.67871805356, Loss: 2.5069968700408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19653 , TFLOPS: 70.57906121262658, Tokens per sec: 174902.6886103187, Loss: 2.497345447540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19654 , TFLOPS: 67.91741399792875, Tokens per sec: 168306.83360765205, Loss: 2.4978878498077393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19655 , TFLOPS: 70.85262316565701, Tokens per sec: 175580.60526526498, Loss: 2.508298873901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19656 , TFLOPS: 70.75715456007576, Tokens per sec: 175344.02354389953, Loss: 2.5166358947753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19657 , TFLOPS: 69.14780134812403, Tokens per sec: 171355.8689997916, Loss: 2.4666666984558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19658 , TFLOPS: 68.7227802148168, Tokens per sec: 170302.61981151332, Loss: 2.4814531803131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19659 , TFLOPS: 71.5026097703306, Tokens per sec: 177191.3436735815, Loss: 2.5012547969818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19660 , TFLOPS: 69.67172670746041, Tokens per sec: 172654.21375537, Loss: 2.4955577850341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19661 , TFLOPS: 68.24996989256752, Tokens per sec: 169130.94374862296, Loss: 2.508589744567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19662 , TFLOPS: 70.53865166729737, Tokens per sec: 174802.54930551403, Loss: 2.483571767807007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19663 , TFLOPS: 70.66871603919537, Tokens per sec: 175124.86314685064, Loss: 2.4808058738708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19664 , TFLOPS: 68.9810041408239, Tokens per sec: 170942.52714587265, Loss: 2.4820642471313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19665 , TFLOPS: 70.6439289110305, Tokens per sec: 175063.43791273047, Loss: 2.5128090381622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19666 , TFLOPS: 70.36583833874937, Tokens per sec: 174374.29884041238, Loss: 2.4846954345703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19667 , TFLOPS: 69.18558234194862, Tokens per sec: 171449.49446440898, Loss: 2.488828182220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19668 , TFLOPS: 69.974224832595, Tokens per sec: 173403.83743237288, Loss: 2.5172371864318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19669 , TFLOPS: 70.50106982397989, Tokens per sec: 174709.4173577915, Loss: 2.4947609901428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19670 , TFLOPS: 69.81640744829775, Tokens per sec: 173012.74856906422, Loss: 2.4838902950286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19671 , TFLOPS: 68.92083550838043, Tokens per sec: 170793.42264655573, Loss: 2.487571954727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19672 , TFLOPS: 70.77827036764923, Tokens per sec: 175396.3508411648, Loss: 2.47334361076355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19673 , TFLOPS: 69.59829015154246, Tokens per sec: 172472.22988010084, Loss: 2.4965474605560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19674 , TFLOPS: 68.79344275745477, Tokens per sec: 170477.7293762919, Loss: 2.482929229736328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19675 , TFLOPS: 70.64452439719372, Tokens per sec: 175064.91359304116, Loss: 2.487327814102173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19676 , TFLOPS: 70.46915227512041, Tokens per sec: 174630.32215570944, Loss: 2.4975368976593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19677 , TFLOPS: 69.00408984783667, Tokens per sec: 170999.73607095086, Loss: 2.4829840660095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19678 , TFLOPS: 69.60029807814323, Tokens per sec: 172477.2057434089, Loss: 2.4770021438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19679 , TFLOPS: 71.47561497816685, Tokens per sec: 177124.4476048779, Loss: 2.492583990097046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19680 , TFLOPS: 69.5300812360328, Tokens per sec: 172303.20067363555, Loss: 2.488914728164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19681 , TFLOPS: 67.69713563436555, Tokens per sec: 167760.95955707753, Loss: 2.4757893085479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19682 , TFLOPS: 71.52320190002546, Tokens per sec: 177242.3732393757, Loss: 2.4870169162750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19683 , TFLOPS: 70.71275632672804, Tokens per sec: 175233.99983079475, Loss: 2.4908437728881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19684 , TFLOPS: 68.04325624253869, Tokens per sec: 168618.68455246315, Loss: 2.487208843231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19685 , TFLOPS: 70.57002829752199, Tokens per sec: 174880.30405163174, Loss: 2.490464210510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19686 , TFLOPS: 71.50135583591131, Tokens per sec: 177188.23628595824, Loss: 2.4800221920013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19687 , TFLOPS: 69.02347135573993, Tokens per sec: 171047.7655825845, Loss: 2.476046085357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19688 , TFLOPS: 68.59726302538202, Tokens per sec: 169991.57438923413, Loss: 2.495590925216675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19689 , TFLOPS: 71.50025292746768, Tokens per sec: 177185.50315733955, Loss: 2.483358383178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19690 , TFLOPS: 70.72967455437407, Tokens per sec: 175275.92506259822, Loss: 2.4959216117858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19691 , TFLOPS: 67.35802643878093, Tokens per sec: 166920.60961445712, Loss: 2.4782896041870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19692 , TFLOPS: 70.56439467692917, Tokens per sec: 174866.34331920824, Loss: 2.4969441890716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19693 , TFLOPS: 70.53119017877208, Tokens per sec: 174784.05891499756, Loss: 2.4642674922943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19694 , TFLOPS: 68.97839709505581, Tokens per sec: 170936.06659926954, Loss: 2.4926340579986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19695 , TFLOPS: 69.55579827255696, Tokens per sec: 172366.9303231079, Loss: 2.485508918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19696 , TFLOPS: 71.48368019428173, Tokens per sec: 177144.43409886747, Loss: 2.4867172241210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19697 , TFLOPS: 69.13455007394128, Tokens per sec: 171323.0308826184, Loss: 2.4653525352478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19698 , TFLOPS: 69.05431504475827, Tokens per sec: 171124.19964168477, Loss: 2.4948806762695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19699 , TFLOPS: 70.55323348966937, Tokens per sec: 174838.68466767218, Loss: 2.48073148727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19700 , TFLOPS: 70.62462586868682, Tokens per sec: 175015.60284739718, Loss: 2.4908628463745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19701 , TFLOPS: 67.39109577436737, Tokens per sec: 167002.5590709284, Loss: 2.4892044067382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19702 , TFLOPS: 71.55896782523806, Tokens per sec: 177331.00514199448, Loss: 2.4947400093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19703 , TFLOPS: 69.6088737997666, Tokens per sec: 172498.45732628537, Loss: 2.4766042232513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19704 , TFLOPS: 69.68052283563402, Tokens per sec: 172676.01152995773, Loss: 2.48063325881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19705 , TFLOPS: 70.12265716365708, Tokens per sec: 173771.6690999158, Loss: 2.5176002979278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19706 , TFLOPS: 70.48882394110186, Tokens per sec: 174679.07071102437, Loss: 2.4884700775146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19707 , TFLOPS: 69.04334407278645, Tokens per sec: 171097.01236458626, Loss: 2.489936113357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19708 , TFLOPS: 69.55655321162989, Tokens per sec: 172368.801145294, Loss: 2.5009870529174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19709 , TFLOPS: 71.51265945564177, Tokens per sec: 177216.24790084682, Loss: 2.496471405029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19710 , TFLOPS: 69.66449312746423, Tokens per sec: 172636.28814729484, Loss: 2.5009913444519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19711 , TFLOPS: 67.98574950529722, Tokens per sec: 168476.1765226887, Loss: 2.494880199432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19712 , TFLOPS: 70.62104801250987, Tokens per sec: 175006.73652565732, Loss: 2.505582332611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19713 , TFLOPS: 70.49820454847038, Tokens per sec: 174702.3168894428, Loss: 2.4740147590637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19714 , TFLOPS: 69.01928311452191, Tokens per sec: 171037.3866594734, Loss: 2.502254009246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19715 , TFLOPS: 70.70370390896593, Tokens per sec: 175211.5669423741, Loss: 2.48911452293396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19716 , TFLOPS: 71.56733091613658, Tokens per sec: 177351.72980251152, Loss: 2.4853835105895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19717 , TFLOPS: 68.32375864123343, Tokens per sec: 169313.80039631878, Loss: 2.481989860534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19718 , TFLOPS: 68.23196568740163, Tokens per sec: 169086.32734489493, Loss: 2.505856513977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19719 , TFLOPS: 71.54662771776795, Tokens per sec: 177300.4249963085, Loss: 2.4997551441192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19720 , TFLOPS: 70.64989610284884, Tokens per sec: 175078.2252713968, Loss: 2.4934208393096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19721 , TFLOPS: 67.83498678058747, Tokens per sec: 168102.5698829728, Loss: 2.5068001747131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19722 , TFLOPS: 70.95203277994355, Tokens per sec: 175826.95324034005, Loss: 2.500060558319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19723 , TFLOPS: 70.62461455429994, Tokens per sec: 175015.5748091001, Loss: 2.4869837760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19724 , TFLOPS: 69.6104720501057, Tokens per sec: 172502.4179666875, Loss: 2.4801785945892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19725 , TFLOPS: 68.84567909316641, Tokens per sec: 170607.17677049278, Loss: 2.5013365745544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19726 , TFLOPS: 71.51991508028505, Tokens per sec: 177234.2281379859, Loss: 2.4818074703216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19727 , TFLOPS: 69.9677547494632, Tokens per sec: 173387.80385363376, Loss: 2.49100923538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19728 , TFLOPS: 68.2428124805183, Tokens per sec: 169113.20689311097, Loss: 2.4744374752044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19729 , TFLOPS: 71.52011779666805, Tokens per sec: 177234.73049151403, Loss: 2.4707579612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19730 , TFLOPS: 70.53835691535679, Tokens per sec: 174801.81887773564, Loss: 2.501016139984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19731 , TFLOPS: 67.84780678763056, Tokens per sec: 168134.33927266594, Loss: 2.497767448425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19732 , TFLOPS: 69.47901520816559, Tokens per sec: 172176.6534311933, Loss: 2.47546648979187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19733 , TFLOPS: 71.49821741122997, Tokens per sec: 177180.45892387186, Loss: 2.483417272567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19734 , TFLOPS: 68.69798149055684, Tokens per sec: 170241.16584099218, Loss: 2.506481170654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19735 , TFLOPS: 69.84120685220269, Tokens per sec: 173074.20422382053, Loss: 2.500826835632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19736 , TFLOPS: 70.39659781145421, Tokens per sec: 174450.52420221065, Loss: 2.4976389408111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19737 , TFLOPS: 69.66349096326705, Tokens per sec: 172633.80467401672, Loss: 2.5025253295898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19738 , TFLOPS: 68.87175505604158, Tokens per sec: 170671.79587900187, Loss: 2.48335862159729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19739 , TFLOPS: 70.85614901300087, Tokens per sec: 175589.34270338723, Loss: 2.4976565837860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19740 , TFLOPS: 69.68406054209274, Tokens per sec: 172684.77835627296, Loss: 2.4694535732269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19741 , TFLOPS: 68.2210076667327, Tokens per sec: 169059.17216255175, Loss: 2.4810736179351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19742 , TFLOPS: 70.60536794783248, Tokens per sec: 174967.879598085, Loss: 2.4757089614868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19743 , TFLOPS: 69.29162674604589, Tokens per sec: 171712.2841795175, Loss: 2.490544319152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19744 , TFLOPS: 69.7613371355825, Tokens per sec: 172876.2782676601, Loss: 2.4966282844543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19745 , TFLOPS: 69.81515402209315, Tokens per sec: 173009.64244085294, Loss: 2.501068592071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19746 , TFLOPS: 71.51534654681114, Tokens per sec: 177222.90680877207, Loss: 2.468127965927124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19747 , TFLOPS: 69.74122684924163, Tokens per sec: 172826.4427627765, Loss: 2.4945526123046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19748 , TFLOPS: 68.16805271433215, Tokens per sec: 168927.94395703357, Loss: 2.500105857849121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19749 , TFLOPS: 71.4653812136797, Tokens per sec: 177099.08720913652, Loss: 2.513108253479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19750 , TFLOPS: 69.57373005481053, Tokens per sec: 172411.36725488113, Loss: 2.488370895385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19751 , TFLOPS: 67.49959520015015, Tokens per sec: 167271.43260021653, Loss: 2.4987833499908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19752 , TFLOPS: 70.74366158903688, Tokens per sec: 175310.58647529173, Loss: 2.5147242546081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19753 , TFLOPS: 71.52584792286008, Tokens per sec: 177248.93037544342, Loss: 2.5109708309173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19754 , TFLOPS: 69.62644197201995, Tokens per sec: 172541.9931924231, Loss: 2.4829888343811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19755 , TFLOPS: 69.0335452014979, Tokens per sec: 171072.72968209843, Loss: 2.4793243408203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19756 , TFLOPS: 71.4960289405755, Tokens per sec: 177175.03565251015, Loss: 2.5104563236236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19757 , TFLOPS: 69.66102949568128, Tokens per sec: 172627.70488618614, Loss: 2.5151262283325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19758 , TFLOPS: 68.67368799658092, Tokens per sec: 170180.96388677057, Loss: 2.4891388416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19759 , TFLOPS: 70.64382727784962, Tokens per sec: 175063.18605451245, Loss: 2.4755544662475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19760 , TFLOPS: 70.71516447228633, Tokens per sec: 175239.9674807673, Loss: 2.481910467147827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19761 , TFLOPS: 68.35243900666553, Tokens per sec: 169384.8735012626, Loss: 2.4979395866394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19762 , TFLOPS: 69.05932237182282, Tokens per sec: 171136.60834975334, Loss: 2.5064260959625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19763 , TFLOPS: 70.59037914955944, Tokens per sec: 174930.73570481202, Loss: 2.5178639888763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19764 , TFLOPS: 70.65400467975032, Tokens per sec: 175088.40677755573, Loss: 2.4940438270568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19765 , TFLOPS: 68.22158164232776, Tokens per sec: 169060.59453730492, Loss: 2.471133232116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19766 , TFLOPS: 71.54964138382023, Tokens per sec: 177307.89319276877, Loss: 2.5017173290252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19767 , TFLOPS: 70.70789514264177, Tokens per sec: 175221.95328112534, Loss: 2.486133098602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19768 , TFLOPS: 68.92008152120809, Tokens per sec: 170791.55418328408, Loss: 2.4775524139404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19769 , TFLOPS: 69.82488367114017, Tokens per sec: 173033.75358300077, Loss: 2.4866714477539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19770 , TFLOPS: 70.58735725571505, Tokens per sec: 174923.2471189765, Loss: 2.4776840209960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19771 , TFLOPS: 68.92244154650993, Tokens per sec: 170797.40258596034, Loss: 2.4909508228302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19772 , TFLOPS: 70.04016735180537, Tokens per sec: 173567.25025914432, Loss: 2.4827427864074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19773 , TFLOPS: 70.50263707233773, Tokens per sec: 174713.30117186886, Loss: 2.489518165588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19774 , TFLOPS: 69.91698391435304, Tokens per sec: 173261.9881313616, Loss: 2.5057005882263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19775 , TFLOPS: 68.86990771897898, Tokens per sec: 170667.21797425978, Loss: 2.494905948638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19776 , TFLOPS: 71.55135506395627, Tokens per sec: 177312.1398809233, Loss: 2.5155391693115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19777 , TFLOPS: 69.49528474820065, Tokens per sec: 172216.97114363793, Loss: 2.4893648624420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19778 , TFLOPS: 68.50131060399589, Tokens per sec: 169753.7937190075, Loss: 2.4772627353668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19779 , TFLOPS: 70.66204586879465, Tokens per sec: 175108.3337298177, Loss: 2.485261917114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19780 , TFLOPS: 69.64560050997179, Tokens per sec: 172589.47015995494, Loss: 2.477590560913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19781 , TFLOPS: 68.09405178206634, Tokens per sec: 168744.5614950033, Loss: 2.4956114292144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19782 , TFLOPS: 70.6886382250944, Tokens per sec: 175174.23251811834, Loss: 2.5055651664733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19783 , TFLOPS: 71.52977356577792, Tokens per sec: 177258.65855103906, Loss: 2.4871463775634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19784 , TFLOPS: 69.11308464840846, Tokens per sec: 171269.83719353622, Loss: 2.494046449661255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19785 , TFLOPS: 68.65765622313174, Tokens per sec: 170141.23538611786, Loss: 2.4878385066986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19786 , TFLOPS: 71.53749310827382, Tokens per sec: 177277.78842772174, Loss: 2.5047640800476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19787 , TFLOPS: 69.55265128941649, Tokens per sec: 172359.131752217, Loss: 2.5030105113983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19788 , TFLOPS: 68.73117478726496, Tokens per sec: 170323.4224867784, Loss: 2.485131025314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19789 , TFLOPS: 70.62184266260718, Tokens per sec: 175008.7057561364, Loss: 2.4847524166107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19790 , TFLOPS: 70.66202595067273, Tokens per sec: 175108.28437051742, Loss: 2.4865450859069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19791 , TFLOPS: 67.94648895310488, Tokens per sec: 168378.88454944923, Loss: 2.5050578117370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19792 , TFLOPS: 69.60670140233398, Tokens per sec: 172493.073886138, Loss: 2.510336399078369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19793 , TFLOPS: 71.50999136283247, Tokens per sec: 177209.6360729507, Loss: 2.4961464405059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19794 , TFLOPS: 70.6843485996942, Tokens per sec: 175163.60235383542, Loss: 2.487938404083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19795 , TFLOPS: 67.93910065168383, Tokens per sec: 168360.57552463928, Loss: 2.5038001537323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19796 , TFLOPS: 71.49558060547118, Tokens per sec: 177173.92462873345, Loss: 2.472032070159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19797 , TFLOPS: 70.61910430539287, Tokens per sec: 175001.91980530435, Loss: 2.4915122985839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19798 , TFLOPS: 68.14704927453873, Tokens per sec: 168875.89511950559, Loss: 2.4975600242614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19799 , TFLOPS: 69.02251814449787, Tokens per sec: 171045.4034201218, Loss: 2.4663538932800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19800 , TFLOPS: 71.51270140714581, Tokens per sec: 177216.35186129517, Loss: 2.5085175037384033 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/optimizer.pt +[rank0]:[2024-08-27 19:50:42,996] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006472877023043111, 'preprocessing_with_comm': 0.0013471419806592166, 'state_converting': 0.9815850659797434, : 0.9907839859952219}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0017800 + eval ppl=9.892834663391113, eval loss=2.2918107509613037 +------------------------------------------------------------------ +iteration: 19801 , TFLOPS: 62.19728016003311, Tokens per sec: 154131.7118325862, Loss: 2.474379539489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19802 , TFLOPS: 68.38287858781378, Tokens per sec: 169460.30613654465, Loss: 2.5041890144348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19803 , TFLOPS: 71.43490462599517, Tokens per sec: 177023.56286758097, Loss: 2.4687671661376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19804 , TFLOPS: 69.78733302895583, Tokens per sec: 172940.69895512325, Loss: 2.492112636566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19805 , TFLOPS: 68.8258022209997, Tokens per sec: 170557.91969164542, Loss: 2.489837408065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19806 , TFLOPS: 70.36408208068892, Tokens per sec: 174369.94663947079, Loss: 2.4716649055480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19807 , TFLOPS: 70.72650609425392, Tokens per sec: 175268.0732693858, Loss: 2.49261474609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19808 , TFLOPS: 68.76519645377992, Tokens per sec: 170407.73192419842, Loss: 2.487941026687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19809 , TFLOPS: 70.97565113297708, Tokens per sec: 175885.4821209335, Loss: 2.4878928661346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19810 , TFLOPS: 69.61440807859222, Tokens per sec: 172512.17187886641, Loss: 2.490797996520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19811 , TFLOPS: 69.00265038852653, Tokens per sec: 170996.1689322108, Loss: 2.4913904666900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19812 , TFLOPS: 69.91663226995067, Tokens per sec: 173261.1167177957, Loss: 2.47739315032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19813 , TFLOPS: 70.44891787877219, Tokens per sec: 174580.17909255577, Loss: 2.4774861335754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19814 , TFLOPS: 69.68638854609387, Tokens per sec: 172690.54740663854, Loss: 2.4882705211639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19815 , TFLOPS: 69.06323435369441, Tokens per sec: 171146.30267177153, Loss: 2.490971565246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19816 , TFLOPS: 71.4904942948127, Tokens per sec: 177161.3201906184, Loss: 2.495107650756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19817 , TFLOPS: 69.6569205537163, Tokens per sec: 172617.5224753596, Loss: 2.481067180633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19818 , TFLOPS: 69.02780286987173, Tokens per sec: 171058.49955175878, Loss: 2.5129599571228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19819 , TFLOPS: 70.58028442742963, Tokens per sec: 174905.71987134888, Loss: 2.493732213973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19820 , TFLOPS: 70.39482186904851, Tokens per sec: 174446.12322129312, Loss: 2.489823341369629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19821 , TFLOPS: 68.89346345376025, Tokens per sec: 170725.59167992618, Loss: 2.481518030166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19822 , TFLOPS: 69.48999377232909, Tokens per sec: 172203.8595225791, Loss: 2.4918203353881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19823 , TFLOPS: 71.52570429236789, Tokens per sec: 177248.57444326158, Loss: 2.4968466758728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19824 , TFLOPS: 68.89240424695822, Tokens per sec: 170722.96684878948, Loss: 2.4971256256103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19825 , TFLOPS: 69.83740831097168, Tokens per sec: 173064.79102020623, Loss: 2.4801111221313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19826 , TFLOPS: 70.96497210868748, Tokens per sec: 175859.0183223522, Loss: 2.4801180362701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19827 , TFLOPS: 70.56976035513503, Tokens per sec: 174879.6400608806, Loss: 2.4824318885803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19828 , TFLOPS: 67.64929619486381, Tokens per sec: 167642.40815604292, Loss: 2.463402271270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19829 , TFLOPS: 70.55001348675134, Tokens per sec: 174830.70514572866, Loss: 2.478896141052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19830 , TFLOPS: 70.67928433083387, Tokens per sec: 175151.05253772426, Loss: 2.5035712718963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19831 , TFLOPS: 69.79432161017054, Tokens per sec: 172958.0174292291, Loss: 2.5091328620910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19832 , TFLOPS: 68.16385782353336, Tokens per sec: 168917.54855552883, Loss: 2.4788401126861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19833 , TFLOPS: 71.5020725025125, Tokens per sec: 177190.01226474333, Loss: 2.504014730453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19834 , TFLOPS: 70.58276799332278, Tokens per sec: 174911.8744212195, Loss: 2.4749491214752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19835 , TFLOPS: 67.15318767150877, Tokens per sec: 166412.99658430374, Loss: 2.5077950954437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19836 , TFLOPS: 71.50892679307638, Tokens per sec: 177206.9979518231, Loss: 2.504330635070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19837 , TFLOPS: 70.73285474209423, Tokens per sec: 175283.8059180954, Loss: 2.4922568798065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19838 , TFLOPS: 68.95797254158421, Tokens per sec: 170885.4523058166, Loss: 2.4745142459869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19839 , TFLOPS: 69.69758413203272, Tokens per sec: 172718.2913018906, Loss: 2.4783458709716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19840 , TFLOPS: 71.53675860320774, Tokens per sec: 177275.96824325557, Loss: 2.470398187637329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19841 , TFLOPS: 68.74369474636464, Tokens per sec: 170354.44832461476, Loss: 2.4938817024230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19842 , TFLOPS: 69.81879441007551, Tokens per sec: 173018.6637232951, Loss: 2.5139379501342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19843 , TFLOPS: 70.44872118004068, Tokens per sec: 174579.6916514309, Loss: 2.487516164779663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19844 , TFLOPS: 70.63298395639151, Tokens per sec: 175036.31510944877, Loss: 2.5014724731445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19845 , TFLOPS: 69.08777193731983, Tokens per sec: 171207.10950703328, Loss: 2.5073904991149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19846 , TFLOPS: 70.50433124480163, Tokens per sec: 174717.49951786295, Loss: 2.496206521987915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19847 , TFLOPS: 69.5199701108242, Tokens per sec: 172278.1441915363, Loss: 2.487839460372925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19848 , TFLOPS: 69.68660423553408, Tokens per sec: 172691.08190883155, Loss: 2.487821102142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19849 , TFLOPS: 70.11208726707346, Tokens per sec: 173745.47573181137, Loss: 2.4965906143188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19850 , TFLOPS: 70.54985576382973, Tokens per sec: 174830.3142909543, Loss: 2.479491949081421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19851 , TFLOPS: 70.14226624297882, Tokens per sec: 173820.26255859347, Loss: 2.484315872192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19852 , TFLOPS: 68.24062951676697, Tokens per sec: 169107.79726846184, Loss: 2.489320993423462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19853 , TFLOPS: 71.52010626074174, Tokens per sec: 177234.7019042179, Loss: 2.468738079071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19854 , TFLOPS: 69.51883667209727, Tokens per sec: 172275.33540551236, Loss: 2.5020477771759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19855 , TFLOPS: 67.99048089545033, Tokens per sec: 168487.90142869376, Loss: 2.5002336502075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19856 , TFLOPS: 71.49940471653682, Tokens per sec: 177183.40119721473, Loss: 2.4865872859954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19857 , TFLOPS: 69.63791098883563, Tokens per sec: 172570.4146795089, Loss: 2.5122063159942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19858 , TFLOPS: 69.00236759585694, Tokens per sec: 170995.4681408234, Loss: 2.4757914543151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19859 , TFLOPS: 70.5520172293224, Tokens per sec: 174835.67064055023, Loss: 2.5085182189941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19860 , TFLOPS: 71.5170275639744, Tokens per sec: 177227.0725544819, Loss: 2.4743165969848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19861 , TFLOPS: 68.68214504819684, Tokens per sec: 170201.92139229595, Loss: 2.4591445922851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19862 , TFLOPS: 69.66786210057626, Tokens per sec: 172644.63683378947, Loss: 2.4953689575195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19863 , TFLOPS: 71.50813290637136, Tokens per sec: 177205.03061311418, Loss: 2.48703932762146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19864 , TFLOPS: 70.65269153394792, Tokens per sec: 175085.15265760245, Loss: 2.500842809677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19865 , TFLOPS: 67.82657743301803, Tokens per sec: 168081.73059332962, Loss: 2.511174201965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19866 , TFLOPS: 70.51115064104854, Tokens per sec: 174734.39873298127, Loss: 2.488257884979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19867 , TFLOPS: 70.6140893892193, Tokens per sec: 174989.49229058766, Loss: 2.4881181716918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19868 , TFLOPS: 69.67541555099783, Tokens per sec: 172663.3551160154, Loss: 2.5007100105285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19869 , TFLOPS: 68.90086258569869, Tokens per sec: 170743.9275439513, Loss: 2.4675405025482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19870 , TFLOPS: 71.5271973020758, Tokens per sec: 177252.27428578638, Loss: 2.4903550148010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19871 , TFLOPS: 69.8413995464503, Tokens per sec: 173074.68174139314, Loss: 2.4953255653381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19872 , TFLOPS: 67.113236923099, Tokens per sec: 166313.99422880608, Loss: 2.4938645362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19873 , TFLOPS: 71.5476913319129, Tokens per sec: 177303.06074932645, Loss: 2.48374342918396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19874 , TFLOPS: 70.51075484938896, Tokens per sec: 174733.41791764877, Loss: 2.4965896606445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19875 , TFLOPS: 68.90181921905362, Tokens per sec: 170746.29818678665, Loss: 2.4848194122314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19876 , TFLOPS: 69.94778956883829, Tokens per sec: 173338.32793669903, Loss: 2.4571902751922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19877 , TFLOPS: 70.65669253275038, Tokens per sec: 175095.06757338138, Loss: 2.4772791862487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19878 , TFLOPS: 68.76186279183268, Tokens per sec: 170399.47074266, Loss: 2.5021257400512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19879 , TFLOPS: 70.60979300537058, Tokens per sec: 174978.84537812477, Loss: 2.476689577102661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19880 , TFLOPS: 70.41314898464536, Tokens per sec: 174491.53983264053, Loss: 2.512238025665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19881 , TFLOPS: 69.76931873337033, Tokens per sec: 172896.05754622392, Loss: 2.4885945320129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19882 , TFLOPS: 69.04403061766395, Tokens per sec: 171098.7136984219, Loss: 2.4755053520202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19883 , TFLOPS: 71.49428475637967, Tokens per sec: 177170.71337193513, Loss: 2.5011191368103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19884 , TFLOPS: 69.69229081191067, Tokens per sec: 172705.17387152085, Loss: 2.4496474266052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19885 , TFLOPS: 68.77544361070082, Tokens per sec: 170433.12550786592, Loss: 2.484989881515503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19886 , TFLOPS: 70.59473773575218, Tokens per sec: 174941.5367615355, Loss: 2.500342845916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19887 , TFLOPS: 70.44513137301222, Tokens per sec: 174570.79571416453, Loss: 2.479569911956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19888 , TFLOPS: 69.10928960495406, Tokens per sec: 171260.43265779768, Loss: 2.4994661808013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19889 , TFLOPS: 69.80195235113126, Tokens per sec: 172976.9272459253, Loss: 2.4673945903778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19890 , TFLOPS: 71.47538129813178, Tokens per sec: 177123.868520009, Loss: 2.466559886932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19891 , TFLOPS: 68.80022713409471, Tokens per sec: 170494.54180896646, Loss: 2.502389907836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19892 , TFLOPS: 68.8326043920832, Tokens per sec: 170574.77622091136, Loss: 2.5059127807617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19893 , TFLOPS: 70.84918263799646, Tokens per sec: 175572.07925871672, Loss: 2.517279863357544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19894 , TFLOPS: 69.70342304682212, Tokens per sec: 172732.76077594835, Loss: 2.472160816192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19895 , TFLOPS: 68.57839758743948, Tokens per sec: 169944.82375581295, Loss: 2.4857845306396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19896 , TFLOPS: 70.61651800218554, Tokens per sec: 174995.5106610518, Loss: 2.505662441253662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19897 , TFLOPS: 70.60483515263326, Tokens per sec: 174966.5592728891, Loss: 2.4819910526275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19898 , TFLOPS: 68.72502513567747, Tokens per sec: 170308.18297270456, Loss: 2.48079514503479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19899 , TFLOPS: 69.92024219389894, Tokens per sec: 173270.06250700547, Loss: 2.5026357173919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19900 , TFLOPS: 71.48299840975203, Tokens per sec: 177142.7445616981, Loss: 2.4751904010772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19901 , TFLOPS: 70.68534351740493, Tokens per sec: 175166.06786952182, Loss: 2.484257698059082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19902 , TFLOPS: 67.3081147213497, Tokens per sec: 166796.92288043298, Loss: 2.4895129203796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19903 , TFLOPS: 71.53249110160792, Tokens per sec: 177265.39290418712, Loss: 2.49513578414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19904 , TFLOPS: 70.57724586039083, Tokens per sec: 174898.18996750095, Loss: 2.504758596420288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19905 , TFLOPS: 68.8091486015161, Tokens per sec: 170516.6501880174, Loss: 2.4625062942504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19906 , TFLOPS: 69.55647536288984, Tokens per sec: 172368.60822754042, Loss: 2.5012025833129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19907 , TFLOPS: 71.52214871447653, Tokens per sec: 177239.76332956753, Loss: 2.5148868560791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19908 , TFLOPS: 68.99646110019575, Tokens per sec: 170980.83119392072, Loss: 2.469768524169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19909 , TFLOPS: 68.7555571139964, Tokens per sec: 170383.84457835392, Loss: 2.484816551208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19910 , TFLOPS: 71.51482711284342, Tokens per sec: 177221.61959417968, Loss: 2.489470958709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19911 , TFLOPS: 69.69211462663316, Tokens per sec: 172704.73726499476, Loss: 2.5128071308135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19912 , TFLOPS: 68.81128483513945, Tokens per sec: 170521.94401026165, Loss: 2.491334915161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19913 , TFLOPS: 69.49306668505737, Tokens per sec: 172211.4745388291, Loss: 2.506930112838745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19914 , TFLOPS: 70.66766396602239, Tokens per sec: 175122.25599363254, Loss: 2.4837868213653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19915 , TFLOPS: 69.78166870710616, Tokens per sec: 172926.66214160324, Loss: 2.4870545864105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19916 , TFLOPS: 70.17315210960847, Tokens per sec: 173896.80113846084, Loss: 2.495619297027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19917 , TFLOPS: 69.67155922660203, Tokens per sec: 172653.79871935336, Loss: 2.483456611633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19918 , TFLOPS: 70.57754636596674, Tokens per sec: 174898.9346534215, Loss: 2.490856170654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19919 , TFLOPS: 69.02643701776053, Tokens per sec: 171055.11481976617, Loss: 2.4876909255981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19920 , TFLOPS: 71.54129147500497, Tokens per sec: 177287.20119890614, Loss: 2.478595495223999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19921 , TFLOPS: 69.64385324191834, Tokens per sec: 172585.140237241, Loss: 2.489936351776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19922 , TFLOPS: 68.96979151578158, Tokens per sec: 170914.74102584517, Loss: 2.510789394378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19923 , TFLOPS: 70.9040593065595, Tokens per sec: 175708.06968858003, Loss: 2.4720780849456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19924 , TFLOPS: 69.62445494130787, Tokens per sec: 172537.0691114291, Loss: 2.488060712814331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19925 , TFLOPS: 68.63028772588707, Tokens per sec: 170073.41323505674, Loss: 2.4683852195739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19926 , TFLOPS: 70.54106474590219, Tokens per sec: 174808.5291801193, Loss: 2.470431327819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19927 , TFLOPS: 71.49737326502543, Tokens per sec: 177178.36703658666, Loss: 2.462747812271118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19928 , TFLOPS: 69.01092054281918, Tokens per sec: 171016.6632855807, Loss: 2.511451005935669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19929 , TFLOPS: 69.32581917128852, Tokens per sec: 171797.0167758753, Loss: 2.4866394996643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19930 , TFLOPS: 71.54214016769916, Tokens per sec: 177289.30435289358, Loss: 2.505666494369507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19931 , TFLOPS: 69.54272387268794, Tokens per sec: 172334.53051996845, Loss: 2.4804608821868896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19932 , TFLOPS: 68.11711876036301, Tokens per sec: 168801.72400825986, Loss: 2.4875566959381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19933 , TFLOPS: 70.66186594406132, Tokens per sec: 175107.88785650814, Loss: 2.4990813732147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19934 , TFLOPS: 70.6717307054969, Tokens per sec: 175132.33382203872, Loss: 2.4860241413116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19935 , TFLOPS: 68.83166324594453, Tokens per sec: 170572.4439571039, Loss: 2.498037099838257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19936 , TFLOPS: 69.70415053296637, Tokens per sec: 172734.563566753, Loss: 2.508373498916626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19937 , TFLOPS: 71.51677772137317, Tokens per sec: 177226.4534169937, Loss: 2.4736368656158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19938 , TFLOPS: 69.98886135696064, Tokens per sec: 173440.1083520972, Loss: 2.5020713806152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19939 , TFLOPS: 67.92311222839895, Tokens per sec: 168320.95445046911, Loss: 2.4812722206115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19940 , TFLOPS: 71.52168863171055, Tokens per sec: 177238.62319379114, Loss: 2.49774169921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19941 , TFLOPS: 70.72420679659884, Tokens per sec: 175262.37535650795, Loss: 2.500126838684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19942 , TFLOPS: 68.796147674304, Tokens per sec: 170484.43245821548, Loss: 2.4772045612335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19943 , TFLOPS: 69.58316723949503, Tokens per sec: 172434.75363812104, Loss: 2.491764545440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19944 , TFLOPS: 70.57760564898523, Tokens per sec: 174899.08156327187, Loss: 2.494525909423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19945 , TFLOPS: 68.16051875745539, Tokens per sec: 168909.2739819591, Loss: 2.4951939582824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19946 , TFLOPS: 69.64553476582715, Tokens per sec: 172589.30723872222, Loss: 2.4867920875549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19947 , TFLOPS: 71.49629754267778, Tokens per sec: 177175.70127810858, Loss: 2.5048904418945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19948 , TFLOPS: 70.75895573577984, Tokens per sec: 175348.4870557101, Loss: 2.491957664489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19949 , TFLOPS: 68.29420424915712, Tokens per sec: 169240.56135706863, Loss: 2.4694321155548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19950 , TFLOPS: 70.52527317771877, Tokens per sec: 174769.39593457594, Loss: 2.499140739440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19951 , TFLOPS: 70.59503282756822, Tokens per sec: 174942.2680315627, Loss: 2.474149227142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19952 , TFLOPS: 67.98440728609108, Tokens per sec: 168472.85035563822, Loss: 2.4895124435424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19953 , TFLOPS: 70.64244673889482, Tokens per sec: 175059.76492690353, Loss: 2.470036506652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19954 , TFLOPS: 70.4651181287682, Tokens per sec: 174620.32509665072, Loss: 2.507418155670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19955 , TFLOPS: 69.23453211231006, Tokens per sec: 171570.79738762698, Loss: 2.4857516288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19956 , TFLOPS: 69.60529526792268, Tokens per sec: 172489.58933016832, Loss: 2.4889743328094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19957 , TFLOPS: 71.5419175951503, Tokens per sec: 177288.7527935991, Loss: 2.4980154037475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19958 , TFLOPS: 68.83832981190245, Tokens per sec: 170588.96444193053, Loss: 2.491419792175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19959 , TFLOPS: 69.61390835834852, Tokens per sec: 172510.93351705375, Loss: 2.5037918090820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19960 , TFLOPS: 70.93355464864734, Tokens per sec: 175781.16239545387, Loss: 2.480692148208618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19961 , TFLOPS: 69.7308605961367, Tokens per sec: 172800.75404564544, Loss: 2.5145864486694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19962 , TFLOPS: 68.83270344887396, Tokens per sec: 170575.02169455087, Loss: 2.497157096862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19963 , TFLOPS: 70.57832080607646, Tokens per sec: 174900.853801325, Loss: 2.507084608078003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19964 , TFLOPS: 70.60515683457682, Tokens per sec: 174967.35643618146, Loss: 2.479213237762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19965 , TFLOPS: 68.01905961924913, Tokens per sec: 168558.72265447353, Loss: 2.4851622581481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19966 , TFLOPS: 69.99488182263833, Tokens per sec: 173455.02772925555, Loss: 2.4838624000549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19967 , TFLOPS: 71.47494380729216, Tokens per sec: 177122.78436951488, Loss: 2.5045347213745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19968 , TFLOPS: 69.7680482045679, Tokens per sec: 172892.90903589118, Loss: 2.514037609100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19969 , TFLOPS: 68.02743160436732, Tokens per sec: 168579.4693558467, Loss: 2.490133762359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19970 , TFLOPS: 71.52806885297531, Tokens per sec: 177254.4340849246, Loss: 2.4915828704833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19971 , TFLOPS: 70.59531326521136, Tokens per sec: 174942.96298693516, Loss: 2.4691972732543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19972 , TFLOPS: 67.85378209411527, Tokens per sec: 168149.14674036944, Loss: 2.490534543991089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19973 , TFLOPS: 70.53257880420855, Tokens per sec: 174787.50008179885, Loss: 2.4791367053985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19974 , TFLOPS: 71.55173237604475, Tokens per sec: 177313.07490184295, Loss: 2.4947152137756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19975 , TFLOPS: 69.03562298044004, Tokens per sec: 171077.87864720332, Loss: 2.5003042221069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19976 , TFLOPS: 69.58638690647143, Tokens per sec: 172442.73232756412, Loss: 2.4742958545684814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19977 , TFLOPS: 71.51239880590406, Tokens per sec: 177215.60198208384, Loss: 2.4920175075531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19978 , TFLOPS: 69.79342488108725, Tokens per sec: 172955.7952357778, Loss: 2.4961488246917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19979 , TFLOPS: 69.60523166817237, Tokens per sec: 172489.43172298095, Loss: 2.484926700592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19980 , TFLOPS: 69.09484229831816, Tokens per sec: 171224.6306404517, Loss: 2.467989683151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19981 , TFLOPS: 70.69143369291936, Tokens per sec: 175181.1599953348, Loss: 2.4975194931030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19982 , TFLOPS: 69.57316747781255, Tokens per sec: 172409.97312710714, Loss: 2.4741344451904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19983 , TFLOPS: 68.2618209647072, Tokens per sec: 169160.31201088845, Loss: 2.4792580604553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19984 , TFLOPS: 70.58477977070605, Tokens per sec: 174916.85982719078, Loss: 2.46897554397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19985 , TFLOPS: 70.67265147652046, Tokens per sec: 175134.6155940664, Loss: 2.496283769607544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19986 , TFLOPS: 68.99637859271124, Tokens per sec: 170980.62673128519, Loss: 2.490863084793091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19987 , TFLOPS: 70.5097731321947, Tokens per sec: 174730.98511429632, Loss: 2.504009246826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19988 , TFLOPS: 70.6610784375029, Tokens per sec: 175105.93632850537, Loss: 2.499389410018921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19989 , TFLOPS: 67.90780949496026, Tokens per sec: 168283.03259127281, Loss: 2.458456516265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19990 , TFLOPS: 71.51164241647129, Tokens per sec: 177213.7275657463, Loss: 2.4836041927337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19991 , TFLOPS: 69.50141504892876, Tokens per sec: 172232.16270415057, Loss: 2.508056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19992 , TFLOPS: 68.202983249799, Tokens per sec: 169014.5056718366, Loss: 2.481529474258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19993 , TFLOPS: 70.65979195260194, Tokens per sec: 175102.7482772028, Loss: 2.4925522804260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19994 , TFLOPS: 71.53194513079744, Tokens per sec: 177264.03992837507, Loss: 2.4975714683532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19995 , TFLOPS: 67.82429667194928, Tokens per sec: 168076.0786161547, Loss: 2.5323739051818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19996 , TFLOPS: 69.65182362825335, Tokens per sec: 172604.89173258975, Loss: 2.49345326423645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19997 , TFLOPS: 71.52707779490513, Tokens per sec: 177251.9781338526, Loss: 2.5029079914093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19998 , TFLOPS: 69.62369917775793, Tokens per sec: 172535.19624609812, Loss: 2.4849743843078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19999 , TFLOPS: 69.15960922102128, Tokens per sec: 171385.13020957558, Loss: 2.4982733726501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20000 , TFLOPS: 70.67803198009484, Tokens per sec: 175147.9490746351, Loss: 2.4845948219299316 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/optimizer.pt +[rank0]:[2024-08-27 20:16:05,625] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006400677026249468, 'preprocessing_with_comm': 0.0012597659952007234, 'state_converting': 0.9725176839856431, : 0.9815680489991792}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000, took 4.51s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018000 + eval ppl=10.886109352111816, eval loss=2.3874876499176025 +------------------------------------------------------------------ +iteration: 20001 , TFLOPS: 64.53294998512237, Tokens per sec: 159919.75895442895, Loss: 2.4941201210021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20002 , TFLOPS: 66.91869821764232, Tokens per sec: 165831.9058864769, Loss: 2.4911420345306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20003 , TFLOPS: 70.61003441328778, Tokens per sec: 174979.44361353686, Loss: 2.50331974029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20004 , TFLOPS: 70.70195129024307, Tokens per sec: 175207.22376011196, Loss: 2.486947536468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20005 , TFLOPS: 69.70246273646501, Tokens per sec: 172730.3810210965, Loss: 2.483466863632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20006 , TFLOPS: 70.3829371095888, Tokens per sec: 174416.67147813758, Loss: 2.5150110721588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20007 , TFLOPS: 70.77653037141633, Tokens per sec: 175392.03893882327, Loss: 2.492826461791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20008 , TFLOPS: 70.32721929681487, Tokens per sec: 174278.59660026003, Loss: 2.502354860305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20009 , TFLOPS: 67.31306100603318, Tokens per sec: 166809.18031875687, Loss: 2.46132230758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20010 , TFLOPS: 70.63548376648842, Tokens per sec: 175042.50991424575, Loss: 2.4971306324005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20011 , TFLOPS: 70.69510200076235, Tokens per sec: 175190.2504662672, Loss: 2.489748239517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20012 , TFLOPS: 68.88745233228043, Tokens per sec: 170710.69545871948, Loss: 2.458637237548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20013 , TFLOPS: 69.43534549489557, Tokens per sec: 172068.43507109414, Loss: 2.496375799179077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20014 , TFLOPS: 71.4890757650602, Tokens per sec: 177157.80491762774, Loss: 2.5087971687316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20015 , TFLOPS: 69.05041058462038, Tokens per sec: 171114.5239593513, Loss: 2.4888181686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20016 , TFLOPS: 68.91150914207583, Tokens per sec: 170770.31088346883, Loss: 2.4893836975097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20017 , TFLOPS: 70.07627854781576, Tokens per sec: 173656.7377808347, Loss: 2.4846527576446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20018 , TFLOPS: 70.73444040225768, Tokens per sec: 175287.73535865528, Loss: 2.4852592945098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20019 , TFLOPS: 69.72130560597091, Tokens per sec: 172777.07572744557, Loss: 2.453543186187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20020 , TFLOPS: 69.49810494977129, Tokens per sec: 172223.95991380085, Loss: 2.450700521469116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20021 , TFLOPS: 70.60318376950632, Tokens per sec: 174962.46696358567, Loss: 2.4838054180145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20022 , TFLOPS: 68.16135680808476, Tokens per sec: 168911.3507637376, Loss: 2.474679470062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20023 , TFLOPS: 70.69181914094916, Tokens per sec: 175182.11517801328, Loss: 2.4927213191986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20024 , TFLOPS: 69.70590644452577, Tokens per sec: 172738.91490902682, Loss: 2.4843263626098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20025 , TFLOPS: 70.66832802256341, Tokens per sec: 175123.90159889308, Loss: 2.485173463821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20026 , TFLOPS: 70.76065327392841, Tokens per sec: 175352.6937422416, Loss: 2.490316867828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20027 , TFLOPS: 70.65393687667007, Tokens per sec: 175088.23875405377, Loss: 2.4939794540405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20028 , TFLOPS: 70.4499451776196, Tokens per sec: 174582.72485226594, Loss: 2.474371910095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20029 , TFLOPS: 67.95547284228464, Tokens per sec: 168401.14761649378, Loss: 2.4741768836975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20030 , TFLOPS: 70.5252163030744, Tokens per sec: 174769.25499294195, Loss: 2.5118942260742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20031 , TFLOPS: 69.5179205731808, Tokens per sec: 172273.06521148072, Loss: 2.5024333000183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20032 , TFLOPS: 68.96355933286118, Tokens per sec: 170899.29699003682, Loss: 2.486666202545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20033 , TFLOPS: 70.56873296402028, Tokens per sec: 174877.09407252178, Loss: 2.4786429405212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20034 , TFLOPS: 71.53876815626725, Tokens per sec: 177280.94813710757, Loss: 2.4792065620422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20035 , TFLOPS: 67.92813076347886, Tokens per sec: 168333.39093323442, Loss: 2.4905617237091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20036 , TFLOPS: 68.71332509348409, Tokens per sec: 170279.18897928332, Loss: 2.494253635406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20037 , TFLOPS: 71.5159777873283, Tokens per sec: 177224.47109231044, Loss: 2.4678831100463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20038 , TFLOPS: 70.63309141946175, Tokens per sec: 175036.58141477488, Loss: 2.4763987064361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20039 , TFLOPS: 68.43676067404539, Tokens per sec: 169593.83188182837, Loss: 2.496638536453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20040 , TFLOPS: 70.62161512510517, Tokens per sec: 175008.1418931411, Loss: 2.4969582557678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20041 , TFLOPS: 70.61552171541115, Tokens per sec: 174993.0417526748, Loss: 2.4691953659057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20042 , TFLOPS: 68.4724185895973, Tokens per sec: 169682.19612460965, Loss: 2.4976813793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20043 , TFLOPS: 68.39877416817959, Tokens per sec: 169499.69713573382, Loss: 2.492656946182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20044 , TFLOPS: 70.70439579872563, Tokens per sec: 175213.2815214166, Loss: 2.508155345916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20045 , TFLOPS: 71.51973333031323, Tokens per sec: 177233.77774153432, Loss: 2.498478412628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20046 , TFLOPS: 69.05407985208966, Tokens per sec: 171123.61680834336, Loss: 2.4975295066833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20047 , TFLOPS: 70.6076164628039, Tokens per sec: 174973.4516658786, Loss: 2.478846311569214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20048 , TFLOPS: 70.5791213164098, Tokens per sec: 174902.83755411446, Loss: 2.502671241760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20049 , TFLOPS: 67.99401850294704, Tokens per sec: 168496.66800977025, Loss: 2.4929182529449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20050 , TFLOPS: 69.33404038500902, Tokens per sec: 171817.38984911615, Loss: 2.4622368812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20051 , TFLOPS: 70.61868879187891, Tokens per sec: 175000.8901170442, Loss: 2.506521463394165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20052 , TFLOPS: 68.84427993099999, Tokens per sec: 170603.70949251394, Loss: 2.5037646293640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20053 , TFLOPS: 70.58368410438979, Tokens per sec: 174914.1446453615, Loss: 2.4926154613494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20054 , TFLOPS: 70.38319762867464, Tokens per sec: 174417.31707313086, Loss: 2.4730031490325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20055 , TFLOPS: 68.90890878137844, Tokens per sec: 170763.86690320735, Loss: 2.4751222133636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20056 , TFLOPS: 69.63907416274061, Tokens per sec: 172573.2971525796, Loss: 2.4824256896972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20057 , TFLOPS: 70.82736546865306, Tokens per sec: 175518.01390972416, Loss: 2.499373197555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20058 , TFLOPS: 69.63121167956311, Tokens per sec: 172553.8130531417, Loss: 2.476423978805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20059 , TFLOPS: 69.7287307482008, Tokens per sec: 172795.47604783816, Loss: 2.495715379714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20060 , TFLOPS: 70.63645362605158, Tokens per sec: 175044.91333308208, Loss: 2.496638298034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20061 , TFLOPS: 69.58425023600785, Tokens per sec: 172437.43742278172, Loss: 2.4897468090057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20062 , TFLOPS: 69.76950886152638, Tokens per sec: 172896.52870473894, Loss: 2.5053610801696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20063 , TFLOPS: 69.05928347485984, Tokens per sec: 171136.5119587942, Loss: 2.504934310913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20064 , TFLOPS: 71.53807258990994, Tokens per sec: 177279.22444705124, Loss: 2.478057384490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20065 , TFLOPS: 69.97853436298237, Tokens per sec: 173414.51692340674, Loss: 2.4945430755615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20066 , TFLOPS: 69.68429536788881, Tokens per sec: 172685.3602804638, Loss: 2.492053270339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20067 , TFLOPS: 70.60628813864983, Tokens per sec: 174970.1599322978, Loss: 2.5196166038513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20068 , TFLOPS: 69.64307120516057, Tokens per sec: 172583.2022640068, Loss: 2.4893157482147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20069 , TFLOPS: 67.99704031522208, Tokens per sec: 168504.1563934679, Loss: 2.479152202606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20070 , TFLOPS: 70.59105579732484, Tokens per sec: 174932.41251251355, Loss: 2.4793789386749268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20071 , TFLOPS: 70.60351347683121, Tokens per sec: 174963.28401465702, Loss: 2.4975082874298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20072 , TFLOPS: 69.26878344180493, Tokens per sec: 171655.67595521067, Loss: 2.5038485527038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20073 , TFLOPS: 70.96206431319924, Tokens per sec: 175851.81248481225, Loss: 2.480440855026245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20074 , TFLOPS: 70.91933876158757, Tokens per sec: 175745.93386131045, Loss: 2.4681246280670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20075 , TFLOPS: 69.85947601416049, Tokens per sec: 173119.47721966633, Loss: 2.4896609783172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20076 , TFLOPS: 67.50570102966711, Tokens per sec: 167286.56351837222, Loss: 2.496352434158325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20077 , TFLOPS: 70.56735045793506, Tokens per sec: 174873.668070147, Loss: 2.492666721343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20078 , TFLOPS: 70.61974087647194, Tokens per sec: 175003.49729856238, Loss: 2.494415283203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20079 , TFLOPS: 69.59147971088912, Tokens per sec: 172455.35285792992, Loss: 2.479818344116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20080 , TFLOPS: 69.70306854572354, Tokens per sec: 172731.88228317452, Loss: 2.492107629776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20081 , TFLOPS: 71.53753573596164, Tokens per sec: 177277.89406382796, Loss: 2.4981441497802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20082 , TFLOPS: 69.1381033238567, Tokens per sec: 171331.83622733157, Loss: 2.5188910961151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20083 , TFLOPS: 67.88532363857107, Tokens per sec: 168227.31016211884, Loss: 2.4736948013305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20084 , TFLOPS: 71.52905709726043, Tokens per sec: 177256.88306312723, Loss: 2.4805397987365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20085 , TFLOPS: 70.5825910127081, Tokens per sec: 174911.43584376026, Loss: 2.5059711933135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20086 , TFLOPS: 69.50199580865015, Tokens per sec: 172233.60189071612, Loss: 2.478102684020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20087 , TFLOPS: 69.6191541204535, Tokens per sec: 172523.93309341834, Loss: 2.5042572021484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20088 , TFLOPS: 70.75223705389512, Tokens per sec: 175331.83742187588, Loss: 2.4772677421569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20089 , TFLOPS: 68.10610277297302, Tokens per sec: 168774.42517799625, Loss: 2.4822566509246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20090 , TFLOPS: 70.60654946645168, Tokens per sec: 174970.80753137844, Loss: 2.4951300621032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20091 , TFLOPS: 69.59928626825604, Tokens per sec: 172474.6983670484, Loss: 2.4727089405059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20092 , TFLOPS: 70.58440995246248, Tokens per sec: 174915.94337684437, Loss: 2.491396188735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20093 , TFLOPS: 69.82271972298152, Tokens per sec: 173028.39108108505, Loss: 2.492354393005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20094 , TFLOPS: 69.61023622939533, Tokens per sec: 172501.83357698936, Loss: 2.4928600788116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20095 , TFLOPS: 70.69189114154253, Tokens per sec: 175182.29360341514, Loss: 2.497098922729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20096 , TFLOPS: 68.4753958310696, Tokens per sec: 169689.57405694824, Loss: 2.4766058921813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20097 , TFLOPS: 70.6358637323842, Tokens per sec: 175043.45151159217, Loss: 2.485583543777466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20098 , TFLOPS: 69.65365495202165, Tokens per sec: 172609.42995462453, Loss: 2.4927573204040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20099 , TFLOPS: 68.82976842323828, Tokens per sec: 170567.74837770587, Loss: 2.4611358642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20100 , TFLOPS: 70.56675896214198, Tokens per sec: 174872.20227841305, Loss: 2.4773924350738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20101 , TFLOPS: 71.49585335590606, Tokens per sec: 177174.600534357, Loss: 2.492091178894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20102 , TFLOPS: 67.9730760847169, Tokens per sec: 168444.77039039816, Loss: 2.474439859390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20103 , TFLOPS: 69.03317256618718, Tokens per sec: 171071.80625074933, Loss: 2.459656000137329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20104 , TFLOPS: 71.51285678916926, Tokens per sec: 177216.73691506605, Loss: 2.5059266090393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20105 , TFLOPS: 69.6642983369567, Tokens per sec: 172635.80543495924, Loss: 2.4891092777252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20106 , TFLOPS: 69.59607361366618, Tokens per sec: 172466.7370550716, Loss: 2.5085859298706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20107 , TFLOPS: 70.60642480517531, Tokens per sec: 174970.498607003, Loss: 2.5088024139404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20108 , TFLOPS: 70.70583641768198, Tokens per sec: 175216.8515338877, Loss: 2.4614648818969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20109 , TFLOPS: 68.02366984434563, Tokens per sec: 168570.14730012088, Loss: 2.497077226638794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20110 , TFLOPS: 69.16165509544854, Tokens per sec: 171390.200111778, Loss: 2.512537956237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20111 , TFLOPS: 71.53806668373167, Tokens per sec: 177279.20981089084, Loss: 2.4830358028411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20112 , TFLOPS: 70.80363428270118, Tokens per sec: 175459.2054167295, Loss: 2.5000813007354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20113 , TFLOPS: 68.82975209251339, Tokens per sec: 170567.70790837065, Loss: 2.481265068054199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20114 , TFLOPS: 70.66954985281345, Tokens per sec: 175126.92942884835, Loss: 2.4967005252838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20115 , TFLOPS: 70.68172206345648, Tokens per sec: 175157.0935077015, Loss: 2.449597120285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20116 , TFLOPS: 68.16697801680212, Tokens per sec: 168925.28073816648, Loss: 2.472891330718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20117 , TFLOPS: 69.54411477348009, Tokens per sec: 172337.97732535176, Loss: 2.495452880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20118 , TFLOPS: 70.60094391562903, Tokens per sec: 174956.91635893748, Loss: 2.506234645843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20119 , TFLOPS: 69.63781854976737, Tokens per sec: 172570.18560531473, Loss: 2.4670753479003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20120 , TFLOPS: 69.59092598510388, Tokens per sec: 172453.98066443615, Loss: 2.4913203716278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20121 , TFLOPS: 71.50253316371658, Tokens per sec: 177191.15383395308, Loss: 2.489260673522949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20122 , TFLOPS: 69.0032342107224, Tokens per sec: 170997.61570792546, Loss: 2.501213312149048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20123 , TFLOPS: 69.9067839195656, Tokens per sec: 173236.71142065615, Loss: 2.4581456184387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20124 , TFLOPS: 69.89748568189516, Tokens per sec: 173213.66936342337, Loss: 2.503451108932495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20125 , TFLOPS: 70.595815624406, Tokens per sec: 174944.2078883589, Loss: 2.4925100803375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20126 , TFLOPS: 69.55305076179008, Tokens per sec: 172360.12168876425, Loss: 2.4744248390197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20127 , TFLOPS: 70.6029946726091, Tokens per sec: 174961.99836064363, Loss: 2.5084004402160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20128 , TFLOPS: 69.63080538372323, Tokens per sec: 172552.80620729332, Loss: 2.4686074256896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20129 , TFLOPS: 69.81162235158776, Tokens per sec: 173000.89057229494, Loss: 2.4957916736602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20130 , TFLOPS: 69.32434528957809, Tokens per sec: 171793.36433463538, Loss: 2.4828386306762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20131 , TFLOPS: 71.55830594947969, Tokens per sec: 177329.36494095362, Loss: 2.498016595840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20132 , TFLOPS: 68.53064940294344, Tokens per sec: 169826.49849473502, Loss: 2.5023179054260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20133 , TFLOPS: 68.69095630321131, Tokens per sec: 170223.7566528617, Loss: 2.494265079498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20134 , TFLOPS: 70.54481728032074, Tokens per sec: 174817.8283737843, Loss: 2.4987683296203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20135 , TFLOPS: 69.6516453385573, Tokens per sec: 172604.4499110827, Loss: 2.5149497985839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20136 , TFLOPS: 68.1662846499488, Tokens per sec: 168923.56249872327, Loss: 2.4925923347473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20137 , TFLOPS: 70.63074466137498, Tokens per sec: 175030.76588971913, Loss: 2.4769251346588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20138 , TFLOPS: 70.6666732529876, Tokens per sec: 175119.80089759626, Loss: 2.4784674644470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20139 , TFLOPS: 68.59742739904837, Tokens per sec: 169991.9817252866, Loss: 2.4903407096862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20140 , TFLOPS: 71.53073130794031, Tokens per sec: 177261.03194162148, Loss: 2.48783278465271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20141 , TFLOPS: 70.80362633827582, Tokens per sec: 175459.18572956827, Loss: 2.4833922386169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20142 , TFLOPS: 68.89026146740804, Tokens per sec: 170717.65680501747, Loss: 2.488067626953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20143 , TFLOPS: 69.10524162183411, Tokens per sec: 171250.4013097043, Loss: 2.4966468811035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20144 , TFLOPS: 70.66183045547984, Tokens per sec: 175107.7999118939, Loss: 2.4812564849853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20145 , TFLOPS: 70.62528134405477, Tokens per sec: 175017.22718756556, Loss: 2.47762393951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20146 , TFLOPS: 68.62629385494543, Tokens per sec: 170063.5159829021, Loss: 2.4919915199279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20147 , TFLOPS: 70.63942673791188, Tokens per sec: 175052.28103178722, Loss: 2.4807872772216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20148 , TFLOPS: 71.51883718226364, Tokens per sec: 177231.55698794857, Loss: 2.5034291744232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20149 , TFLOPS: 69.25904102628057, Tokens per sec: 171631.53317632564, Loss: 2.5092663764953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20150 , TFLOPS: 68.73005663556563, Tokens per sec: 170320.65158369287, Loss: 2.4964215755462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20151 , TFLOPS: 71.53261277194045, Tokens per sec: 177265.69441667423, Loss: 2.4762792587280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20152 , TFLOPS: 70.6381943187124, Tokens per sec: 175049.22696124876, Loss: 2.482351541519165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20153 , TFLOPS: 68.69591631165726, Tokens per sec: 170236.04810018078, Loss: 2.4951555728912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20154 , TFLOPS: 69.61214901489804, Tokens per sec: 172506.5736701758, Loss: 2.497504949569702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20155 , TFLOPS: 70.61036427860776, Tokens per sec: 174980.26105613747, Loss: 2.475398063659668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20156 , TFLOPS: 68.1569836069011, Tokens per sec: 168900.51348945659, Loss: 2.4855730533599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20157 , TFLOPS: 69.60689830498677, Tokens per sec: 172493.56183260225, Loss: 2.468903064727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20158 , TFLOPS: 70.55618875877074, Tokens per sec: 174846.00815005513, Loss: 2.492962121963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20159 , TFLOPS: 70.61679330782755, Tokens per sec: 174996.1928987601, Loss: 2.4940414428710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20160 , TFLOPS: 70.71011010346075, Tokens per sec: 175227.44219803283, Loss: 2.4834136962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20161 , TFLOPS: 69.34658627265708, Tokens per sec: 171848.47994075165, Loss: 2.483672618865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20162 , TFLOPS: 70.63667241879493, Tokens per sec: 175045.45552560213, Loss: 2.5091779232025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20163 , TFLOPS: 68.33737582143381, Tokens per sec: 169347.54526891155, Loss: 2.4915945529937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20164 , TFLOPS: 70.41687425684229, Tokens per sec: 174500.77146751713, Loss: 2.4632415771484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20165 , TFLOPS: 69.55973066131847, Tokens per sec: 172376.67521564726, Loss: 2.4934189319610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20166 , TFLOPS: 69.54649978799944, Tokens per sec: 172343.88765406172, Loss: 2.478436231613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20167 , TFLOPS: 70.64298245812266, Tokens per sec: 175061.09249816448, Loss: 2.4858808517456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20168 , TFLOPS: 70.49217537369785, Tokens per sec: 174687.375930188, Loss: 2.478881359100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20169 , TFLOPS: 68.85285672314032, Tokens per sec: 170624.96372824933, Loss: 2.471052885055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20170 , TFLOPS: 69.82798283901819, Tokens per sec: 173041.43366240404, Loss: 2.4954066276550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20171 , TFLOPS: 70.48262514576925, Tokens per sec: 174663.7094133365, Loss: 2.5094354152679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20172 , TFLOPS: 69.71297060213004, Tokens per sec: 172756.42066975142, Loss: 2.4806809425354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20173 , TFLOPS: 68.86184325681982, Tokens per sec: 170647.23334865554, Loss: 2.478375196456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20174 , TFLOPS: 70.61043506831015, Tokens per sec: 174980.4364808181, Loss: 2.5003910064697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20175 , TFLOPS: 70.58893752844584, Tokens per sec: 174927.16320888483, Loss: 2.4852733612060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20176 , TFLOPS: 68.7585576626396, Tokens per sec: 170391.2802684295, Loss: 2.4875171184539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20177 , TFLOPS: 70.0412898703517, Tokens per sec: 173570.03198375774, Loss: 2.4935526847839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20178 , TFLOPS: 70.72743810058392, Tokens per sec: 175270.38288374053, Loss: 2.480677843093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20179 , TFLOPS: 70.48526029048857, Tokens per sec: 174670.2395922361, Loss: 2.4927167892456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20180 , TFLOPS: 69.97090669187145, Tokens per sec: 173395.6147141365, Loss: 2.5075442790985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20181 , TFLOPS: 70.59154550246379, Tokens per sec: 174933.62605579337, Loss: 2.493368625640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20182 , TFLOPS: 70.54321242904793, Tokens per sec: 174813.85137554127, Loss: 2.468629837036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20183 , TFLOPS: 67.8896637095528, Tokens per sec: 168238.06533610987, Loss: 2.470067024230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20184 , TFLOPS: 70.69735164700592, Tokens per sec: 175195.82533747776, Loss: 2.515397071838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20185 , TFLOPS: 70.75406641205531, Tokens per sec: 175336.3707729173, Loss: 2.5152273178100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20186 , TFLOPS: 69.63760983946774, Tokens per sec: 172569.6683982009, Loss: 2.485551357269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20187 , TFLOPS: 69.63165702102268, Tokens per sec: 172554.9166583369, Loss: 2.500991106033325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20188 , TFLOPS: 71.53651034324882, Tokens per sec: 177275.3530277293, Loss: 2.5016255378723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20189 , TFLOPS: 69.71222208590045, Tokens per sec: 172754.5657640786, Loss: 2.4956183433532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20190 , TFLOPS: 69.11359411182043, Tokens per sec: 171271.09969999312, Loss: 2.505399227142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20191 , TFLOPS: 69.58717914970622, Tokens per sec: 172444.69559357275, Loss: 2.4912712574005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20192 , TFLOPS: 70.69917460867387, Tokens per sec: 175200.34283730676, Loss: 2.495844602584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20193 , TFLOPS: 68.76997743114116, Tokens per sec: 170419.5797127677, Loss: 2.508490800857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20194 , TFLOPS: 70.44927245135624, Tokens per sec: 174581.05776247845, Loss: 2.487287998199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20195 , TFLOPS: 70.56268177038808, Tokens per sec: 174862.09854810717, Loss: 2.4987101554870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20196 , TFLOPS: 69.71204416595474, Tokens per sec: 172754.1248588536, Loss: 2.466895580291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20197 , TFLOPS: 69.21490090248938, Tokens per sec: 171522.14908713504, Loss: 2.4949405193328857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20198 , TFLOPS: 70.49693591820458, Tokens per sec: 174699.17308389297, Loss: 2.5104129314422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20199 , TFLOPS: 70.96179716101027, Tokens per sec: 175851.1504522587, Loss: 2.501399517059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20200 , TFLOPS: 69.12817563494346, Tokens per sec: 171307.23432057936, Loss: 2.4807186126708984 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/model.pt +[rank0]:[2024-08-27 20:41:28,538] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006466332008130848, 'preprocessing_with_comm': 0.001391254976624623, 'state_converting': 1.0074351470102556, : 1.016651362995617}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018200 + eval ppl=10.105237007141113, eval loss=2.313053846359253 +------------------------------------------------------------------ +iteration: 20201 , TFLOPS: 65.35551305405184, Tokens per sec: 161958.1608520385, Loss: 2.4854736328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20202 , TFLOPS: 70.60012598312557, Tokens per sec: 174954.8894320912, Loss: 2.4941277503967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20203 , TFLOPS: 68.178002960997, Tokens per sec: 168952.60176437924, Loss: 2.486781120300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20204 , TFLOPS: 69.82701799723115, Tokens per sec: 173039.04267816967, Loss: 2.509626626968384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20205 , TFLOPS: 69.63695201517956, Tokens per sec: 172568.03823714945, Loss: 2.4810919761657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20206 , TFLOPS: 68.11199624695216, Tokens per sec: 168789.02985573633, Loss: 2.4891433715820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20207 , TFLOPS: 68.94147477018552, Tokens per sec: 170844.56901091107, Loss: 2.506070852279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20208 , TFLOPS: 71.49855407302049, Tokens per sec: 177181.29320887514, Loss: 2.4849424362182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20209 , TFLOPS: 67.88629157243486, Tokens per sec: 168229.70880886, Loss: 2.491905927658081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20210 , TFLOPS: 69.02841407239241, Tokens per sec: 171060.0141789345, Loss: 2.4998953342437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20211 , TFLOPS: 70.64467097733448, Tokens per sec: 175065.27683477712, Loss: 2.490476369857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20212 , TFLOPS: 69.47492521155108, Tokens per sec: 172166.51796903223, Loss: 2.492098808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20213 , TFLOPS: 68.92141552424144, Tokens per sec: 170794.8599897533, Loss: 2.48954439163208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20214 , TFLOPS: 69.68786245254893, Tokens per sec: 172694.1999091984, Loss: 2.4713685512542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20215 , TFLOPS: 70.58589069728119, Tokens per sec: 174919.61282562267, Loss: 2.4971964359283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20216 , TFLOPS: 67.69364976239169, Tokens per sec: 167752.3211823258, Loss: 2.491208791732788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20217 , TFLOPS: 70.73161124607209, Tokens per sec: 175280.72439797013, Loss: 2.476823568344116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20218 , TFLOPS: 69.8649290891477, Tokens per sec: 173132.9905401904, Loss: 2.4880664348602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20219 , TFLOPS: 70.70770402066559, Tokens per sec: 175221.47965981453, Loss: 2.5026681423187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20220 , TFLOPS: 69.60985831237466, Tokens per sec: 172500.89705698116, Loss: 2.489997386932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20221 , TFLOPS: 69.67899909559108, Tokens per sec: 172672.23553427748, Loss: 2.5044333934783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20222 , TFLOPS: 70.56563187716546, Tokens per sec: 174869.40923768262, Loss: 2.49470853805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20223 , TFLOPS: 68.3764598312949, Tokens per sec: 169444.39975080645, Loss: 2.482391357421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20224 , TFLOPS: 68.85901155942727, Tokens per sec: 170640.2160905796, Loss: 2.5033535957336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20225 , TFLOPS: 70.65879298266745, Tokens per sec: 175100.27271965952, Loss: 2.4589860439300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20226 , TFLOPS: 69.66905828952873, Tokens per sec: 172647.60112178483, Loss: 2.475665807723999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20227 , TFLOPS: 69.71994294739082, Tokens per sec: 172773.69890937486, Loss: 2.4931082725524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20228 , TFLOPS: 70.9609163860647, Tokens per sec: 175848.96779491825, Loss: 2.4682962894439697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20229 , TFLOPS: 68.92587714647456, Tokens per sec: 170805.91638113523, Loss: 2.49947452545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20230 , TFLOPS: 69.70581907444821, Tokens per sec: 172738.69839635002, Loss: 2.4932026863098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20231 , TFLOPS: 68.84707516099195, Tokens per sec: 170610.63638035403, Loss: 2.4836013317108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20232 , TFLOPS: 70.60734158662548, Tokens per sec: 174972.77049242836, Loss: 2.4899697303771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20233 , TFLOPS: 68.89887471899773, Tokens per sec: 170739.00139128472, Loss: 2.4968419075012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20234 , TFLOPS: 69.91565119246576, Tokens per sec: 173258.68549971347, Loss: 2.4840354919433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20235 , TFLOPS: 69.0402412271502, Tokens per sec: 171089.32317129296, Loss: 2.4802792072296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20236 , TFLOPS: 68.78694512492177, Tokens per sec: 170461.62752710184, Loss: 2.5157485008239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20237 , TFLOPS: 70.01755169699454, Tokens per sec: 173511.20617520306, Loss: 2.4810447692871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20238 , TFLOPS: 69.51160750539401, Tokens per sec: 172257.42073406326, Loss: 2.4753646850585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20239 , TFLOPS: 71.53243431115978, Tokens per sec: 177265.2521712006, Loss: 2.4998068809509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20240 , TFLOPS: 69.85114449344552, Tokens per sec: 173098.83079354183, Loss: 2.497124671936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20241 , TFLOPS: 69.93273263019213, Tokens per sec: 173301.01518407514, Loss: 2.485408067703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20242 , TFLOPS: 69.60247032276625, Tokens per sec: 172482.58880487728, Loss: 2.462179660797119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20243 , TFLOPS: 67.95841896670991, Tokens per sec: 168408.448437362, Loss: 2.471025228500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20244 , TFLOPS: 69.76375700285182, Tokens per sec: 172882.27496532386, Loss: 2.528672218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20245 , TFLOPS: 70.57597896550297, Tokens per sec: 174895.05046240884, Loss: 2.484971523284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20246 , TFLOPS: 68.77444083762252, Tokens per sec: 170430.64052571333, Loss: 2.4717977046966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20247 , TFLOPS: 70.70923839836712, Tokens per sec: 175225.2820167846, Loss: 2.49398136138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20248 , TFLOPS: 70.67795340271, Tokens per sec: 175147.75435121948, Loss: 2.4903018474578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20249 , TFLOPS: 68.71098268456642, Tokens per sec: 170273.38423194768, Loss: 2.488926410675049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20250 , TFLOPS: 69.14350489616098, Tokens per sec: 171345.2219185339, Loss: 2.4689371585845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20251 , TFLOPS: 69.76262694448707, Tokens per sec: 172879.4745562098, Loss: 2.473227024078369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20252 , TFLOPS: 70.55947847079004, Tokens per sec: 174854.16041883067, Loss: 2.4956917762756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20253 , TFLOPS: 67.92589650232316, Tokens per sec: 168327.85418796638, Loss: 2.4854586124420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20254 , TFLOPS: 69.89511029680195, Tokens per sec: 173207.78289749208, Loss: 2.4833614826202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20255 , TFLOPS: 70.69468802768027, Tokens per sec: 175189.22459536695, Loss: 2.493318796157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20256 , TFLOPS: 69.92455735116663, Tokens per sec: 173280.75594206862, Loss: 2.495110511779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20257 , TFLOPS: 68.1788023803188, Tokens per sec: 168954.582813522, Loss: 2.4786930084228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20258 , TFLOPS: 70.723042650281, Tokens per sec: 175259.49047369108, Loss: 2.4890661239624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20259 , TFLOPS: 70.89498840496259, Tokens per sec: 175685.59099518057, Loss: 2.482365131378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20260 , TFLOPS: 69.62017904111774, Tokens per sec: 172526.47295972862, Loss: 2.5008432865142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20261 , TFLOPS: 68.82103362483954, Tokens per sec: 170546.10258505636, Loss: 2.51357364654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20262 , TFLOPS: 70.6807886386283, Tokens per sec: 175154.780378152, Loss: 2.486747980117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20263 , TFLOPS: 68.18652604465738, Tokens per sec: 168973.72290458484, Loss: 2.47823429107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20264 , TFLOPS: 69.50328225002444, Tokens per sec: 172236.78983415486, Loss: 2.4938430786132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20265 , TFLOPS: 70.14414582603396, Tokens per sec: 173824.92037245727, Loss: 2.503882884979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20266 , TFLOPS: 69.70164624633381, Tokens per sec: 172728.35766860755, Loss: 2.486994981765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20267 , TFLOPS: 71.5040026279076, Tokens per sec: 177194.79532809326, Loss: 2.4870128631591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20268 , TFLOPS: 69.14516323925815, Tokens per sec: 171349.33147541026, Loss: 2.4804348945617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20269 , TFLOPS: 70.76127707818674, Tokens per sec: 175354.23959791142, Loss: 2.494811773300171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20270 , TFLOPS: 68.421809947058, Tokens per sec: 169556.78233339504, Loss: 2.479766607284546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20271 , TFLOPS: 69.79377038347357, Tokens per sec: 172956.65142875136, Loss: 2.5000500679016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20272 , TFLOPS: 69.47661920212747, Tokens per sec: 172170.71586428897, Loss: 2.48911190032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20273 , TFLOPS: 68.7735618341266, Tokens per sec: 170428.46225822018, Loss: 2.490577220916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20274 , TFLOPS: 69.66838556039492, Tokens per sec: 172645.93402488396, Loss: 2.4829821586608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20275 , TFLOPS: 71.47119695112855, Tokens per sec: 177113.49924719112, Loss: 2.468646764755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20276 , TFLOPS: 68.80876726891417, Tokens per sec: 170515.7052038226, Loss: 2.499483108520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20277 , TFLOPS: 69.00652798912797, Tokens per sec: 171005.77805365415, Loss: 2.487936496734619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20278 , TFLOPS: 70.62749291297682, Tokens per sec: 175022.70769897883, Loss: 2.4929065704345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20279 , TFLOPS: 69.47862418070719, Tokens per sec: 172175.68442207613, Loss: 2.5019781589508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20280 , TFLOPS: 69.65946580422573, Tokens per sec: 172623.82988650448, Loss: 2.4940240383148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20281 , TFLOPS: 69.57718540333437, Tokens per sec: 172419.92998916126, Loss: 2.5047080516815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20282 , TFLOPS: 70.70264698897935, Tokens per sec: 175208.947778218, Loss: 2.491523504257202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20283 , TFLOPS: 67.99503382161814, Tokens per sec: 168499.18408128383, Loss: 2.4997172355651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20284 , TFLOPS: 70.77938575635069, Tokens per sec: 175399.11489723728, Loss: 2.4968111515045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20285 , TFLOPS: 69.71985331120105, Tokens per sec: 172773.47678102236, Loss: 2.4870636463165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20286 , TFLOPS: 69.6046577104518, Tokens per sec: 172488.00939252274, Loss: 2.47489333152771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20287 , TFLOPS: 70.73283534124279, Tokens per sec: 175283.75784064832, Loss: 2.4946844577789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20288 , TFLOPS: 69.81305871677166, Tokens per sec: 173004.45004344938, Loss: 2.4938008785247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20289 , TFLOPS: 70.73668799761839, Tokens per sec: 175293.30514755222, Loss: 2.485809087753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20290 , TFLOPS: 67.43457405524683, Tokens per sec: 167110.3030404757, Loss: 2.4747111797332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20291 , TFLOPS: 69.7906994869884, Tokens per sec: 172949.04140897395, Loss: 2.4968814849853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20292 , TFLOPS: 70.56859101620388, Tokens per sec: 174876.7423101959, Loss: 2.5054893493652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20293 , TFLOPS: 68.66628887598756, Tokens per sec: 170162.6280508598, Loss: 2.4892737865448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20294 , TFLOPS: 68.77338554643472, Tokens per sec: 170428.02539790006, Loss: 2.4697799682617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20295 , TFLOPS: 71.51249060882773, Tokens per sec: 177215.82947984163, Loss: 2.4931867122650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20296 , TFLOPS: 68.9621475244432, Tokens per sec: 170895.7983732535, Loss: 2.486764907836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20297 , TFLOPS: 69.10172529094176, Tokens per sec: 171241.6874543972, Loss: 2.4770455360412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20298 , TFLOPS: 69.6385682079649, Tokens per sec: 172572.04334090988, Loss: 2.4883458614349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20299 , TFLOPS: 70.61257924572907, Tokens per sec: 174985.7499886628, Loss: 2.4921882152557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20300 , TFLOPS: 69.67206408384492, Tokens per sec: 172655.04981121587, Loss: 2.485501766204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20301 , TFLOPS: 69.49802299877317, Tokens per sec: 172223.75683019985, Loss: 2.493316888809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20302 , TFLOPS: 70.6660890608872, Tokens per sec: 175118.35320521743, Loss: 2.501804828643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20303 , TFLOPS: 69.04431469742791, Tokens per sec: 171099.41767937093, Loss: 2.5102124214172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20304 , TFLOPS: 70.07566904424124, Tokens per sec: 173655.22736383468, Loss: 2.5032122135162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20305 , TFLOPS: 68.57441510050812, Tokens per sec: 169934.9547144899, Loss: 2.503544807434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20306 , TFLOPS: 71.53924127675292, Tokens per sec: 177282.12058178958, Loss: 2.484091281890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20307 , TFLOPS: 69.90550988389307, Tokens per sec: 173233.55421991303, Loss: 2.509359359741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20308 , TFLOPS: 69.81215104884546, Tokens per sec: 173002.20074234036, Loss: 2.4720165729522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20309 , TFLOPS: 69.69835527297911, Tokens per sec: 172720.20227410397, Loss: 2.47743558883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20310 , TFLOPS: 68.13069064338904, Tokens per sec: 168835.3566294647, Loss: 2.4904544353485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20311 , TFLOPS: 69.85128459264021, Tokens per sec: 173099.17797477945, Loss: 2.4971067905426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20312 , TFLOPS: 69.53548256832238, Tokens per sec: 172316.58577005504, Loss: 2.493757963180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20313 , TFLOPS: 68.72496385722376, Tokens per sec: 170308.03111794585, Loss: 2.481363296508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20314 , TFLOPS: 70.6227379042661, Tokens per sec: 175010.9242635862, Loss: 2.4568891525268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20315 , TFLOPS: 70.89789377209091, Tokens per sec: 175692.79081497714, Loss: 2.491209030151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20316 , TFLOPS: 68.12818840518989, Tokens per sec: 168829.1558075629, Loss: 2.470546007156372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20317 , TFLOPS: 69.69953183622387, Tokens per sec: 172723.11792743366, Loss: 2.509624481201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20318 , TFLOPS: 70.05880567338492, Tokens per sec: 173613.43807319907, Loss: 2.503192901611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20319 , TFLOPS: 70.65630021629732, Tokens per sec: 175094.09536999345, Loss: 2.4764509201049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20320 , TFLOPS: 69.3401845078432, Tokens per sec: 171832.61566233027, Loss: 2.4941389560699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20321 , TFLOPS: 69.8175392960449, Tokens per sec: 173015.55341246497, Loss: 2.4779748916625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20322 , TFLOPS: 70.62705579081259, Tokens per sec: 175021.6244621029, Loss: 2.4899020195007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20323 , TFLOPS: 68.95029571811217, Tokens per sec: 170866.42829158096, Loss: 2.505286455154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20324 , TFLOPS: 69.28838151232556, Tokens per sec: 171704.2421328665, Loss: 2.5072410106658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20325 , TFLOPS: 70.61278132447292, Tokens per sec: 174986.25076205088, Loss: 2.4774668216705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20326 , TFLOPS: 70.91130839557036, Tokens per sec: 175726.033729702, Loss: 2.4846670627593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20327 , TFLOPS: 68.65655034477918, Tokens per sec: 170138.49489773766, Loss: 2.4754014015197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20328 , TFLOPS: 69.83192196962148, Tokens per sec: 173051.19526197005, Loss: 2.4922142028808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20329 , TFLOPS: 70.65173984942484, Tokens per sec: 175082.79427851757, Loss: 2.497602939605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20330 , TFLOPS: 68.16457695168049, Tokens per sec: 168919.33063429588, Loss: 2.4803011417388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20331 , TFLOPS: 69.7194212923896, Tokens per sec: 172772.40619081678, Loss: 2.484473466873169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20332 , TFLOPS: 69.90747247561572, Tokens per sec: 173238.41773839906, Loss: 2.487729549407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20333 , TFLOPS: 69.59130795935762, Tokens per sec: 172454.92723871485, Loss: 2.496579170227051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20334 , TFLOPS: 70.77925485393335, Tokens per sec: 175398.790506627, Loss: 2.46079683303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20335 , TFLOPS: 69.02939103670928, Tokens per sec: 171062.43520413322, Loss: 2.4997615814208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20336 , TFLOPS: 69.93187441643072, Tokens per sec: 173298.8884358343, Loss: 2.4818549156188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20337 , TFLOPS: 69.28507844473923, Tokens per sec: 171696.05676752463, Loss: 2.492277145385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20338 , TFLOPS: 68.95324730915794, Tokens per sec: 170873.74265933697, Loss: 2.4915339946746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20339 , TFLOPS: 70.5806718258841, Tokens per sec: 174906.6798873945, Loss: 2.4870686531066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20340 , TFLOPS: 69.58577106038722, Tokens per sec: 172441.20619312642, Loss: 2.4909496307373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20341 , TFLOPS: 69.83847748548331, Tokens per sec: 173067.44055242534, Loss: 2.484715461730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20342 , TFLOPS: 70.46908370113067, Tokens per sec: 174630.15222180882, Loss: 2.5005767345428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20343 , TFLOPS: 69.34442263152991, Tokens per sec: 171843.1181996938, Loss: 2.5027358531951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20344 , TFLOPS: 69.00917176549947, Tokens per sec: 171012.32962273873, Loss: 2.489957809448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20345 , TFLOPS: 70.60198024795372, Tokens per sec: 174959.4845045995, Loss: 2.505333423614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20346 , TFLOPS: 69.67686434464412, Tokens per sec: 172666.94538626866, Loss: 2.491584300994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20347 , TFLOPS: 69.56132639957949, Tokens per sec: 172380.6296308432, Loss: 2.5028233528137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20348 , TFLOPS: 69.90570955121854, Tokens per sec: 173234.04901754067, Loss: 2.5133302211761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20349 , TFLOPS: 70.61935689229571, Tokens per sec: 175002.54574347436, Loss: 2.473703622817993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20350 , TFLOPS: 67.28611710311036, Tokens per sec: 166742.41035919817, Loss: 2.5014119148254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20351 , TFLOPS: 70.56355584718041, Tokens per sec: 174864.26460668608, Loss: 2.489255428314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20352 , TFLOPS: 69.80283139929324, Tokens per sec: 172979.10562410578, Loss: 2.472975969314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20353 , TFLOPS: 69.50411376234663, Tokens per sec: 172238.85041328787, Loss: 2.4770686626434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20354 , TFLOPS: 69.899686806922, Tokens per sec: 173219.1239937138, Loss: 2.4834353923797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20355 , TFLOPS: 69.92453166820584, Tokens per sec: 173280.69229686257, Loss: 2.474276065826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20356 , TFLOPS: 70.54057093814743, Tokens per sec: 174807.30547010558, Loss: 2.48879337310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20357 , TFLOPS: 67.49968429558332, Tokens per sec: 167271.65338851462, Loss: 2.509556293487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20358 , TFLOPS: 69.87551408240141, Tokens per sec: 173159.22131950635, Loss: 2.4892451763153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20359 , TFLOPS: 70.71445261236276, Tokens per sec: 175238.2034134589, Loss: 2.494474411010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20360 , TFLOPS: 68.58655258056291, Tokens per sec: 169965.03272711948, Loss: 2.4967336654663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20361 , TFLOPS: 69.79427023272072, Tokens per sec: 172957.89011024876, Loss: 2.479306936264038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20362 , TFLOPS: 71.54471849390907, Tokens per sec: 177295.6937292678, Loss: 2.4882593154907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20363 , TFLOPS: 68.84049384295488, Tokens per sec: 170594.32714926614, Loss: 2.507917881011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20364 , TFLOPS: 68.09293893829408, Tokens per sec: 168741.80374554574, Loss: 2.4754176139831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20365 , TFLOPS: 70.7424865460009, Tokens per sec: 175307.6745892069, Loss: 2.4848010540008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20366 , TFLOPS: 70.67177340229823, Tokens per sec: 175132.43962941587, Loss: 2.49546217918396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20367 , TFLOPS: 69.60396968007065, Tokens per sec: 172486.3043774454, Loss: 2.485323429107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20368 , TFLOPS: 68.40630904357678, Tokens per sec: 169518.36938700313, Loss: 2.4758737087249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20369 , TFLOPS: 70.60462171684388, Tokens per sec: 174966.03035549115, Loss: 2.485525608062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20370 , TFLOPS: 68.86608576665658, Tokens per sec: 170657.74675538208, Loss: 2.4879915714263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20371 , TFLOPS: 70.07927504911176, Tokens per sec: 173664.16344113805, Loss: 2.4837708473205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20372 , TFLOPS: 69.693040377245, Tokens per sec: 172707.03137699075, Loss: 2.495537757873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20373 , TFLOPS: 70.99371263669155, Tokens per sec: 175930.24051677686, Loss: 2.4896621704101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20374 , TFLOPS: 69.09987967475402, Tokens per sec: 171237.11381417257, Loss: 2.4734718799591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20375 , TFLOPS: 68.90184732159693, Tokens per sec: 170746.36782798474, Loss: 2.489025115966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20376 , TFLOPS: 70.52735692397498, Tokens per sec: 174774.55968734244, Loss: 2.4865763187408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20377 , TFLOPS: 68.04178513020076, Tokens per sec: 168615.03897403335, Loss: 2.491894245147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20378 , TFLOPS: 69.72338213018755, Tokens per sec: 172782.2215832024, Loss: 2.5099711418151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20379 , TFLOPS: 70.39113159168458, Tokens per sec: 174436.97830746768, Loss: 2.473257064819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20380 , TFLOPS: 69.64989718104361, Tokens per sec: 172600.11778418833, Loss: 2.4692139625549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20381 , TFLOPS: 70.66000696704378, Tokens per sec: 175103.28110667548, Loss: 2.4814820289611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20382 , TFLOPS: 70.99686473510475, Tokens per sec: 175938.05176387704, Loss: 2.4828238487243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20383 , TFLOPS: 67.94533396254691, Tokens per sec: 168376.02235561385, Loss: 2.508378505706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20384 , TFLOPS: 69.67462893548995, Tokens per sec: 172661.405796134, Loss: 2.490645170211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20385 , TFLOPS: 69.87291811581251, Tokens per sec: 173152.7882283297, Loss: 2.4998397827148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20386 , TFLOPS: 69.6848255319587, Tokens per sec: 172686.67408543135, Loss: 2.506100654602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20387 , TFLOPS: 69.6770264592306, Tokens per sec: 172667.34712407237, Loss: 2.496011734008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20388 , TFLOPS: 69.89343712859996, Tokens per sec: 173203.6366023729, Loss: 2.4950411319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20389 , TFLOPS: 70.53891280456824, Tokens per sec: 174803.1964324379, Loss: 2.496326208114624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20390 , TFLOPS: 68.76889444310626, Tokens per sec: 170416.89594911624, Loss: 2.4956393241882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20391 , TFLOPS: 69.20685201955405, Tokens per sec: 171502.20306856366, Loss: 2.4966325759887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20392 , TFLOPS: 70.53413502164932, Tokens per sec: 174791.35656004853, Loss: 2.4905571937561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20393 , TFLOPS: 70.71217121193364, Tokens per sec: 175232.5498518785, Loss: 2.5079684257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20394 , TFLOPS: 67.760113334815, Tokens per sec: 167917.0252363577, Loss: 2.489902973175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20395 , TFLOPS: 69.90921702649344, Tokens per sec: 173242.7409276522, Loss: 2.5021400451660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20396 , TFLOPS: 70.5621337632996, Tokens per sec: 174860.74052617396, Loss: 2.490880250930786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20397 , TFLOPS: 67.7876284885238, Tokens per sec: 167985.2108182925, Loss: 2.4960551261901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20398 , TFLOPS: 70.61192246134533, Tokens per sec: 174984.12240460917, Loss: 2.4726076126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20399 , TFLOPS: 70.07524433748063, Tokens per sec: 173654.17489369656, Loss: 2.4694643020629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20400 , TFLOPS: 69.6772301699168, Tokens per sec: 172667.85194159337, Loss: 2.5030672550201416 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/model.pt +[rank0]:[2024-08-27 21:06:55,714] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006451747001847252, 'preprocessing_with_comm': 0.0012879339919891208, 'state_converting': 1.0148146699939389, : 1.0239566560194362}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400, took 4.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018400 + eval ppl=10.243027687072754, eval loss=2.326597213745117 +------------------------------------------------------------------ +iteration: 20401 , TFLOPS: 64.95039244837383, Tokens per sec: 160954.22736344955, Loss: 2.4644312858581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20402 , TFLOPS: 70.94318539140126, Tokens per sec: 175805.02843691193, Loss: 2.4866983890533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20403 , TFLOPS: 69.63055611020707, Tokens per sec: 172552.18848006055, Loss: 2.497763156890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20404 , TFLOPS: 70.11081961785852, Tokens per sec: 173742.33435741367, Loss: 2.493940591812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20405 , TFLOPS: 69.7742923706716, Tokens per sec: 172908.38276734785, Loss: 2.4841294288635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20406 , TFLOPS: 70.69538718942287, Tokens per sec: 175190.95719518414, Loss: 2.4867348670959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20407 , TFLOPS: 68.92845475583802, Tokens per sec: 170812.303981085, Loss: 2.479219675064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20408 , TFLOPS: 68.71039889322266, Tokens per sec: 170271.9375326879, Loss: 2.4614901542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20409 , TFLOPS: 70.60672915672922, Tokens per sec: 174971.25282368067, Loss: 2.500892162322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20410 , TFLOPS: 69.01900047329232, Tokens per sec: 171036.68624337096, Loss: 2.4947407245635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20411 , TFLOPS: 69.82416710707099, Tokens per sec: 173031.9778583013, Loss: 2.4878180027008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20412 , TFLOPS: 69.51383253002489, Tokens per sec: 172262.93459020526, Loss: 2.4578170776367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20413 , TFLOPS: 69.63217270178002, Tokens per sec: 172556.19457206057, Loss: 2.494675874710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20414 , TFLOPS: 70.54559402916313, Tokens per sec: 174819.75324298168, Loss: 2.471473455429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20415 , TFLOPS: 69.74335622984965, Tokens per sec: 172831.71960249383, Loss: 2.4756274223327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20416 , TFLOPS: 69.63497100691644, Tokens per sec: 172563.12908044737, Loss: 2.4874842166900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20417 , TFLOPS: 69.15113141443611, Tokens per sec: 171364.12127095, Loss: 2.4975974559783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20418 , TFLOPS: 69.90110094771055, Tokens per sec: 173222.6283903683, Loss: 2.489027738571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20419 , TFLOPS: 70.53975727050833, Tokens per sec: 174805.289112063, Loss: 2.4822745323181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20420 , TFLOPS: 68.58013473612917, Tokens per sec: 169949.12860162873, Loss: 2.4948513507843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20421 , TFLOPS: 69.79942062171818, Tokens per sec: 172970.65334154663, Loss: 2.486389398574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20422 , TFLOPS: 71.49740758976276, Tokens per sec: 177178.4520970669, Loss: 2.503835916519165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20423 , TFLOPS: 69.50960651799507, Tokens per sec: 172252.4620668617, Loss: 2.4948532581329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20424 , TFLOPS: 69.06432530789698, Tokens per sec: 171149.006176464, Loss: 2.490097999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20425 , TFLOPS: 70.73798569546865, Tokens per sec: 175296.52098577836, Loss: 2.487558364868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20426 , TFLOPS: 70.5962571231659, Tokens per sec: 174945.30197092105, Loss: 2.4785804748535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20427 , TFLOPS: 68.65569612885967, Tokens per sec: 170136.37805658948, Loss: 2.4907171726226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20428 , TFLOPS: 69.74721241175298, Tokens per sec: 172841.27564604377, Loss: 2.479503631591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20429 , TFLOPS: 71.41436274220041, Tokens per sec: 176972.65781666225, Loss: 2.4924018383026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20430 , TFLOPS: 69.15201652856116, Tokens per sec: 171366.31468125526, Loss: 2.4858858585357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20431 , TFLOPS: 68.82919340289155, Tokens per sec: 170566.32341394294, Loss: 2.478295087814331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20432 , TFLOPS: 70.56252257082916, Tokens per sec: 174861.7040340629, Loss: 2.4833755493164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20433 , TFLOPS: 70.59143115341864, Tokens per sec: 174933.34268626288, Loss: 2.472837209701538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20434 , TFLOPS: 69.89464472110986, Tokens per sec: 173206.6291496397, Loss: 2.4902901649475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20435 , TFLOPS: 68.83438089763108, Tokens per sec: 170579.1785973572, Loss: 2.45904541015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20436 , TFLOPS: 70.72339046478076, Tokens per sec: 175260.35239633638, Loss: 2.4895687103271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20437 , TFLOPS: 68.96362617646412, Tokens per sec: 170899.4626358483, Loss: 2.4747257232666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20438 , TFLOPS: 68.90121937537727, Tokens per sec: 170744.8117080784, Loss: 2.502920389175415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20439 , TFLOPS: 70.46984588550373, Tokens per sec: 174632.0409986468, Loss: 2.4780325889587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20440 , TFLOPS: 70.61710027296547, Tokens per sec: 174996.95359218688, Loss: 2.5069758892059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20441 , TFLOPS: 69.8807815266584, Tokens per sec: 173172.27462663202, Loss: 2.477714776992798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20442 , TFLOPS: 69.80418217867512, Tokens per sec: 172982.45300421474, Loss: 2.4945411682128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20443 , TFLOPS: 69.80500159728649, Tokens per sec: 172984.4836138001, Loss: 2.482489585876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20444 , TFLOPS: 69.8415365845991, Tokens per sec: 173075.02133702184, Loss: 2.490201950073242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20445 , TFLOPS: 69.08954385937771, Tokens per sec: 171211.50052508604, Loss: 2.4890904426574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20446 , TFLOPS: 70.54009291766775, Tokens per sec: 174806.12088269865, Loss: 2.502070903778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20447 , TFLOPS: 68.81296086475025, Tokens per sec: 170526.0973962671, Loss: 2.4855966567993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20448 , TFLOPS: 69.8044914151987, Tokens per sec: 172983.21932638536, Loss: 2.4750685691833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20449 , TFLOPS: 70.46945604467973, Tokens per sec: 174631.07493014043, Loss: 2.4726967811584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20450 , TFLOPS: 68.8867565465269, Tokens per sec: 170708.97122497507, Loss: 2.4940690994262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20451 , TFLOPS: 69.88275565316096, Tokens per sec: 173177.1667295734, Loss: 2.4898273944854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20452 , TFLOPS: 70.59686386482636, Tokens per sec: 174946.80554359377, Loss: 2.476123809814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20453 , TFLOPS: 68.94811055542266, Tokens per sec: 170861.0132177208, Loss: 2.498910427093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20454 , TFLOPS: 69.56398390252315, Tokens per sec: 172387.21521588572, Loss: 2.50990891456604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20455 , TFLOPS: 69.76030299902573, Tokens per sec: 172873.7155633529, Loss: 2.5077006816864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20456 , TFLOPS: 70.64418162836117, Tokens per sec: 175064.064174115, Loss: 2.5155863761901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20457 , TFLOPS: 68.81501191571633, Tokens per sec: 170531.180126503, Loss: 2.5119998455047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20458 , TFLOPS: 70.04693248362763, Tokens per sec: 173584.01500104068, Loss: 2.486473798751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20459 , TFLOPS: 69.94002544416976, Tokens per sec: 173319.0875804815, Loss: 2.488513946533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20460 , TFLOPS: 69.56574238414308, Tokens per sec: 172391.57292705262, Loss: 2.4715373516082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20461 , TFLOPS: 70.57386350176068, Tokens per sec: 174889.8081102179, Loss: 2.4719595909118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20462 , TFLOPS: 70.03370486623747, Tokens per sec: 173551.23550800554, Loss: 2.4938037395477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20463 , TFLOPS: 69.90724475526766, Tokens per sec: 173237.85342229105, Loss: 2.4883759021759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20464 , TFLOPS: 68.70727586076194, Tokens per sec: 170264.19831421983, Loss: 2.4997031688690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20465 , TFLOPS: 69.94561390229342, Tokens per sec: 173332.9363953314, Loss: 2.483790874481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20466 , TFLOPS: 70.58780358961533, Tokens per sec: 174924.35318354916, Loss: 2.4811975955963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20467 , TFLOPS: 68.91870676755083, Tokens per sec: 170788.1473922798, Loss: 2.4824295043945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20468 , TFLOPS: 68.7924119374918, Tokens per sec: 170475.1748908695, Loss: 2.4797205924987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20469 , TFLOPS: 71.53812664383675, Tokens per sec: 177279.35839863634, Loss: 2.473567247390747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20470 , TFLOPS: 68.9704795723861, Tokens per sec: 170916.44610590683, Loss: 2.489187717437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20471 , TFLOPS: 69.65530838450165, Tokens per sec: 172613.5273424506, Loss: 2.502232074737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20472 , TFLOPS: 69.7068763580458, Tokens per sec: 172741.31846157426, Loss: 2.4943292140960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20473 , TFLOPS: 70.6613541661822, Tokens per sec: 175106.61961454674, Loss: 2.505112409591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20474 , TFLOPS: 69.55950309879343, Tokens per sec: 172376.11129064215, Loss: 2.4922258853912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20475 , TFLOPS: 69.5467349210178, Tokens per sec: 172344.47033958326, Loss: 2.4900550842285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20476 , TFLOPS: 70.6022623977647, Tokens per sec: 174960.1837029125, Loss: 2.4749348163604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20477 , TFLOPS: 69.83563148097761, Tokens per sec: 173060.38783974695, Loss: 2.4876983165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20478 , TFLOPS: 69.73559343135786, Tokens per sec: 172812.48253268827, Loss: 2.5216803550720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20479 , TFLOPS: 69.78953546448244, Tokens per sec: 172946.1568329765, Loss: 2.4884276390075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20480 , TFLOPS: 70.0374991362206, Tokens per sec: 173560.63812699707, Loss: 2.4925332069396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20481 , TFLOPS: 70.54733536046629, Tokens per sec: 174824.06845377438, Loss: 2.478081703186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20482 , TFLOPS: 68.79109247864768, Tokens per sec: 170471.90512650338, Loss: 2.487229347229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20483 , TFLOPS: 70.42933876357623, Tokens per sec: 174531.65988828262, Loss: 2.4906468391418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20484 , TFLOPS: 69.18025089529094, Tokens per sec: 171436.28255228393, Loss: 2.4848504066467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20485 , TFLOPS: 69.86682384915554, Tokens per sec: 173137.68596421485, Loss: 2.4938509464263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20486 , TFLOPS: 69.71688572308547, Tokens per sec: 172766.12277076513, Loss: 2.497546672821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20487 , TFLOPS: 69.75517894911923, Tokens per sec: 172861.01760322385, Loss: 2.4947690963745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20488 , TFLOPS: 70.1532726277429, Tokens per sec: 173847.53759249224, Loss: 2.4807279109954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20489 , TFLOPS: 70.83140835571264, Tokens per sec: 175528.0326292186, Loss: 2.4886202812194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20490 , TFLOPS: 69.60784054489449, Tokens per sec: 172495.89680688985, Loss: 2.4927444458007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20491 , TFLOPS: 70.1169334507624, Tokens per sec: 173757.48510884374, Loss: 2.497138500213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20492 , TFLOPS: 69.98489046739799, Tokens per sec: 173430.26805033136, Loss: 2.4828133583068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20493 , TFLOPS: 70.6649886901349, Tokens per sec: 175115.6263652774, Loss: 2.4852583408355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20494 , TFLOPS: 68.58752164402833, Tokens per sec: 169967.4341731379, Loss: 2.4783103466033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20495 , TFLOPS: 69.83583741879613, Tokens per sec: 173060.8981763472, Loss: 2.4833436012268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20496 , TFLOPS: 71.56420586983734, Tokens per sec: 177343.9855935297, Loss: 2.498225212097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20497 , TFLOPS: 68.97325363747316, Tokens per sec: 170923.3205447811, Loss: 2.473867416381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20498 , TFLOPS: 69.83095677048283, Tokens per sec: 173048.80339217902, Loss: 2.504082202911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20499 , TFLOPS: 70.68437249418899, Tokens per sec: 175163.6615670257, Loss: 2.4890246391296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20500 , TFLOPS: 70.73938143621811, Tokens per sec: 175299.97978510908, Loss: 2.503854990005493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20501 , TFLOPS: 68.80595431336143, Tokens per sec: 170508.73439009025, Loss: 2.469341278076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20502 , TFLOPS: 69.9565103554051, Tokens per sec: 173359.93900648554, Loss: 2.500547170639038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20503 , TFLOPS: 70.56464607307272, Tokens per sec: 174866.96630654554, Loss: 2.5061471462249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20504 , TFLOPS: 68.94189581918809, Tokens per sec: 170845.61241672188, Loss: 2.4783504009246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20505 , TFLOPS: 68.5738610544463, Tokens per sec: 169933.5817273155, Loss: 2.4727327823638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20506 , TFLOPS: 70.70380554440194, Tokens per sec: 175211.8188061806, Loss: 2.4791219234466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20507 , TFLOPS: 70.54447957639209, Tokens per sec: 174816.99150624793, Loss: 2.491292953491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20508 , TFLOPS: 69.14890187550493, Tokens per sec: 171358.59622787457, Loss: 2.5013179779052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20509 , TFLOPS: 69.61524947402268, Tokens per sec: 172514.25694943042, Loss: 2.493993043899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20510 , TFLOPS: 69.77975880775219, Tokens per sec: 172921.9292006113, Loss: 2.475421190261841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20511 , TFLOPS: 69.81363641130775, Tokens per sec: 173005.8816341481, Loss: 2.4945461750030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20512 , TFLOPS: 68.63199731679407, Tokens per sec: 170077.64978965124, Loss: 2.498317003250122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20513 , TFLOPS: 70.76252295115225, Tokens per sec: 175357.32700836417, Loss: 2.47999906539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20514 , TFLOPS: 69.49356375041891, Tokens per sec: 172212.70632155126, Loss: 2.487461566925049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20515 , TFLOPS: 69.91883569990301, Tokens per sec: 173266.57705994547, Loss: 2.4877092838287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20516 , TFLOPS: 70.38327387672807, Tokens per sec: 174417.50602420667, Loss: 2.4898481369018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20517 , TFLOPS: 68.97059650217285, Tokens per sec: 170916.7358707993, Loss: 2.4862468242645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20518 , TFLOPS: 69.92110123992184, Tokens per sec: 173272.19131768198, Loss: 2.486410140991211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20519 , TFLOPS: 69.56161243047833, Tokens per sec: 172381.3384469194, Loss: 2.47695255279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20520 , TFLOPS: 70.70799784769768, Tokens per sec: 175222.20779556775, Loss: 2.4918484687805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20521 , TFLOPS: 69.24995427238699, Tokens per sec: 171609.01519918878, Loss: 2.484576463699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20522 , TFLOPS: 69.77501535987562, Tokens per sec: 172910.17441423927, Loss: 2.4771854877471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20523 , TFLOPS: 70.50236811103402, Tokens per sec: 174712.63465612973, Loss: 2.4768879413604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20524 , TFLOPS: 69.73964943149483, Tokens per sec: 172822.53374783302, Loss: 2.47759747505188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20525 , TFLOPS: 70.20481190438663, Tokens per sec: 173975.2575975354, Loss: 2.497133255004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20526 , TFLOPS: 70.92272618497567, Tokens per sec: 175754.3282696217, Loss: 2.490673780441284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20527 , TFLOPS: 69.15154399085043, Tokens per sec: 171365.14368075374, Loss: 2.4569621086120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20528 , TFLOPS: 69.66027697065704, Tokens per sec: 172625.84004627858, Loss: 2.473339319229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20529 , TFLOPS: 70.01686470241401, Tokens per sec: 173509.50372695382, Loss: 2.509002685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20530 , TFLOPS: 70.59057499329703, Tokens per sec: 174931.22102716772, Loss: 2.493335247039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20531 , TFLOPS: 67.86903762934206, Tokens per sec: 168186.95163719734, Loss: 2.4865505695343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20532 , TFLOPS: 69.73638594333237, Tokens per sec: 172814.44646466355, Loss: 2.4831485748291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20533 , TFLOPS: 70.73164296824993, Tokens per sec: 175280.8030090213, Loss: 2.496122360229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20534 , TFLOPS: 67.94870897693764, Tokens per sec: 168384.38601306264, Loss: 2.462430000305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20535 , TFLOPS: 69.7378384772787, Tokens per sec: 172818.04600379235, Loss: 2.5049636363983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20536 , TFLOPS: 71.5296459516576, Tokens per sec: 177258.34230919104, Loss: 2.4910776615142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20537 , TFLOPS: 68.87778358209907, Tokens per sec: 170686.73523066932, Loss: 2.4746971130371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20538 , TFLOPS: 68.95332141544381, Tokens per sec: 170873.92630287685, Loss: 2.490644931793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20539 , TFLOPS: 70.69638690203647, Tokens per sec: 175193.43459316803, Loss: 2.4628326892852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20540 , TFLOPS: 70.65707612141027, Tokens per sec: 175096.01814833636, Loss: 2.4921109676361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20541 , TFLOPS: 69.58414210840836, Tokens per sec: 172437.16947067893, Loss: 2.479602336883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20542 , TFLOPS: 68.72101572671171, Tokens per sec: 170298.24721561454, Loss: 2.50018310546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20543 , TFLOPS: 70.56878276961453, Tokens per sec: 174877.2174962709, Loss: 2.484701633453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20544 , TFLOPS: 69.6907538884382, Tokens per sec: 172701.36520584993, Loss: 2.4925575256347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20545 , TFLOPS: 69.952228362444, Tokens per sec: 173349.32775622746, Loss: 2.512233257293701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20546 , TFLOPS: 70.34383759246424, Tokens per sec: 174319.77856753557, Loss: 2.480557441711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20547 , TFLOPS: 69.8580802683141, Tokens per sec: 173116.01840770483, Loss: 2.4662153720855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20548 , TFLOPS: 69.67212691666691, Tokens per sec: 172655.20551787037, Loss: 2.524038791656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20549 , TFLOPS: 68.29579912596634, Tokens per sec: 169244.51363749205, Loss: 2.484978675842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20550 , TFLOPS: 70.53798694419564, Tokens per sec: 174800.90204844263, Loss: 2.4902920722961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20551 , TFLOPS: 68.98497285400536, Tokens per sec: 170952.36205432616, Loss: 2.49912691116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20552 , TFLOPS: 69.73037918700052, Tokens per sec: 172799.56106077437, Loss: 2.4970240592956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20553 , TFLOPS: 69.61734904298264, Tokens per sec: 172519.45991259982, Loss: 2.498819351196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20554 , TFLOPS: 69.46620136399648, Tokens per sec: 172144.89931378077, Loss: 2.4921982288360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20555 , TFLOPS: 69.37489699096744, Tokens per sec: 171918.63701940782, Loss: 2.4462838172912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20556 , TFLOPS: 70.34626215762026, Tokens per sec: 174325.78690708007, Loss: 2.4815104007720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20557 , TFLOPS: 69.74110198214812, Tokens per sec: 172826.13332836353, Loss: 2.4747703075408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20558 , TFLOPS: 69.21460162722356, Tokens per sec: 171521.40745005844, Loss: 2.4909515380859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20559 , TFLOPS: 70.65689730656813, Tokens per sec: 175095.57502545943, Loss: 2.4883315563201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20560 , TFLOPS: 69.72146921328894, Tokens per sec: 172777.48116440244, Loss: 2.498286724090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20561 , TFLOPS: 69.64359863500972, Tokens per sec: 172584.5092932742, Loss: 2.5086488723754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20562 , TFLOPS: 69.82032764961994, Tokens per sec: 173022.46325978645, Loss: 2.503253936767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20563 , TFLOPS: 71.40871252873362, Tokens per sec: 176958.65596526916, Loss: 2.4889025688171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20564 , TFLOPS: 67.93790071847363, Tokens per sec: 168357.60195796084, Loss: 2.496180772781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20565 , TFLOPS: 69.89581309204317, Tokens per sec: 173209.52450151916, Loss: 2.4936130046844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20566 , TFLOPS: 70.58000489948515, Tokens per sec: 174905.02717030988, Loss: 2.473935127258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20567 , TFLOPS: 70.70440016130676, Tokens per sec: 175213.29233237318, Loss: 2.4689908027648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20568 , TFLOPS: 68.22616232005893, Tokens per sec: 169071.94596132607, Loss: 2.4822678565979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20569 , TFLOPS: 69.86006457080028, Tokens per sec: 173120.93572785455, Loss: 2.4758918285369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20570 , TFLOPS: 70.63437457634828, Tokens per sec: 175039.76121889107, Loss: 2.5126211643218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20571 , TFLOPS: 67.77298799288003, Tokens per sec: 167948.9300573035, Loss: 2.4763834476470947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20572 , TFLOPS: 69.72501791954316, Tokens per sec: 172786.2752494227, Loss: 2.4850733280181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20573 , TFLOPS: 70.70373047911531, Tokens per sec: 175211.63278613117, Loss: 2.469205141067505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20574 , TFLOPS: 70.64860865815736, Tokens per sec: 175075.03484162755, Loss: 2.4717118740081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20575 , TFLOPS: 68.88493126887285, Tokens per sec: 170704.44798587743, Loss: 2.491260051727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20576 , TFLOPS: 70.90093086293913, Tokens per sec: 175700.31706066223, Loss: 2.4730825424194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20577 , TFLOPS: 69.70781189908118, Tokens per sec: 172743.63683531826, Loss: 2.489727735519409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20578 , TFLOPS: 69.72152474365085, Tokens per sec: 172777.61877475644, Loss: 2.506466865539551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20579 , TFLOPS: 69.047895733726, Tokens per sec: 171108.2918818013, Loss: 2.4912643432617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20580 , TFLOPS: 70.66432289510199, Tokens per sec: 175113.97645184243, Loss: 2.495556116104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20581 , TFLOPS: 69.70486732465096, Tokens per sec: 172736.33985550853, Loss: 2.4914908409118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20582 , TFLOPS: 69.92408737510746, Tokens per sec: 173279.59128962184, Loss: 2.485900402069092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20583 , TFLOPS: 70.38932810247127, Tokens per sec: 174432.50906252634, Loss: 2.5273070335388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20584 , TFLOPS: 68.75809851478209, Tokens per sec: 170390.14244945926, Loss: 2.4710164070129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20585 , TFLOPS: 69.81387105263425, Tokens per sec: 173006.46310120306, Loss: 2.4996917247772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20586 , TFLOPS: 68.87219138935046, Tokens per sec: 170672.87716100737, Loss: 2.4697823524475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20587 , TFLOPS: 70.64369893169685, Tokens per sec: 175062.86799860737, Loss: 2.4938998222351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20588 , TFLOPS: 68.84825859502607, Tokens per sec: 170613.5690602566, Loss: 2.4766204357147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20589 , TFLOPS: 69.86031566547335, Tokens per sec: 173121.55796811476, Loss: 2.495518445968628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20590 , TFLOPS: 69.62685874725757, Tokens per sec: 172543.02600737353, Loss: 2.4763894081115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20591 , TFLOPS: 69.60723255522652, Tokens per sec: 172494.39014151704, Loss: 2.51664662361145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20592 , TFLOPS: 70.03835579390535, Tokens per sec: 173562.76101910853, Loss: 2.4892356395721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20593 , TFLOPS: 69.91355662710336, Tokens per sec: 173253.49493601, Loss: 2.4821903705596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20594 , TFLOPS: 70.66288736268085, Tokens per sec: 175110.41904436593, Loss: 2.495615005493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20595 , TFLOPS: 70.46097998888936, Tokens per sec: 174610.07033017726, Loss: 2.4970176219940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20596 , TFLOPS: 70.07058412274495, Tokens per sec: 173642.62636821653, Loss: 2.4986159801483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20597 , TFLOPS: 68.7578710772012, Tokens per sec: 170389.5788340795, Loss: 2.482755661010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20598 , TFLOPS: 69.79606751102831, Tokens per sec: 172962.34396388152, Loss: 2.4501006603240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20599 , TFLOPS: 69.87642812971482, Tokens per sec: 173161.48642943962, Loss: 2.4852116107940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20600 , TFLOPS: 70.71947232935774, Tokens per sec: 175250.64282513992, Loss: 2.5025858879089355 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/model.pt +[rank0]:[2024-08-27 21:32:19,984] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0064131669932976365, 'preprocessing_with_comm': 0.0012419970007613301, 'state_converting': 1.025029349984834, : 1.0340400749992114}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600, took 4.52s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018600 + eval ppl=10.197172164916992, eval loss=2.322110414505005 +------------------------------------------------------------------ +iteration: 20601 , TFLOPS: 65.00882130414422, Tokens per sec: 161099.02050451838, Loss: 2.4852688312530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20602 , TFLOPS: 69.8742252600686, Tokens per sec: 173156.02747579015, Loss: 2.486452102661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20603 , TFLOPS: 70.84598902382393, Tokens per sec: 175564.16513099155, Loss: 2.4684555530548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20604 , TFLOPS: 68.989124550904, Tokens per sec: 170962.65041658818, Loss: 2.4835007190704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20605 , TFLOPS: 69.87591962386016, Tokens per sec: 173160.2262959152, Loss: 2.4922776222229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20606 , TFLOPS: 69.88450284493419, Tokens per sec: 173181.4964632565, Loss: 2.5106914043426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20607 , TFLOPS: 70.60027306627035, Tokens per sec: 174955.2539203265, Loss: 2.477886438369751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20608 , TFLOPS: 68.60654406524674, Tokens per sec: 170014.57382840983, Loss: 2.4848053455352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20609 , TFLOPS: 69.97610331910026, Tokens per sec: 173408.49252886526, Loss: 2.4974210262298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20610 , TFLOPS: 70.61030574112392, Tokens per sec: 174980.11599380404, Loss: 2.471498727798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20611 , TFLOPS: 69.65882784030711, Tokens per sec: 172622.2489416366, Loss: 2.486706256866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20612 , TFLOPS: 68.51633672665353, Tokens per sec: 169791.0301062172, Loss: 2.477713108062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20613 , TFLOPS: 71.45259441579547, Tokens per sec: 177067.40011539735, Loss: 2.4917948246002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20614 , TFLOPS: 69.67965046016954, Tokens per sec: 172673.84968745662, Loss: 2.50530743598938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20615 , TFLOPS: 69.00835024554883, Tokens per sec: 171010.29380580314, Loss: 2.4886977672576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20616 , TFLOPS: 69.40218602611326, Tokens per sec: 171986.26225463452, Loss: 2.4855847358703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20617 , TFLOPS: 70.64324534863296, Tokens per sec: 175061.7439698088, Loss: 2.4917447566986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20618 , TFLOPS: 69.71010139812913, Tokens per sec: 172749.31046616822, Loss: 2.473031997680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20619 , TFLOPS: 68.82493076137634, Tokens per sec: 170555.76011869955, Loss: 2.4939773082733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20620 , TFLOPS: 70.6009205953746, Tokens per sec: 174956.85856877812, Loss: 2.4969875812530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20621 , TFLOPS: 70.62953995486708, Tokens per sec: 175027.7804942832, Loss: 2.475268840789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20622 , TFLOPS: 69.32092007240051, Tokens per sec: 171784.8762691505, Loss: 2.492912769317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20623 , TFLOPS: 68.92064370599336, Tokens per sec: 170792.9473391118, Loss: 2.505190134048462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20624 , TFLOPS: 71.53517712180556, Tokens per sec: 177272.0491581264, Loss: 2.486825942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20625 , TFLOPS: 69.19397897123304, Tokens per sec: 171470.30223674086, Loss: 2.4915170669555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20626 , TFLOPS: 68.76380673225462, Tokens per sec: 170404.28804116827, Loss: 2.4866225719451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20627 , TFLOPS: 70.56935484707438, Tokens per sec: 174878.6351672359, Loss: 2.5082454681396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20628 , TFLOPS: 68.86768234724995, Tokens per sec: 170661.70325797034, Loss: 2.492157220840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20629 , TFLOPS: 69.96673186736591, Tokens per sec: 173385.26903911692, Loss: 2.5026023387908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20630 , TFLOPS: 69.55250407084336, Tokens per sec: 172358.76692837532, Loss: 2.4888646602630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20631 , TFLOPS: 69.70124777222661, Tokens per sec: 172727.37020587438, Loss: 2.4966304302215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20632 , TFLOPS: 69.59912912618469, Tokens per sec: 172474.30895168497, Loss: 2.5056259632110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20633 , TFLOPS: 71.4711138315282, Tokens per sec: 177113.29326766508, Loss: 2.4811344146728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20634 , TFLOPS: 68.40624581307627, Tokens per sec: 169518.21269485744, Loss: 2.5054876804351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20635 , TFLOPS: 69.80834744814999, Tokens per sec: 172992.77500081575, Loss: 2.491285800933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20636 , TFLOPS: 70.07319234955328, Tokens per sec: 173649.0898415675, Loss: 2.4883179664611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20637 , TFLOPS: 70.6322679077495, Tokens per sec: 175034.5406620346, Loss: 2.4932093620300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20638 , TFLOPS: 68.57122844765507, Tokens per sec: 169927.05783768126, Loss: 2.492126703262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20639 , TFLOPS: 68.98760613342459, Tokens per sec: 170958.88761080668, Loss: 2.4696173667907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20640 , TFLOPS: 70.58743937709157, Tokens per sec: 174923.45062479394, Loss: 2.4749653339385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20641 , TFLOPS: 69.64391158053733, Tokens per sec: 172585.28480676544, Loss: 2.4996724128723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20642 , TFLOPS: 69.02103826205227, Tokens per sec: 171041.73610839908, Loss: 2.5136702060699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20643 , TFLOPS: 70.57813495657844, Tokens per sec: 174900.39324579586, Loss: 2.499178647994995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20644 , TFLOPS: 70.93038627475457, Tokens per sec: 175773.31081592233, Loss: 2.4947352409362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20645 , TFLOPS: 67.29872378954941, Tokens per sec: 166773.65111693536, Loss: 2.490504026412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20646 , TFLOPS: 69.97905184636596, Tokens per sec: 173415.79930423677, Loss: 2.488772392272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20647 , TFLOPS: 70.62715950469412, Tokens per sec: 175021.88147652632, Loss: 2.4775524139404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20648 , TFLOPS: 69.86454694489683, Tokens per sec: 173132.04354464007, Loss: 2.4812142848968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20649 , TFLOPS: 68.01891176920537, Tokens per sec: 168558.35626577804, Loss: 2.4639008045196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20650 , TFLOPS: 70.69151468776916, Tokens per sec: 175181.3607094949, Loss: 2.463796377182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20651 , TFLOPS: 70.50614840655226, Tokens per sec: 174722.0026448579, Loss: 2.4800546169281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20652 , TFLOPS: 70.00573128880116, Tokens per sec: 173481.91390157441, Loss: 2.468682289123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20653 , TFLOPS: 70.36298508254102, Tokens per sec: 174367.22815721598, Loss: 2.474820613861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20654 , TFLOPS: 69.79605702131965, Tokens per sec: 172962.3179692279, Loss: 2.483293294906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20655 , TFLOPS: 69.78798227179047, Tokens per sec: 172942.3078503875, Loss: 2.4828741550445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20656 , TFLOPS: 69.6848288428134, Tokens per sec: 172686.68229009406, Loss: 2.4811196327209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20657 , TFLOPS: 70.69118735780069, Tokens per sec: 175180.5495497747, Loss: 2.4717724323272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20658 , TFLOPS: 69.71576497649066, Tokens per sec: 172763.34543724277, Loss: 2.5034379959106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20659 , TFLOPS: 69.02720640151877, Tokens per sec: 171057.0214374737, Loss: 2.4986987113952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20660 , TFLOPS: 70.42503297996834, Tokens per sec: 174520.989682181, Loss: 2.4997901916503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20661 , TFLOPS: 70.11279803520505, Tokens per sec: 173747.237093539, Loss: 2.501831531524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20662 , TFLOPS: 69.03798607283126, Tokens per sec: 171083.73465046487, Loss: 2.4556941986083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20663 , TFLOPS: 69.57577289075017, Tokens per sec: 172416.42962737652, Loss: 2.505479574203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20664 , TFLOPS: 70.67938333887226, Tokens per sec: 175151.29789055, Loss: 2.458310604095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20665 , TFLOPS: 68.76587623664959, Tokens per sec: 170409.41650103373, Loss: 2.472093105316162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20666 , TFLOPS: 69.8300397031436, Tokens per sec: 173046.53079828908, Loss: 2.4953184127807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20667 , TFLOPS: 69.43411634037922, Tokens per sec: 172065.38909080005, Loss: 2.484987258911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20668 , TFLOPS: 69.84326819922839, Tokens per sec: 173079.31246882633, Loss: 2.4491968154907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20669 , TFLOPS: 68.89879623741226, Tokens per sec: 170738.80690527047, Loss: 2.492506504058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20670 , TFLOPS: 70.62256958769768, Tokens per sec: 175010.50715658805, Loss: 2.478095531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20671 , TFLOPS: 69.60962010733694, Tokens per sec: 172500.30675865724, Loss: 2.4874215126037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20672 , TFLOPS: 69.7952948964932, Tokens per sec: 172960.429339953, Loss: 2.503756284713745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20673 , TFLOPS: 70.64688048957075, Tokens per sec: 175070.75224949644, Loss: 2.4945363998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20674 , TFLOPS: 70.70705482374001, Tokens per sec: 175219.870878318, Loss: 2.4970788955688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20675 , TFLOPS: 68.00651846388988, Tokens per sec: 168527.64428997098, Loss: 2.4567415714263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20676 , TFLOPS: 69.7436481237788, Tokens per sec: 172832.442947805, Loss: 2.509227991104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20677 , TFLOPS: 70.52802668911566, Tokens per sec: 174776.21943914177, Loss: 2.4982101917266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20678 , TFLOPS: 69.4951608425414, Tokens per sec: 172216.66409176498, Loss: 2.48384952545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20679 , TFLOPS: 69.02728955872423, Tokens per sec: 171057.22751018938, Loss: 2.4704463481903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20680 , TFLOPS: 71.52091219077327, Tokens per sec: 177236.69908761652, Loss: 2.4795453548431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20681 , TFLOPS: 69.8833737786843, Tokens per sec: 173178.6985127124, Loss: 2.485903263092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20682 , TFLOPS: 67.95638377728417, Tokens per sec: 168403.4050137704, Loss: 2.502293109893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20683 , TFLOPS: 70.5930376063217, Tokens per sec: 174937.32365352206, Loss: 2.4975028038024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20684 , TFLOPS: 70.57262771580832, Tokens per sec: 174886.74569649488, Loss: 2.4790267944335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20685 , TFLOPS: 69.87474881013333, Tokens per sec: 173157.32489052432, Loss: 2.4738144874572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20686 , TFLOPS: 68.38470014497778, Tokens per sec: 169464.82015585812, Loss: 2.490581512451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20687 , TFLOPS: 70.69618895085246, Tokens per sec: 175192.94404832792, Loss: 2.4646339416503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20688 , TFLOPS: 70.65824620700624, Tokens per sec: 175098.91774933873, Loss: 2.4950599670410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20689 , TFLOPS: 69.0758815209747, Tokens per sec: 171177.64374549256, Loss: 2.5109195709228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20690 , TFLOPS: 69.93417378015397, Tokens per sec: 173304.58651243636, Loss: 2.5025453567504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20691 , TFLOPS: 70.82731471000989, Tokens per sec: 175517.88812421483, Loss: 2.4871363639831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20692 , TFLOPS: 69.13087372115343, Tokens per sec: 171313.9204754264, Loss: 2.4887642860412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20693 , TFLOPS: 69.50652719656189, Tokens per sec: 172244.83116913502, Loss: 2.493976354598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20694 , TFLOPS: 70.70051607895849, Tokens per sec: 175203.66714844716, Loss: 2.4945075511932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20695 , TFLOPS: 68.82537651353934, Tokens per sec: 170556.86474166313, Loss: 2.499941110610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20696 , TFLOPS: 69.73907515655269, Tokens per sec: 172821.11063126478, Loss: 2.4772984981536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20697 , TFLOPS: 69.67226584931173, Tokens per sec: 172655.54980826893, Loss: 2.505429744720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20698 , TFLOPS: 69.80476684066353, Tokens per sec: 172983.90186102784, Loss: 2.4685254096984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20699 , TFLOPS: 68.97290512280657, Tokens per sec: 170922.4568870452, Loss: 2.4921975135803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20700 , TFLOPS: 70.44550552584953, Tokens per sec: 174571.72290611174, Loss: 2.4886088371276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20701 , TFLOPS: 69.8333783293346, Tokens per sec: 173054.80428177043, Loss: 2.498508930206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20702 , TFLOPS: 69.7152098955026, Tokens per sec: 172761.96988540713, Loss: 2.511054277420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20703 , TFLOPS: 70.10036161571276, Tokens per sec: 173716.4182760524, Loss: 2.4773919582366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20704 , TFLOPS: 69.60164282378312, Tokens per sec: 172480.53817124057, Loss: 2.4863083362579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20705 , TFLOPS: 69.89626183343809, Tokens per sec: 173210.63653212867, Loss: 2.4681785106658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20706 , TFLOPS: 68.83489365275193, Tokens per sec: 170580.44926103135, Loss: 2.5064890384674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20707 , TFLOPS: 70.69524003993872, Tokens per sec: 175190.59254255253, Loss: 2.4991366863250732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20708 , TFLOPS: 69.47060859711465, Tokens per sec: 172155.82092294437, Loss: 2.5042307376861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20709 , TFLOPS: 68.9894066688499, Tokens per sec: 170963.34953593594, Loss: 2.464120388031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20710 , TFLOPS: 70.60026274800066, Tokens per sec: 174955.2283505176, Loss: 2.475670337677002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20711 , TFLOPS: 70.51221815580178, Tokens per sec: 174737.04415213625, Loss: 2.5170071125030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20712 , TFLOPS: 68.01731033768664, Tokens per sec: 168554.38774206242, Loss: 2.4946846961975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20713 , TFLOPS: 70.62964932019482, Tokens per sec: 175028.05151361288, Loss: 2.4709925651550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20714 , TFLOPS: 70.73577549018732, Tokens per sec: 175291.043853617, Loss: 2.5124402046203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20715 , TFLOPS: 67.78823495564521, Tokens per sec: 167986.71371062723, Loss: 2.485111713409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20716 , TFLOPS: 69.94068475634137, Tokens per sec: 173320.72142867197, Loss: 2.509810447692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20717 , TFLOPS: 70.58404338486413, Tokens per sec: 174915.03498195493, Loss: 2.469115734100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20718 , TFLOPS: 69.86753215145478, Tokens per sec: 173139.44121533833, Loss: 2.4810190200805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20719 , TFLOPS: 68.69424867933574, Tokens per sec: 170231.91552358321, Loss: 2.492103338241577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20720 , TFLOPS: 70.89257732737885, Tokens per sec: 175679.6160793259, Loss: 2.4670932292938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20721 , TFLOPS: 69.75956443164209, Tokens per sec: 172871.8853120163, Loss: 2.495950222015381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20722 , TFLOPS: 69.78881747339501, Tokens per sec: 172944.37757196862, Loss: 2.478762149810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20723 , TFLOPS: 69.72645642591995, Tokens per sec: 172789.8400266889, Loss: 2.5145459175109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20724 , TFLOPS: 70.59187418807625, Tokens per sec: 174934.44057494873, Loss: 2.4695165157318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20725 , TFLOPS: 69.79022215453409, Tokens per sec: 172947.85852656967, Loss: 2.5011284351348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20726 , TFLOPS: 68.47557637739448, Tokens per sec: 169690.0214706301, Loss: 2.4907288551330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20727 , TFLOPS: 70.38832672381955, Tokens per sec: 174430.0275359164, Loss: 2.496677875518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20728 , TFLOPS: 70.0491061904182, Tokens per sec: 173589.4016859213, Loss: 2.469590663909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20729 , TFLOPS: 69.06045321754968, Tokens per sec: 171139.41071003742, Loss: 2.474628448486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20730 , TFLOPS: 69.66399761960291, Tokens per sec: 172635.06022422976, Loss: 2.490981101989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20731 , TFLOPS: 70.56127082807818, Tokens per sec: 174858.6020776351, Loss: 2.4863317012786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20732 , TFLOPS: 68.73605301729327, Tokens per sec: 170335.51127816993, Loss: 2.4982314109802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20733 , TFLOPS: 70.5753440582054, Tokens per sec: 174893.47709218465, Loss: 2.4891440868377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20734 , TFLOPS: 69.58338420202188, Tokens per sec: 172435.29129516296, Loss: 2.4879724979400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20735 , TFLOPS: 69.79382562445635, Tokens per sec: 172956.788321992, Loss: 2.4791336059570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20736 , TFLOPS: 69.23129909545342, Tokens per sec: 171562.78561571002, Loss: 2.482790470123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20737 , TFLOPS: 69.56949520302632, Tokens per sec: 172400.87282565242, Loss: 2.489722728729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20738 , TFLOPS: 70.66284896819455, Tokens per sec: 175110.32389859937, Loss: 2.508021116256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20739 , TFLOPS: 70.05117333079987, Tokens per sec: 173594.52428750135, Loss: 2.4702770709991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20740 , TFLOPS: 70.07377017420217, Tokens per sec: 173650.52175470005, Loss: 2.4885671138763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20741 , TFLOPS: 69.65999948589553, Tokens per sec: 172625.15240847212, Loss: 2.491626501083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20742 , TFLOPS: 69.07579858028886, Tokens per sec: 171177.43820933637, Loss: 2.480250597000122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20743 , TFLOPS: 69.8255425271558, Tokens per sec: 173035.38630078651, Loss: 2.4830188751220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20744 , TFLOPS: 70.51249323096421, Tokens per sec: 174737.72581869081, Loss: 2.4953808784484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20745 , TFLOPS: 68.92684241631684, Tokens per sec: 170808.30842613766, Loss: 2.464879274368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20746 , TFLOPS: 68.92522877020754, Tokens per sec: 170804.30963332104, Loss: 2.4920201301574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20747 , TFLOPS: 71.51767529618598, Tokens per sec: 177228.67770625584, Loss: 2.4980056285858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20748 , TFLOPS: 69.73115167848519, Tokens per sec: 172801.47537977036, Loss: 2.4750888347625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20749 , TFLOPS: 69.45498517736777, Tokens per sec: 172117.10436774988, Loss: 2.4788901805877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20750 , TFLOPS: 70.053252541565, Tokens per sec: 173599.67680082133, Loss: 2.4780890941619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20751 , TFLOPS: 70.59118328415103, Tokens per sec: 174932.7284389127, Loss: 2.504061698913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20752 , TFLOPS: 68.82816942116874, Tokens per sec: 170563.78587443294, Loss: 2.4831981658935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20753 , TFLOPS: 69.97085415708568, Tokens per sec: 173395.48452715023, Loss: 2.4745256900787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20754 , TFLOPS: 70.63946422122281, Tokens per sec: 175052.37391956084, Loss: 2.4889187812805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20755 , TFLOPS: 70.50947898776269, Tokens per sec: 174730.25619199112, Loss: 2.480219602584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20756 , TFLOPS: 69.12779133552093, Tokens per sec: 171306.28198427623, Loss: 2.4859225749969482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20757 , TFLOPS: 69.92760839693497, Tokens per sec: 173288.316769583, Loss: 2.482140064239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20758 , TFLOPS: 71.50079197414587, Tokens per sec: 177186.838974389, Loss: 2.468658924102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20759 , TFLOPS: 68.7763070095888, Tokens per sec: 170435.2651054216, Loss: 2.4761648178100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20760 , TFLOPS: 69.55257549625479, Tokens per sec: 172358.943928413, Loss: 2.489609956741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20761 , TFLOPS: 70.63921994471833, Tokens per sec: 175051.76857547348, Loss: 2.493488311767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20762 , TFLOPS: 68.99528038830289, Tokens per sec: 170977.90525978408, Loss: 2.5013115406036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20763 , TFLOPS: 69.78350790470934, Tokens per sec: 172931.2198758682, Loss: 2.470970630645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20764 , TFLOPS: 69.54497565529384, Tokens per sec: 172340.11068532016, Loss: 2.49877667427063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20765 , TFLOPS: 69.91133267534084, Tokens per sec: 173247.983738554, Loss: 2.5042245388031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20766 , TFLOPS: 68.94416702859775, Tokens per sec: 170851.24072383335, Loss: 2.5000967979431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20767 , TFLOPS: 70.48269880397109, Tokens per sec: 174663.8919464748, Loss: 2.504075527191162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20768 , TFLOPS: 69.80372373118982, Tokens per sec: 172981.316920844, Loss: 2.481680393218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20769 , TFLOPS: 69.74090193990929, Tokens per sec: 172825.63760165923, Loss: 2.4984593391418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20770 , TFLOPS: 69.77004703478725, Tokens per sec: 172897.86235736407, Loss: 2.4849421977996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20771 , TFLOPS: 70.58199968505346, Tokens per sec: 174909.97046869196, Loss: 2.4993367195129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20772 , TFLOPS: 69.79494713391055, Tokens per sec: 172959.56754596398, Loss: 2.5008864402770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20773 , TFLOPS: 69.66760844535601, Tokens per sec: 172644.00824821126, Loss: 2.504030227661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20774 , TFLOPS: 70.47314496160661, Tokens per sec: 174640.21647265294, Loss: 2.482727527618408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20775 , TFLOPS: 70.58270605775701, Tokens per sec: 174911.72093806474, Loss: 2.462172508239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20776 , TFLOPS: 68.95427383062594, Tokens per sec: 170876.28649261533, Loss: 2.4769043922424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20777 , TFLOPS: 70.58194195673435, Tokens per sec: 174909.82741155793, Loss: 2.47261905670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20778 , TFLOPS: 69.95208287631658, Tokens per sec: 173348.9672255771, Loss: 2.4762001037597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20779 , TFLOPS: 68.95232999020656, Tokens per sec: 170871.46944192445, Loss: 2.496051788330078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20780 , TFLOPS: 69.93860521931886, Tokens per sec: 173315.56810685026, Loss: 2.5089027881622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20781 , TFLOPS: 70.52656397606638, Tokens per sec: 174772.59467507454, Loss: 2.488663673400879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20782 , TFLOPS: 67.93898065457778, Tokens per sec: 168360.27815859174, Loss: 2.4955742359161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20783 , TFLOPS: 69.74607504245206, Tokens per sec: 172838.45711962454, Loss: 2.4937543869018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20784 , TFLOPS: 70.74464487671807, Tokens per sec: 175313.02317048382, Loss: 2.4767889976501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20785 , TFLOPS: 69.84145559997478, Tokens per sec: 173074.82064820162, Loss: 2.472923994064331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20786 , TFLOPS: 68.76788828835132, Tokens per sec: 170414.4025867964, Loss: 2.472071409225464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20787 , TFLOPS: 71.05376520083155, Tokens per sec: 176079.0573860518, Loss: 2.499829053878784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20788 , TFLOPS: 70.10044316487071, Tokens per sec: 173716.62036384922, Loss: 2.4771530628204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20789 , TFLOPS: 68.62695627001655, Tokens per sec: 170065.1575204194, Loss: 2.4907925128936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20790 , TFLOPS: 69.83597654479021, Tokens per sec: 173061.24294588648, Loss: 2.47536301612854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20791 , TFLOPS: 70.68027860562533, Tokens per sec: 175153.51646018593, Loss: 2.492208242416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20792 , TFLOPS: 69.83706136296077, Tokens per sec: 173063.9312448157, Loss: 2.487605571746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20793 , TFLOPS: 69.54594076656308, Tokens per sec: 172342.50233736113, Loss: 2.479257583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20794 , TFLOPS: 71.50212905140316, Tokens per sec: 177190.15239912373, Loss: 2.4829654693603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20795 , TFLOPS: 69.95120676569522, Tokens per sec: 173346.79612694582, Loss: 2.4674830436706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20796 , TFLOPS: 68.85357128211818, Tokens per sec: 170626.73448411166, Loss: 2.486666679382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20797 , TFLOPS: 69.3289585617427, Tokens per sec: 171804.79653125315, Loss: 2.4891276359558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20798 , TFLOPS: 70.71431945652984, Tokens per sec: 175237.87343863654, Loss: 2.487531900405884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20799 , TFLOPS: 68.90856170600436, Tokens per sec: 170763.00681219687, Loss: 2.5020294189453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20800 , TFLOPS: 69.80766096285184, Tokens per sec: 172991.07381462437, Loss: 2.482595443725586 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/optimizer.pt +[rank0]:[2024-08-27 21:57:44,439] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0064520589949097484, 'preprocessing_with_comm': 0.001323596981819719, 'state_converting': 1.016329272009898, : 1.0254861209832598}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800, took 4.51s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0018800 + eval ppl=10.44660472869873, eval loss=2.3462769985198975 +------------------------------------------------------------------ +iteration: 20801 , TFLOPS: 65.05939027130599, Tokens per sec: 161224.33597577686, Loss: 2.481354236602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20802 , TFLOPS: 69.88271296374047, Tokens per sec: 173177.0609404868, Loss: 2.4824860095977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20803 , TFLOPS: 69.2448787809154, Tokens per sec: 171596.43757221065, Loss: 2.5036609172821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20804 , TFLOPS: 70.11604191450611, Tokens per sec: 173755.27578378987, Loss: 2.501995325088501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20805 , TFLOPS: 71.53701054763182, Tokens per sec: 177276.5925892924, Loss: 2.492279052734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20806 , TFLOPS: 69.8023393236295, Tokens per sec: 172977.88620640442, Loss: 2.4744110107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20807 , TFLOPS: 70.16836595605938, Tokens per sec: 173884.94052272785, Loss: 2.479846477508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20808 , TFLOPS: 70.44093665767441, Tokens per sec: 174560.4007474714, Loss: 2.4949207305908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20809 , TFLOPS: 68.56724808993425, Tokens per sec: 169917.19407277644, Loss: 2.4671051502227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20810 , TFLOPS: 69.53680287291894, Tokens per sec: 172319.8576302889, Loss: 2.4769468307495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20811 , TFLOPS: 70.69306721311244, Tokens per sec: 175185.2080383204, Loss: 2.510500431060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20812 , TFLOPS: 68.63673976906102, Tokens per sec: 170089.402108793, Loss: 2.5048465728759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20813 , TFLOPS: 69.10298534925373, Tokens per sec: 171244.81001770115, Loss: 2.5041897296905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20814 , TFLOPS: 71.54249308841871, Tokens per sec: 177290.17892931405, Loss: 2.4849579334259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20815 , TFLOPS: 68.81596297320147, Tokens per sec: 170533.5369517188, Loss: 2.498903751373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20816 , TFLOPS: 69.80356440382982, Tokens per sec: 172980.9220900946, Loss: 2.5110156536102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20817 , TFLOPS: 69.89606492341015, Tokens per sec: 173210.14856738804, Loss: 2.4962270259857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20818 , TFLOPS: 70.64888739108427, Tokens per sec: 175075.7255725254, Loss: 2.481842517852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20819 , TFLOPS: 67.68435428398587, Tokens per sec: 167729.28596285472, Loss: 2.4876606464385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20820 , TFLOPS: 69.81111384948518, Tokens per sec: 172999.63044806864, Loss: 2.516324996948242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20821 , TFLOPS: 70.63525347433979, Tokens per sec: 175041.93922493266, Loss: 2.49342679977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20822 , TFLOPS: 69.84841071003098, Tokens per sec: 173092.05617708946, Loss: 2.4897007942199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20823 , TFLOPS: 68.74449548113824, Tokens per sec: 170356.43263359196, Loss: 2.5001964569091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20824 , TFLOPS: 70.68071758530894, Tokens per sec: 175154.60430019948, Loss: 2.495487689971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20825 , TFLOPS: 71.46315891756998, Tokens per sec: 177093.58011457053, Loss: 2.465029239654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20826 , TFLOPS: 69.59873977423716, Tokens per sec: 172473.34409466834, Loss: 2.4858131408691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20827 , TFLOPS: 69.4887034632415, Tokens per sec: 172200.66199452084, Loss: 2.4942851066589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20828 , TFLOPS: 70.56984151381114, Tokens per sec: 174879.84118102037, Loss: 2.4726474285125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20829 , TFLOPS: 68.39123043060279, Tokens per sec: 169481.00292300724, Loss: 2.5153632164001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20830 , TFLOPS: 68.84539974325074, Tokens per sec: 170606.48451062877, Loss: 2.4930691719055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20831 , TFLOPS: 70.69646485937385, Tokens per sec: 175193.62778003776, Loss: 2.500086784362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20832 , TFLOPS: 69.7299955471663, Tokens per sec: 172798.61035900377, Loss: 2.4892520904541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20833 , TFLOPS: 69.70514065200052, Tokens per sec: 172737.01719079027, Loss: 2.488579511642456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20834 , TFLOPS: 69.84050825936944, Tokens per sec: 173072.47303382342, Loss: 2.502264976501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20835 , TFLOPS: 70.59947297969006, Tokens per sec: 174953.27121764378, Loss: 2.4778738021850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20836 , TFLOPS: 69.19114466896285, Tokens per sec: 171463.27852349114, Loss: 2.4942498207092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20837 , TFLOPS: 68.8257712632055, Tokens per sec: 170557.8429748211, Loss: 2.478942394256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20838 , TFLOPS: 70.67664627311491, Tokens per sec: 175144.51514009063, Loss: 2.4946212768554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20839 , TFLOPS: 69.80070904179456, Tokens per sec: 172973.84618842707, Loss: 2.4846649169921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20840 , TFLOPS: 69.16954227248502, Tokens per sec: 171409.74540531865, Loss: 2.463111400604248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20841 , TFLOPS: 69.56377934544992, Tokens per sec: 172386.7083009244, Loss: 2.4873738288879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20842 , TFLOPS: 70.64285969491348, Tokens per sec: 175060.78827740863, Loss: 2.489337682723999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20843 , TFLOPS: 69.20470344868394, Tokens per sec: 171496.8786732629, Loss: 2.5050771236419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20844 , TFLOPS: 71.01005902058994, Tokens per sec: 175970.74865677985, Loss: 2.465939998626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20845 , TFLOPS: 69.9262828612324, Tokens per sec: 173285.0319460922, Loss: 2.4851536750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20846 , TFLOPS: 69.61692731907311, Tokens per sec: 172518.4148342953, Loss: 2.481325626373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20847 , TFLOPS: 69.72162206916946, Tokens per sec: 172777.85995811282, Loss: 2.4985129833221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20848 , TFLOPS: 70.71758500522634, Tokens per sec: 175245.96582803637, Loss: 2.4975459575653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20849 , TFLOPS: 68.11244669233557, Tokens per sec: 168790.14610901705, Loss: 2.4982354640960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20850 , TFLOPS: 69.80853523020939, Tokens per sec: 172993.24034544485, Loss: 2.4872758388519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20851 , TFLOPS: 70.55558025264546, Tokens per sec: 174844.5002048442, Loss: 2.4824576377868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20852 , TFLOPS: 68.74185238058655, Tokens per sec: 170349.8827392633, Loss: 2.501319169998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20853 , TFLOPS: 69.70581278770007, Tokens per sec: 172738.68281709557, Loss: 2.4965035915374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20854 , TFLOPS: 70.59177089747998, Tokens per sec: 174934.18460947275, Loss: 2.48844313621521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20855 , TFLOPS: 70.71999068365288, Tokens per sec: 175251.9273641847, Loss: 2.4978153705596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20856 , TFLOPS: 67.97351662221821, Tokens per sec: 168445.8620908556, Loss: 2.4846978187561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20857 , TFLOPS: 69.89488055315218, Tokens per sec: 173207.21356741962, Loss: 2.505809783935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20858 , TFLOPS: 70.60856309826221, Tokens per sec: 174975.79753282477, Loss: 2.4833645820617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20859 , TFLOPS: 69.64315000799571, Tokens per sec: 172583.39754611318, Loss: 2.4915690422058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20860 , TFLOPS: 68.87479459761138, Tokens per sec: 170679.328197845, Loss: 2.4874401092529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20861 , TFLOPS: 70.58190467548076, Tokens per sec: 174909.73502450465, Loss: 2.5035948753356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20862 , TFLOPS: 70.71047187133624, Tokens per sec: 175228.3386986803, Loss: 2.476431369781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20863 , TFLOPS: 68.84689556884724, Tokens per sec: 170610.19133123572, Loss: 2.4832937717437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20864 , TFLOPS: 69.618937173, Tokens per sec: 172523.39547372985, Loss: 2.493650197982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20865 , TFLOPS: 70.59539404022438, Tokens per sec: 174943.16315631554, Loss: 2.4687039852142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20866 , TFLOPS: 69.69197781645425, Tokens per sec: 172704.39823430055, Loss: 2.4978668689727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20867 , TFLOPS: 69.635053354244, Tokens per sec: 172563.33314619635, Loss: 2.4908385276794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20868 , TFLOPS: 70.7000453765161, Tokens per sec: 175202.5006959428, Loss: 2.4965429306030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20869 , TFLOPS: 68.91017082276882, Tokens per sec: 170766.99438079802, Loss: 2.4872705936431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20870 , TFLOPS: 69.62513791970771, Tokens per sec: 172538.76160714016, Loss: 2.4949209690093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20871 , TFLOPS: 69.55328242134607, Tokens per sec: 172360.6957666634, Loss: 2.4851059913635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20872 , TFLOPS: 70.71461099650281, Tokens per sec: 175238.59590680536, Loss: 2.4966049194335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20873 , TFLOPS: 68.976715513585, Tokens per sec: 170931.8994551434, Loss: 2.487151622772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20874 , TFLOPS: 69.66409197964589, Tokens per sec: 172635.29405881092, Loss: 2.495327949523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20875 , TFLOPS: 70.74005005511573, Tokens per sec: 175301.63669639183, Loss: 2.5012519359588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20876 , TFLOPS: 68.88870098470439, Tokens per sec: 170713.78975697645, Loss: 2.4867100715637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20877 , TFLOPS: 69.6683465063356, Tokens per sec: 172645.8372446228, Loss: 2.4937405586242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20878 , TFLOPS: 69.64030759837205, Tokens per sec: 172576.35374194843, Loss: 2.500473976135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20879 , TFLOPS: 69.74135370149256, Tokens per sec: 172826.75711662823, Loss: 2.4795949459075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20880 , TFLOPS: 69.06973067592453, Tokens per sec: 171162.40127388673, Loss: 2.4847211837768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20881 , TFLOPS: 71.47595907914597, Tokens per sec: 177125.30032501, Loss: 2.4844319820404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20882 , TFLOPS: 69.16749101585002, Tokens per sec: 171404.66216541256, Loss: 2.4749932289123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20883 , TFLOPS: 69.67787740257896, Tokens per sec: 172669.45585542877, Loss: 2.479130268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20884 , TFLOPS: 70.11665598367001, Tokens per sec: 173756.7975148233, Loss: 2.4818296432495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20885 , TFLOPS: 70.62830369716657, Tokens per sec: 175024.71691151627, Loss: 2.4666898250579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20886 , TFLOPS: 68.70012115940384, Tokens per sec: 170246.468176099, Loss: 2.4771671295166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20887 , TFLOPS: 69.73581628734016, Tokens per sec: 172813.03479436206, Loss: 2.4963934421539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20888 , TFLOPS: 70.70537572903618, Tokens per sec: 175215.70989667435, Loss: 2.4867069721221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20889 , TFLOPS: 68.8244867775613, Tokens per sec: 170554.65987789704, Loss: 2.4961905479431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20890 , TFLOPS: 69.25596133413393, Tokens per sec: 171623.90135993017, Loss: 2.4667069911956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20891 , TFLOPS: 70.56360621917364, Tokens per sec: 174864.3894340342, Loss: 2.4787685871124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20892 , TFLOPS: 71.55407522667304, Tokens per sec: 177318.8807437861, Loss: 2.4835710525512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20893 , TFLOPS: 68.9550488626373, Tokens per sec: 170878.20710731632, Loss: 2.4767956733703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20894 , TFLOPS: 70.05460449320363, Tokens per sec: 173603.02708591163, Loss: 2.4944875240325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20895 , TFLOPS: 70.71336003817161, Tokens per sec: 175235.49589425622, Loss: 2.498504638671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20896 , TFLOPS: 68.93136666689104, Tokens per sec: 170819.52001744785, Loss: 2.511073350906372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20897 , TFLOPS: 68.93156880483517, Tokens per sec: 170820.02093754074, Loss: 2.480320453643799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20898 , TFLOPS: 70.56826306449028, Tokens per sec: 174875.92960972272, Loss: 2.512235164642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20899 , TFLOPS: 69.80571486864878, Tokens per sec: 172986.25117880918, Loss: 2.4842677116394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20900 , TFLOPS: 69.81321083342446, Tokens per sec: 173004.82700527206, Loss: 2.4798758029937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20901 , TFLOPS: 69.9308202539045, Tokens per sec: 173296.27610496874, Loss: 2.4795937538146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20902 , TFLOPS: 69.94500639228403, Tokens per sec: 173331.43091860536, Loss: 2.4854576587677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20903 , TFLOPS: 69.66821534502844, Tokens per sec: 172645.51221245507, Loss: 2.4786219596862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20904 , TFLOPS: 68.9266318081195, Tokens per sec: 170807.78651582423, Loss: 2.4936554431915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20905 , TFLOPS: 70.60496981718231, Tokens per sec: 174966.89298647622, Loss: 2.49211049079895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20906 , TFLOPS: 70.60266352934295, Tokens per sec: 174961.17775115167, Loss: 2.5135610103607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20907 , TFLOPS: 69.2321251631585, Tokens per sec: 171564.83270248206, Loss: 2.4888107776641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20908 , TFLOPS: 69.66641055518878, Tokens per sec: 172641.03974441835, Loss: 2.5045809745788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20909 , TFLOPS: 70.61633883691883, Tokens per sec: 174995.0666697842, Loss: 2.4968230724334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20910 , TFLOPS: 69.28990283417312, Tokens per sec: 171708.01213599218, Loss: 2.4830093383789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20911 , TFLOPS: 69.8140004799196, Tokens per sec: 173006.7838362737, Loss: 2.4831957817077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20912 , TFLOPS: 71.51515481207531, Tokens per sec: 177222.43166897536, Loss: 2.4763522148132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20913 , TFLOPS: 68.96482936883373, Tokens per sec: 170902.4442790828, Loss: 2.5039515495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20914 , TFLOPS: 69.86842860956604, Tokens per sec: 173141.66273729104, Loss: 2.484274387359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20915 , TFLOPS: 69.66675661623358, Tokens per sec: 172641.89732180917, Loss: 2.5009520053863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20916 , TFLOPS: 68.94169477865147, Tokens per sec: 170845.11421612566, Loss: 2.4839110374450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20917 , TFLOPS: 69.43072111337439, Tokens per sec: 172056.97534426698, Loss: 2.515834093093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20918 , TFLOPS: 70.67387985268562, Tokens per sec: 175137.65964552047, Loss: 2.4910314083099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20919 , TFLOPS: 68.8420755594348, Tokens per sec: 170598.24681694384, Loss: 2.5109806060791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20920 , TFLOPS: 69.87897388781474, Tokens per sec: 173167.7950984496, Loss: 2.508577585220337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20921 , TFLOPS: 70.54786718451898, Tokens per sec: 174825.38637236218, Loss: 2.4728477001190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20922 , TFLOPS: 70.65117133807547, Tokens per sec: 175081.38544476713, Loss: 2.4785656929016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20923 , TFLOPS: 68.19503051142199, Tokens per sec: 168994.7979100719, Loss: 2.5179669857025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20924 , TFLOPS: 69.87963847817466, Tokens per sec: 173169.44202657213, Loss: 2.4838852882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20925 , TFLOPS: 70.65850394907261, Tokens per sec: 175099.55646257184, Loss: 2.5068349838256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20926 , TFLOPS: 69.66941591024802, Tokens per sec: 172648.48734532238, Loss: 2.500633955001831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20927 , TFLOPS: 68.96635229104417, Tokens per sec: 170906.218248084, Loss: 2.500314950942993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20928 , TFLOPS: 70.65937401725903, Tokens per sec: 175101.7125873837, Loss: 2.4645190238952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20929 , TFLOPS: 70.2515496046372, Tokens per sec: 174091.07876733903, Loss: 2.4895431995391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20930 , TFLOPS: 67.94598655797792, Tokens per sec: 168377.63955898123, Loss: 2.467717170715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20931 , TFLOPS: 70.63558791050036, Tokens per sec: 175042.76799457974, Loss: 2.476400375366211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20932 , TFLOPS: 70.89010619677252, Tokens per sec: 175673.49234546593, Loss: 2.460663080215454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20933 , TFLOPS: 68.91796220216042, Tokens per sec: 170786.30227722175, Loss: 2.4956161975860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20934 , TFLOPS: 68.7568696382161, Tokens per sec: 170387.09715795683, Loss: 2.5047619342803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20935 , TFLOPS: 70.65755586098389, Tokens per sec: 175097.20699584743, Loss: 2.4928224086761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20936 , TFLOPS: 68.81093686450855, Tokens per sec: 170521.08170070616, Loss: 2.4931373596191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20937 , TFLOPS: 69.93151122343608, Tokens per sec: 173297.98840358458, Loss: 2.4909632205963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20938 , TFLOPS: 69.66757927472389, Tokens per sec: 172643.93596017145, Loss: 2.471057653427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20939 , TFLOPS: 70.72192283939346, Tokens per sec: 175256.71545895436, Loss: 2.481245994567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20940 , TFLOPS: 69.19239406457557, Tokens per sec: 171466.37466345174, Loss: 2.5069022178649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20941 , TFLOPS: 69.58312835411338, Tokens per sec: 172434.65727586174, Loss: 2.5058088302612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20942 , TFLOPS: 70.62889304994644, Tokens per sec: 175026.17739262735, Loss: 2.4961371421813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20943 , TFLOPS: 68.84684155714689, Tokens per sec: 170610.05748429234, Loss: 2.481966972351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20944 , TFLOPS: 69.91440486370125, Tokens per sec: 173255.5969597406, Loss: 2.4840548038482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20945 , TFLOPS: 70.50198839686541, Tokens per sec: 174711.69368259097, Loss: 2.4878199100494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20946 , TFLOPS: 70.45603132857066, Tokens per sec: 174597.8070047676, Loss: 2.4923205375671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20947 , TFLOPS: 68.96945244310324, Tokens per sec: 170913.90076639628, Loss: 2.4879889488220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20948 , TFLOPS: 70.43769954175735, Tokens per sec: 174552.37881763116, Loss: 2.4904420375823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20949 , TFLOPS: 69.97163624881242, Tokens per sec: 173397.4226366036, Loss: 2.4546117782592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20950 , TFLOPS: 69.85814082844583, Tokens per sec: 173116.1684823825, Loss: 2.500319719314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20951 , TFLOPS: 70.0702421154757, Tokens per sec: 173641.7788365281, Loss: 2.5037946701049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20952 , TFLOPS: 69.7587137991663, Tokens per sec: 172869.77735103638, Loss: 2.46887469291687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20953 , TFLOPS: 68.84122634315666, Tokens per sec: 170596.14236545772, Loss: 2.4775681495666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20954 , TFLOPS: 69.85057478543325, Tokens per sec: 173097.41899432888, Loss: 2.4905197620391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20955 , TFLOPS: 70.59507575145501, Tokens per sec: 174942.37440168264, Loss: 2.511101722717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20956 , TFLOPS: 68.69117263022676, Tokens per sec: 170224.29273503638, Loss: 2.4561548233032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20957 , TFLOPS: 69.18374300217319, Tokens per sec: 171444.93637782545, Loss: 2.4610235691070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20958 , TFLOPS: 70.74103293980075, Tokens per sec: 175304.072392915, Loss: 2.5052967071533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20959 , TFLOPS: 71.47511563199826, Tokens per sec: 177123.21017006465, Loss: 2.4858920574188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20960 , TFLOPS: 68.37954833081484, Tokens per sec: 169452.05339284314, Loss: 2.498891830444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20961 , TFLOPS: 70.0622650180795, Tokens per sec: 173622.01071043054, Loss: 2.4741153717041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20962 , TFLOPS: 70.58675414084722, Tokens per sec: 174921.75253389546, Loss: 2.517979145050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20963 , TFLOPS: 67.87698155406679, Tokens per sec: 168206.63755776073, Loss: 2.5057456493377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20964 , TFLOPS: 69.75023802065064, Tokens per sec: 172848.77343819052, Loss: 2.509489059448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20965 , TFLOPS: 70.69070747294013, Tokens per sec: 175179.36034222654, Loss: 2.463164806365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20966 , TFLOPS: 70.48980498106076, Tokens per sec: 174681.50183611296, Loss: 2.4894323348999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20967 , TFLOPS: 67.99586979887069, Tokens per sec: 168501.25572500614, Loss: 2.481109380722046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20968 , TFLOPS: 70.89498498695689, Tokens per sec: 175685.58252498592, Loss: 2.4710326194763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20969 , TFLOPS: 69.72581610495816, Tokens per sec: 172788.2532408084, Loss: 2.4765591621398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20970 , TFLOPS: 69.81460099344615, Tokens per sec: 173008.27197494457, Loss: 2.474104404449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20971 , TFLOPS: 69.73202189046859, Tokens per sec: 172803.63186092698, Loss: 2.490795373916626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20972 , TFLOPS: 70.68572950300782, Tokens per sec: 175167.0243843656, Loss: 2.4836065769195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20973 , TFLOPS: 69.93539166774592, Tokens per sec: 173307.6045720506, Loss: 2.478031635284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20974 , TFLOPS: 69.08441173141702, Tokens per sec: 171198.78254664887, Loss: 2.499138832092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20975 , TFLOPS: 69.49784317821785, Tokens per sec: 172223.31121505488, Loss: 2.4682445526123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20976 , TFLOPS: 70.57730575434724, Tokens per sec: 174898.3383913228, Loss: 2.477771043777466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20977 , TFLOPS: 69.19982475318044, Tokens per sec: 171484.78872837254, Loss: 2.49727725982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20978 , TFLOPS: 69.19833288944714, Tokens per sec: 171481.09172569908, Loss: 2.5176784992218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20979 , TFLOPS: 71.53109598410146, Tokens per sec: 177261.93564932118, Loss: 2.489232063293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20980 , TFLOPS: 69.08689479585459, Tokens per sec: 171204.9358538573, Loss: 2.4960436820983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20981 , TFLOPS: 69.85561915132779, Tokens per sec: 173109.91948869836, Loss: 2.484529733657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20982 , TFLOPS: 69.48021760446854, Tokens per sec: 172179.63310168684, Loss: 2.4608569145202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20983 , TFLOPS: 68.79098449614479, Tokens per sec: 170471.63753396593, Loss: 2.5027990341186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20984 , TFLOPS: 69.80847832753612, Tokens per sec: 172993.09933435218, Loss: 2.4945201873779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20985 , TFLOPS: 69.6970823087558, Tokens per sec: 172717.04772853001, Loss: 2.5001862049102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20986 , TFLOPS: 70.49146364123028, Tokens per sec: 174685.61217872959, Loss: 2.458676338195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20987 , TFLOPS: 69.80033002919987, Tokens per sec: 172972.90695346575, Loss: 2.473755121231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20988 , TFLOPS: 70.77605006514325, Tokens per sec: 175390.8486869685, Loss: 2.503223419189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20989 , TFLOPS: 69.5004693012601, Tokens per sec: 172229.81903724495, Loss: 2.4726767539978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20990 , TFLOPS: 69.23276758389584, Tokens per sec: 171566.42469183766, Loss: 2.469099760055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20991 , TFLOPS: 69.72831253199534, Tokens per sec: 172794.4396620107, Loss: 2.4823801517486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20992 , TFLOPS: 70.66108614247108, Tokens per sec: 175105.9554222653, Loss: 2.496365785598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20993 , TFLOPS: 68.75609428632646, Tokens per sec: 170385.17575056225, Loss: 2.484637975692749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20994 , TFLOPS: 69.10651792810503, Tokens per sec: 171253.56413724038, Loss: 2.493201732635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20995 , TFLOPS: 70.58930574708752, Tokens per sec: 174928.07569524154, Loss: 2.5022785663604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20996 , TFLOPS: 70.578850040056, Tokens per sec: 174902.16530142623, Loss: 2.49530029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20997 , TFLOPS: 69.19479610863299, Tokens per sec: 171472.327193233, Loss: 2.4982852935791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20998 , TFLOPS: 70.05462081707617, Tokens per sec: 173603.067538266, Loss: 2.4641776084899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20999 , TFLOPS: 71.01291494445515, Tokens per sec: 175977.82595072387, Loss: 2.4974894523620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21000 , TFLOPS: 67.79598638768172, Tokens per sec: 168005.92261310443, Loss: 2.4719290733337402 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/model.pt +[rank0]:[2024-08-27 22:23:09,078] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006372844974976033, 'preprocessing_with_comm': 0.0012180129997432232, 'state_converting': 0.9626875900139567, : 0.9716451200074516}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000, took 4.32s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019000 + eval ppl=9.373334884643555, eval loss=2.2378690242767334 +------------------------------------------------------------------ +iteration: 21001 , TFLOPS: 64.77442183250692, Tokens per sec: 160518.15279256774, Loss: 2.4727892875671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21002 , TFLOPS: 70.52231082265013, Tokens per sec: 174762.0548923855, Loss: 2.4917080402374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21003 , TFLOPS: 69.25322990448733, Tokens per sec: 171617.1325763741, Loss: 2.473727226257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21004 , TFLOPS: 70.08317693801733, Tokens per sec: 173673.83275167464, Loss: 2.4785778522491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21005 , TFLOPS: 70.7406068594465, Tokens per sec: 175303.01651886053, Loss: 2.458923101425171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21006 , TFLOPS: 70.54925460566517, Tokens per sec: 174828.8245547995, Loss: 2.481926441192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21007 , TFLOPS: 68.04551835453762, Tokens per sec: 168624.29031518873, Loss: 2.493598222732544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21008 , TFLOPS: 70.69863690372227, Tokens per sec: 175199.0103452036, Loss: 2.5036401748657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21009 , TFLOPS: 70.5559749019297, Tokens per sec: 174845.47818924475, Loss: 2.492842435836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21010 , TFLOPS: 68.77006512378652, Tokens per sec: 170419.79702480306, Loss: 2.4945716857910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21011 , TFLOPS: 69.5693504909544, Tokens per sec: 172400.51421319688, Loss: 2.494938850402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21012 , TFLOPS: 70.68565610896748, Tokens per sec: 175166.84250584865, Loss: 2.4740991592407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21013 , TFLOPS: 69.76204424772216, Tokens per sec: 172878.030569437, Loss: 2.4960126876831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21014 , TFLOPS: 68.93560283001426, Tokens per sec: 170830.01769631912, Loss: 2.507638454437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21015 , TFLOPS: 70.44982278456928, Tokens per sec: 174582.42154880456, Loss: 2.490291118621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21016 , TFLOPS: 69.7022374509187, Tokens per sec: 172729.82273869548, Loss: 2.4891269207000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21017 , TFLOPS: 68.99682808837234, Tokens per sec: 170981.7406310494, Loss: 2.4977400302886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21018 , TFLOPS: 69.7114395037947, Tokens per sec: 172752.626439412, Loss: 2.4745163917541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21019 , TFLOPS: 70.6327133584472, Tokens per sec: 175035.64453793396, Loss: 2.484518051147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21020 , TFLOPS: 69.78011955250847, Tokens per sec: 172922.82316585668, Loss: 2.490109443664551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21021 , TFLOPS: 69.05348509127734, Tokens per sec: 171122.1429255321, Loss: 2.484347105026245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21022 , TFLOPS: 69.3954375401683, Tokens per sec: 171969.53876305584, Loss: 2.4921586513519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21023 , TFLOPS: 70.67124232137955, Tokens per sec: 175131.12355239596, Loss: 2.495727062225342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21024 , TFLOPS: 68.42218637479228, Tokens per sec: 169557.71516278756, Loss: 2.487562417984009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21025 , TFLOPS: 70.70594935298823, Tokens per sec: 175217.13140001742, Loss: 2.4860806465148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21026 , TFLOPS: 69.65952905392408, Tokens per sec: 172623.9866262245, Loss: 2.502702236175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21027 , TFLOPS: 68.25606505230891, Tokens per sec: 169146.0482258994, Loss: 2.465198278427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21028 , TFLOPS: 70.53867394096858, Tokens per sec: 174802.60450212492, Loss: 2.456251621246338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21029 , TFLOPS: 70.57849053012981, Tokens per sec: 174901.27439622587, Loss: 2.49826979637146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21030 , TFLOPS: 67.8568479942016, Tokens per sec: 168156.74437852012, Loss: 2.493833065032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21031 , TFLOPS: 69.88973189883039, Tokens per sec: 173194.45463483915, Loss: 2.492284059524536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21032 , TFLOPS: 70.71370710542301, Tokens per sec: 175236.3559651378, Loss: 2.497985601425171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21033 , TFLOPS: 68.6640844623842, Tokens per sec: 170157.16527111447, Loss: 2.458458185195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21034 , TFLOPS: 69.02030899329745, Tokens per sec: 171039.92890008903, Loss: 2.5007715225219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21035 , TFLOPS: 71.55187209838182, Tokens per sec: 177313.4211491859, Loss: 2.4892330169677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21036 , TFLOPS: 69.73578717879062, Tokens per sec: 172812.96266016972, Loss: 2.4679272174835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21037 , TFLOPS: 69.47971525912648, Tokens per sec: 172178.38823459134, Loss: 2.4583566188812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21038 , TFLOPS: 70.09722388431082, Tokens per sec: 173708.64263198874, Loss: 2.4820432662963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21039 , TFLOPS: 70.65929754692448, Tokens per sec: 175101.52308547078, Loss: 2.4748575687408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21040 , TFLOPS: 68.55706247475807, Tokens per sec: 169891.95299633176, Loss: 2.459040403366089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21041 , TFLOPS: 68.92590893006911, Tokens per sec: 170805.9951443838, Loss: 2.4748570919036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21042 , TFLOPS: 70.63467306208842, Tokens per sec: 175040.5008994361, Loss: 2.4877028465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21043 , TFLOPS: 70.69718195498791, Tokens per sec: 175195.404821964, Loss: 2.487107515335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21044 , TFLOPS: 66.8086119340063, Tokens per sec: 165559.09994862156, Loss: 2.5102481842041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21045 , TFLOPS: 71.45925351698959, Tokens per sec: 177083.90210171693, Loss: 2.502293109893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21046 , TFLOPS: 70.90994850503229, Tokens per sec: 175722.66377114435, Loss: 2.4764184951782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21047 , TFLOPS: 68.6961270187354, Tokens per sec: 170236.5702555317, Loss: 2.4582550525665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21048 , TFLOPS: 69.95484579099595, Tokens per sec: 173355.81403251275, Loss: 2.4786713123321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21049 , TFLOPS: 70.5814339985913, Tokens per sec: 174908.5686353232, Loss: 2.490069627761841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21050 , TFLOPS: 69.85971258601687, Tokens per sec: 173120.06347078693, Loss: 2.4968576431274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21051 , TFLOPS: 69.29433258197159, Tokens per sec: 171718.98953901383, Loss: 2.490811586380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21052 , TFLOPS: 69.6032570211319, Tokens per sec: 172484.5383300893, Loss: 2.504354238510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21053 , TFLOPS: 70.5434485412952, Tokens per sec: 174814.43648769995, Loss: 2.481003522872925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21054 , TFLOPS: 68.93546626309661, Tokens per sec: 170829.67926845312, Loss: 2.4914398193359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21055 , TFLOPS: 69.9267609019403, Tokens per sec: 173286.21658362693, Loss: 2.4923722743988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21056 , TFLOPS: 69.74371716759366, Tokens per sec: 172832.61404598402, Loss: 2.486767530441284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21057 , TFLOPS: 69.7181437758796, Tokens per sec: 172769.24036417494, Loss: 2.4845542907714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21058 , TFLOPS: 69.90177904597091, Tokens per sec: 173224.30879255617, Loss: 2.493989944458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21059 , TFLOPS: 69.64908429526773, Tokens per sec: 172598.10336368892, Loss: 2.4816627502441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21060 , TFLOPS: 69.66159533676536, Tokens per sec: 172629.10710272504, Loss: 2.486168384552002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21061 , TFLOPS: 68.96102489008635, Tokens per sec: 170893.01636164886, Loss: 2.4740219116210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21062 , TFLOPS: 71.5177875259719, Tokens per sec: 177228.9558240285, Loss: 2.4863343238830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21063 , TFLOPS: 68.26543882458078, Tokens per sec: 169169.27746619537, Loss: 2.4983818531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21064 , TFLOPS: 68.89512828395739, Tokens per sec: 170729.71731255087, Loss: 2.480381488800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21065 , TFLOPS: 70.93357751427867, Tokens per sec: 175781.21905900716, Loss: 2.4894657135009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21066 , TFLOPS: 70.66100472685353, Tokens per sec: 175105.75366539636, Loss: 2.4879727363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21067 , TFLOPS: 67.93015329625132, Tokens per sec: 168338.40299224496, Loss: 2.4727303981781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21068 , TFLOPS: 69.84006437940766, Tokens per sec: 173071.3730503807, Loss: 2.4870333671569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21069 , TFLOPS: 70.63734804119473, Tokens per sec: 175047.12979233477, Loss: 2.4916090965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21070 , TFLOPS: 68.55382357948868, Tokens per sec: 169883.9266570604, Loss: 2.4663705825805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21071 , TFLOPS: 68.92727231412249, Tokens per sec: 170809.37376025724, Loss: 2.4840481281280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21072 , TFLOPS: 70.6322984197554, Tokens per sec: 175034.61627414642, Loss: 2.4969162940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21073 , TFLOPS: 70.4721335607758, Tokens per sec: 174637.71010997923, Loss: 2.492469072341919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21074 , TFLOPS: 67.72609070714358, Tokens per sec: 167832.71341708608, Loss: 2.485461473464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21075 , TFLOPS: 70.68499009416702, Tokens per sec: 175165.19204780553, Loss: 2.476940155029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21076 , TFLOPS: 70.5572054505109, Tokens per sec: 174848.5276241851, Loss: 2.5011239051818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21077 , TFLOPS: 68.22677835587363, Tokens per sec: 169073.47256593686, Loss: 2.464787483215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21078 , TFLOPS: 69.8050818807742, Tokens per sec: 172984.6825651266, Loss: 2.4940967559814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21079 , TFLOPS: 70.67150536543221, Tokens per sec: 175131.77540453515, Loss: 2.482954502105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21080 , TFLOPS: 69.6772978997364, Tokens per sec: 172668.0197835474, Loss: 2.4873385429382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21081 , TFLOPS: 67.74941178487883, Tokens per sec: 167890.50561674283, Loss: 2.5080623626708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21082 , TFLOPS: 71.47551029137792, Tokens per sec: 177124.18817948273, Loss: 2.503669500350952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21083 , TFLOPS: 69.3800026805317, Tokens per sec: 171931.28948058616, Loss: 2.4785990715026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21084 , TFLOPS: 68.82103175517273, Tokens per sec: 170546.09795181602, Loss: 2.4913973808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21085 , TFLOPS: 69.73576788930399, Tokens per sec: 172812.91485869692, Loss: 2.4773318767547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21086 , TFLOPS: 70.59657867023755, Tokens per sec: 174946.09879998595, Loss: 2.4782042503356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21087 , TFLOPS: 69.73107299792746, Tokens per sec: 172801.2804006809, Loss: 2.496743679046631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21088 , TFLOPS: 69.34380275939904, Tokens per sec: 171841.58208826891, Loss: 2.4715170860290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21089 , TFLOPS: 69.8257717508794, Tokens per sec: 173035.95434242464, Loss: 2.4914026260375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21090 , TFLOPS: 70.6650287943508, Tokens per sec: 175115.72574794223, Loss: 2.494100570678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21091 , TFLOPS: 68.29611283316612, Tokens per sec: 169245.2910384908, Loss: 2.480651378631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21092 , TFLOPS: 70.4760685793362, Tokens per sec: 174647.46151944998, Loss: 2.4509198665618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21093 , TFLOPS: 70.83906920401168, Tokens per sec: 175547.0170552157, Loss: 2.4850714206695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21094 , TFLOPS: 68.1986476462969, Tokens per sec: 169003.76156875223, Loss: 2.5103073120117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21095 , TFLOPS: 70.7240978184979, Tokens per sec: 175262.1052967689, Loss: 2.482551097869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21096 , TFLOPS: 69.60650601973285, Tokens per sec: 172492.58970652922, Loss: 2.4823966026306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21097 , TFLOPS: 68.9564400561424, Tokens per sec: 170881.65463807457, Loss: 2.4650142192840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21098 , TFLOPS: 69.75203989665708, Tokens per sec: 172853.2386854275, Loss: 2.4761273860931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21099 , TFLOPS: 70.61240433793533, Tokens per sec: 174985.31654788228, Loss: 2.506484031677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21100 , TFLOPS: 68.79968978142522, Tokens per sec: 170493.21018985717, Loss: 2.4880716800689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21101 , TFLOPS: 69.02383214326275, Tokens per sec: 171048.65965381006, Loss: 2.5023772716522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21102 , TFLOPS: 71.52019119495091, Tokens per sec: 177234.91238054447, Loss: 2.499190330505371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21103 , TFLOPS: 69.81710903925558, Tokens per sec: 173014.4871887444, Loss: 2.517886161804199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21104 , TFLOPS: 68.64738501277021, Tokens per sec: 170115.78219537478, Loss: 2.513061046600342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21105 , TFLOPS: 70.03915767623259, Tokens per sec: 173564.74817185002, Loss: 2.462942361831665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21106 , TFLOPS: 70.62499585401453, Tokens per sec: 175016.5197117965, Loss: 2.4882442951202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21107 , TFLOPS: 69.49699273158458, Tokens per sec: 172221.20371461325, Loss: 2.4860076904296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21108 , TFLOPS: 68.30249624580851, Tokens per sec: 169261.10983820964, Loss: 2.491285562515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21109 , TFLOPS: 70.67913067169631, Tokens per sec: 175150.6717534544, Loss: 2.5067505836486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21110 , TFLOPS: 70.58194011258016, Tokens per sec: 174909.82284154068, Loss: 2.4630112648010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21111 , TFLOPS: 67.20781664085067, Tokens per sec: 166548.37318821068, Loss: 2.5108389854431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21112 , TFLOPS: 71.47797030317815, Tokens per sec: 177130.2843597164, Loss: 2.49355149269104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21113 , TFLOPS: 70.59685607056976, Tokens per sec: 174946.7862285673, Loss: 2.4745235443115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21114 , TFLOPS: 67.86481948634164, Tokens per sec: 168176.4986141755, Loss: 2.4867656230926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21115 , TFLOPS: 70.18026322525716, Tokens per sec: 173914.42326639235, Loss: 2.453096389770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21116 , TFLOPS: 70.74347977383141, Tokens per sec: 175310.13591718397, Loss: 2.499959945678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21117 , TFLOPS: 68.81609367334954, Tokens per sec: 170533.86084108351, Loss: 2.4852659702301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21118 , TFLOPS: 68.9492973850028, Tokens per sec: 170863.95431216047, Loss: 2.4821231365203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21119 , TFLOPS: 70.6246396233559, Tokens per sec: 175015.6369329824, Loss: 2.4794373512268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21120 , TFLOPS: 69.7368902984906, Tokens per sec: 172815.69631230494, Loss: 2.488772392272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21121 , TFLOPS: 69.69059728726356, Tokens per sec: 172700.97713088815, Loss: 2.496781587600708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21122 , TFLOPS: 69.89323939476579, Tokens per sec: 173203.14659614966, Loss: 2.4860799312591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21123 , TFLOPS: 69.78754801927768, Tokens per sec: 172941.23172482397, Loss: 2.5061333179473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21124 , TFLOPS: 69.54370836962754, Tokens per sec: 172336.97021183601, Loss: 2.500988245010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21125 , TFLOPS: 69.93488495483932, Tokens per sec: 173306.34888164903, Loss: 2.472154140472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21126 , TFLOPS: 69.59174573737877, Tokens per sec: 172456.0121008765, Loss: 2.4955081939697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21127 , TFLOPS: 69.8507476809632, Tokens per sec: 173097.84744849813, Loss: 2.49179744720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21128 , TFLOPS: 68.87916907870111, Tokens per sec: 170690.1686438742, Loss: 2.501997470855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21129 , TFLOPS: 70.54622774306515, Tokens per sec: 174821.3236558402, Loss: 2.489588737487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21130 , TFLOPS: 70.08633489080718, Tokens per sec: 173681.6585065651, Loss: 2.516207218170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21131 , TFLOPS: 68.9583426323861, Tokens per sec: 170886.36943159258, Loss: 2.4754064083099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21132 , TFLOPS: 70.88691713050764, Tokens per sec: 175665.5894879569, Loss: 2.5045249462127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21133 , TFLOPS: 69.5111043137433, Tokens per sec: 172256.17376972164, Loss: 2.490041971206665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21134 , TFLOPS: 68.8328925604995, Tokens per sec: 170575.49033399133, Loss: 2.4847536087036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21135 , TFLOPS: 69.67256271381689, Tokens per sec: 172656.28547121506, Loss: 2.476424217224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21136 , TFLOPS: 70.72621167891499, Tokens per sec: 175267.3436757434, Loss: 2.4782931804656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21137 , TFLOPS: 68.73533766482346, Tokens per sec: 170333.7385559472, Loss: 2.4944722652435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21138 , TFLOPS: 69.29559010595271, Tokens per sec: 171722.1058219667, Loss: 2.492676258087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21139 , TFLOPS: 70.52412733294494, Tokens per sec: 174766.55640500117, Loss: 2.4838290214538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21140 , TFLOPS: 70.65261851007124, Tokens per sec: 175084.9716963918, Loss: 2.501927137374878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21141 , TFLOPS: 68.2106504032297, Tokens per sec: 169033.50572264456, Loss: 2.482865810394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21142 , TFLOPS: 70.60237700769203, Tokens per sec: 174960.46771893773, Loss: 2.474520683288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21143 , TFLOPS: 70.65001230059403, Tokens per sec: 175078.5132222096, Loss: 2.489124059677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21144 , TFLOPS: 68.11601699353572, Tokens per sec: 168798.9937086922, Loss: 2.4876487255096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21145 , TFLOPS: 69.79891123921678, Tokens per sec: 172969.39103559506, Loss: 2.4979028701782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21146 , TFLOPS: 70.59496597538964, Tokens per sec: 174942.10236449982, Loss: 2.486492395401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21147 , TFLOPS: 69.89984325119943, Tokens per sec: 173219.51167986702, Loss: 2.4872851371765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21148 , TFLOPS: 67.89591135647865, Tokens per sec: 168253.54769342722, Loss: 2.4744129180908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21149 , TFLOPS: 71.5420307037488, Tokens per sec: 177289.0330891661, Loss: 2.5177934169769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21150 , TFLOPS: 69.77689325384266, Tokens per sec: 172914.82804235665, Loss: 2.491719961166382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21151 , TFLOPS: 68.09293216859895, Tokens per sec: 168741.78696949547, Loss: 2.4621407985687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21152 , TFLOPS: 70.15050995803622, Tokens per sec: 173840.69139262591, Loss: 2.497343063354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21153 , TFLOPS: 70.6084659175657, Tokens per sec: 174975.55670835346, Loss: 2.5004220008850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21154 , TFLOPS: 69.85826582016387, Tokens per sec: 173116.4782256288, Loss: 2.5122427940368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21155 , TFLOPS: 68.1119823845168, Tokens per sec: 168788.99550309443, Loss: 2.487987995147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21156 , TFLOPS: 70.65861372495154, Tokens per sec: 175099.8284992926, Loss: 2.5058228969573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21157 , TFLOPS: 70.56402754612824, Tokens per sec: 174865.43352864074, Loss: 2.4703221321105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21158 , TFLOPS: 68.26130622078101, Tokens per sec: 169159.03641873566, Loss: 2.4961657524108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21159 , TFLOPS: 68.58501630151719, Tokens per sec: 169961.22565840906, Loss: 2.4722228050231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21160 , TFLOPS: 71.47237265890429, Tokens per sec: 177116.41278057435, Loss: 2.485837936401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21161 , TFLOPS: 69.28558841584139, Tokens per sec: 171697.32053209373, Loss: 2.468235969543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21162 , TFLOPS: 69.1177553136341, Tokens per sec: 171281.4116164816, Loss: 2.4543137550354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21163 , TFLOPS: 70.63565959884296, Tokens per sec: 175042.94564618982, Loss: 2.4779605865478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21164 , TFLOPS: 68.71863215660031, Tokens per sec: 170292.34046630654, Loss: 2.4785830974578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21165 , TFLOPS: 69.81053296604385, Tokens per sec: 172998.19095491144, Loss: 2.4837868213653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21166 , TFLOPS: 69.62916746391825, Tokens per sec: 172548.74726158482, Loss: 2.4934635162353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21167 , TFLOPS: 69.94857461037925, Tokens per sec: 173340.27335611702, Loss: 2.48065447807312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21168 , TFLOPS: 68.92157499510209, Tokens per sec: 170795.25517611316, Loss: 2.46297025680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21169 , TFLOPS: 71.5519790688102, Tokens per sec: 177313.68623369114, Loss: 2.4865169525146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21170 , TFLOPS: 68.85544585240207, Tokens per sec: 170631.37987577607, Loss: 2.502133846282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21171 , TFLOPS: 69.75830439616855, Tokens per sec: 172868.76280530854, Loss: 2.47550106048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21172 , TFLOPS: 69.85808561464407, Tokens per sec: 173116.03165649943, Loss: 2.497182846069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21173 , TFLOPS: 70.59029557202115, Tokens per sec: 174930.52859046528, Loss: 2.5024378299713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21174 , TFLOPS: 68.77870367823847, Tokens per sec: 170441.20431435033, Loss: 2.4798152446746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21175 , TFLOPS: 68.83682750285178, Tokens per sec: 170585.24155460994, Loss: 2.475813865661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21176 , TFLOPS: 70.67321305981324, Tokens per sec: 175136.0072593294, Loss: 2.4849631786346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21177 , TFLOPS: 69.65601142416979, Tokens per sec: 172615.26955219448, Loss: 2.486661195755005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21178 , TFLOPS: 68.25278891230043, Tokens per sec: 169137.9295900618, Loss: 2.4999306201934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21179 , TFLOPS: 71.52747462269711, Tokens per sec: 177252.96151683535, Loss: 2.49086594581604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21180 , TFLOPS: 70.86829560817829, Tokens per sec: 175619.44330429475, Loss: 2.478494644165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21181 , TFLOPS: 67.97890574331481, Tokens per sec: 168459.2169265937, Loss: 2.4826693534851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21182 , TFLOPS: 69.62597419065219, Tokens per sec: 172540.83397866343, Loss: 2.4685068130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21183 , TFLOPS: 70.6846850299226, Tokens per sec: 175164.43606500226, Loss: 2.486550807952881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21184 , TFLOPS: 68.8582830060965, Tokens per sec: 170638.41065516917, Loss: 2.488525629043579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21185 , TFLOPS: 69.02405052695272, Tokens per sec: 171049.20083265076, Loss: 2.5042004585266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21186 , TFLOPS: 70.70866405043417, Tokens per sec: 175223.8587193371, Loss: 2.488955020904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21187 , TFLOPS: 69.87816836457934, Tokens per sec: 173165.79892313658, Loss: 2.514890193939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21188 , TFLOPS: 69.64404122122593, Tokens per sec: 172585.6060706729, Loss: 2.49497389793396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21189 , TFLOPS: 69.83426935160425, Tokens per sec: 173057.0123331088, Loss: 2.504450798034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21190 , TFLOPS: 70.61876528728872, Tokens per sec: 175001.0796810964, Loss: 2.4854981899261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21191 , TFLOPS: 69.06931050578966, Tokens per sec: 171161.36004600694, Loss: 2.4837005138397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21192 , TFLOPS: 68.91341361695086, Tokens per sec: 170775.0303820044, Loss: 2.4831998348236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21193 , TFLOPS: 70.6693196119185, Tokens per sec: 175126.35886654753, Loss: 2.501688241958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21194 , TFLOPS: 69.75945371819888, Tokens per sec: 172871.61095190793, Loss: 2.4838082790374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21195 , TFLOPS: 68.97030938086661, Tokens per sec: 170916.02435257344, Loss: 2.4897618293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21196 , TFLOPS: 70.55922217453549, Tokens per sec: 174853.52528847853, Loss: 2.5012764930725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21197 , TFLOPS: 69.90683727184259, Tokens per sec: 173236.84363347577, Loss: 2.4972572326660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21198 , TFLOPS: 68.83583620263191, Tokens per sec: 170582.78500346435, Loss: 2.5037238597869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21199 , TFLOPS: 69.84482425014693, Tokens per sec: 173083.16853441563, Loss: 2.4949564933776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21200 , TFLOPS: 70.66967681233015, Tokens per sec: 175127.2440485165, Loss: 2.4771406650543213 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/model.pt +[rank0]:[2024-08-27 22:48:35,179] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063305920048151165, 'preprocessing_with_comm': 0.0012366950104478747, 'state_converting': 1.0052183410152793, : 1.01413477800088}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200, took 4.40s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019200 + eval ppl=10.19859504699707, eval loss=2.3222498893737793 +------------------------------------------------------------------ +iteration: 21201 , TFLOPS: 64.28659099937366, Tokens per sec: 159309.254249061, Loss: 2.4993984699249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21202 , TFLOPS: 70.70455516725522, Tokens per sec: 175213.6764541888, Loss: 2.487579345703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21203 , TFLOPS: 70.04164164873936, Tokens per sec: 173570.90372935394, Loss: 2.4782135486602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21204 , TFLOPS: 68.99032388666271, Tokens per sec: 170965.62250271597, Loss: 2.489596128463745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21205 , TFLOPS: 69.75612190831414, Tokens per sec: 172863.3543599844, Loss: 2.5076065063476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21206 , TFLOPS: 70.70132010966233, Tokens per sec: 175205.65962510262, Loss: 2.5007340908050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21207 , TFLOPS: 69.50690912977758, Tokens per sec: 172245.7776417169, Loss: 2.4723081588745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21208 , TFLOPS: 69.21931673053365, Tokens per sec: 171533.09199547235, Loss: 2.5048980712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21209 , TFLOPS: 70.97951587945289, Tokens per sec: 175895.0593884375, Loss: 2.4881482124328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21210 , TFLOPS: 69.44702282590401, Tokens per sec: 172097.37278369724, Loss: 2.5005085468292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21211 , TFLOPS: 70.49888218072748, Tokens per sec: 174703.9961368232, Loss: 2.5036706924438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21212 , TFLOPS: 69.91332442642154, Tokens per sec: 173252.91951714145, Loss: 2.4876513481140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21213 , TFLOPS: 70.67756924765692, Tokens per sec: 175146.80237267978, Loss: 2.493865489959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21214 , TFLOPS: 68.79879791989772, Tokens per sec: 170491.00005874538, Loss: 2.4768311977386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21215 , TFLOPS: 69.28093051598559, Tokens per sec: 171685.77774314105, Loss: 2.492992639541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21216 , TFLOPS: 70.57239827549431, Tokens per sec: 174886.17711812182, Loss: 2.478017807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21217 , TFLOPS: 70.58884539926564, Tokens per sec: 174926.9349026274, Loss: 2.502716302871704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21218 , TFLOPS: 68.02080471731271, Tokens per sec: 168563.0471997135, Loss: 2.513896942138672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21219 , TFLOPS: 69.76822607950179, Tokens per sec: 172893.34982957193, Loss: 2.497506856918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21220 , TFLOPS: 71.51142934537691, Tokens per sec: 177213.19955210268, Loss: 2.485380172729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21221 , TFLOPS: 69.16523255585558, Tokens per sec: 171399.06545275645, Loss: 2.4937796592712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21222 , TFLOPS: 68.6736663951546, Tokens per sec: 170180.91035605653, Loss: 2.490955352783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21223 , TFLOPS: 70.1338100941899, Tokens per sec: 173799.30729037285, Loss: 2.4820117950439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21224 , TFLOPS: 68.87147642585384, Tokens per sec: 170671.10540270305, Loss: 2.486452102661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21225 , TFLOPS: 68.76733975772981, Tokens per sec: 170413.04326749066, Loss: 2.4888553619384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21226 , TFLOPS: 70.65792340001579, Tokens per sec: 175098.11779805625, Loss: 2.4873721599578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21227 , TFLOPS: 70.71398350942107, Tokens per sec: 175237.0409246934, Loss: 2.497918128967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21228 , TFLOPS: 68.959126989351, Tokens per sec: 170888.31315455562, Loss: 2.478389024734497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21229 , TFLOPS: 69.56751196959587, Tokens per sec: 172395.95815474057, Loss: 2.4840872287750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21230 , TFLOPS: 69.37416741603191, Tokens per sec: 171916.82905234824, Loss: 2.483382225036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21231 , TFLOPS: 70.5587754778568, Tokens per sec: 174852.418324901, Loss: 2.4439189434051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21232 , TFLOPS: 69.81017283664565, Tokens per sec: 172997.29851459115, Loss: 2.4788527488708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21233 , TFLOPS: 69.67124225469486, Tokens per sec: 172653.01322805005, Loss: 2.493309736251831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21234 , TFLOPS: 69.73555184035091, Tokens per sec: 172812.37946559145, Loss: 2.5005407333374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21235 , TFLOPS: 68.78554212913151, Tokens per sec: 170458.1507489808, Loss: 2.4805727005004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21236 , TFLOPS: 70.62043296130763, Tokens per sec: 175005.2123610247, Loss: 2.4905200004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21237 , TFLOPS: 69.54803074836755, Tokens per sec: 172347.68154250312, Loss: 2.509463310241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21238 , TFLOPS: 69.08276205250809, Tokens per sec: 171194.6944605854, Loss: 2.473869562149048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21239 , TFLOPS: 70.69483230499596, Tokens per sec: 175189.58213044863, Loss: 2.485414505004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21240 , TFLOPS: 69.83628772101981, Tokens per sec: 173062.01407486235, Loss: 2.4869039058685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21241 , TFLOPS: 68.80352198236524, Tokens per sec: 170502.70680593836, Loss: 2.4716532230377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21242 , TFLOPS: 69.6610809247054, Tokens per sec: 172627.8323329733, Loss: 2.503546714782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21243 , TFLOPS: 70.63651440251573, Tokens per sec: 175045.06394385535, Loss: 2.5028462409973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21244 , TFLOPS: 68.77398985764646, Tokens per sec: 170429.5229476531, Loss: 2.4822232723236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21245 , TFLOPS: 70.2430956608246, Tokens per sec: 174070.12896329263, Loss: 2.4842841625213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21246 , TFLOPS: 70.17689381489875, Tokens per sec: 173906.073496353, Loss: 2.474825620651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21247 , TFLOPS: 70.59933439171698, Tokens per sec: 174952.92778137972, Loss: 2.492560386657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21248 , TFLOPS: 68.68941651096925, Tokens per sec: 170219.94087806376, Loss: 2.487856864929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21249 , TFLOPS: 69.93641130579924, Tokens per sec: 173310.1313474691, Loss: 2.483081579208374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21250 , TFLOPS: 70.57048568169749, Tokens per sec: 174881.43750000367, Loss: 2.5082108974456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21251 , TFLOPS: 69.93951894107518, Tokens per sec: 173317.83241001717, Loss: 2.4880354404449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21252 , TFLOPS: 68.89569040249717, Tokens per sec: 170731.1103042149, Loss: 2.4796206951141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21253 , TFLOPS: 70.6696730014625, Tokens per sec: 175127.23460476668, Loss: 2.488934278488159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21254 , TFLOPS: 69.69928421748055, Tokens per sec: 172722.50430091022, Loss: 2.493154287338257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21255 , TFLOPS: 67.74355507386802, Tokens per sec: 167875.99204168806, Loss: 2.4810874462127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21256 , TFLOPS: 71.46665558304947, Tokens per sec: 177102.24523681818, Loss: 2.4774506092071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21257 , TFLOPS: 70.17815115695183, Tokens per sec: 173909.1893284682, Loss: 2.4803659915924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21258 , TFLOPS: 69.04911636728735, Tokens per sec: 171111.31674623024, Loss: 2.4750287532806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21259 , TFLOPS: 69.6169862900976, Tokens per sec: 172518.5609709902, Loss: 2.4659030437469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21260 , TFLOPS: 70.60690434663469, Tokens per sec: 174971.6869635652, Loss: 2.4926607608795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21261 , TFLOPS: 68.83269379290272, Tokens per sec: 170574.99776599044, Loss: 2.4934000968933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21262 , TFLOPS: 68.84882228608105, Tokens per sec: 170614.96594878653, Loss: 2.4865009784698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21263 , TFLOPS: 70.69243314235337, Tokens per sec: 175183.63674113073, Loss: 2.4975192546844482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21264 , TFLOPS: 70.70481286968692, Tokens per sec: 175214.31506920283, Loss: 2.4871768951416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21265 , TFLOPS: 69.0492484226769, Tokens per sec: 171111.64399403264, Loss: 2.4926600456237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21266 , TFLOPS: 69.61618795821099, Tokens per sec: 172516.58261663164, Loss: 2.511813163757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21267 , TFLOPS: 70.61135538189248, Tokens per sec: 174982.717119256, Loss: 2.484833240509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21268 , TFLOPS: 69.65852880155187, Tokens per sec: 172621.50789065927, Loss: 2.468506336212158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21269 , TFLOPS: 69.81730092878759, Tokens per sec: 173014.96271214305, Loss: 2.472161054611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21270 , TFLOPS: 69.7862972846532, Tokens per sec: 172938.13226664468, Loss: 2.488457441329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21271 , TFLOPS: 69.62976122150896, Tokens per sec: 172550.2186583024, Loss: 2.5119881629943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21272 , TFLOPS: 69.8212424176843, Tokens per sec: 173024.73015581994, Loss: 2.5114974975585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21273 , TFLOPS: 70.58066484414728, Tokens per sec: 174906.66258588157, Loss: 2.5122735500335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21274 , TFLOPS: 68.78494222602151, Tokens per sec: 170456.66412298946, Loss: 2.487306833267212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21275 , TFLOPS: 69.01898297298776, Tokens per sec: 171036.64287568844, Loss: 2.479545831680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21276 , TFLOPS: 71.54206600886431, Tokens per sec: 177289.12057913153, Loss: 2.4747695922851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21277 , TFLOPS: 68.9520814739208, Tokens per sec: 170870.85359119202, Loss: 2.499891757965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21278 , TFLOPS: 68.89568141012938, Tokens per sec: 170731.08802013693, Loss: 2.464385986328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21279 , TFLOPS: 70.60148987056236, Tokens per sec: 174958.2692954042, Loss: 2.5018980503082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21280 , TFLOPS: 70.57929841161415, Tokens per sec: 174903.2764155395, Loss: 2.496249198913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21281 , TFLOPS: 69.32073924276648, Tokens per sec: 171784.42815339734, Loss: 2.517699718475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21282 , TFLOPS: 69.39762956325642, Tokens per sec: 171974.97083773947, Loss: 2.505171537399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21283 , TFLOPS: 70.75422191929968, Tokens per sec: 175336.75613699935, Loss: 2.4618778228759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21284 , TFLOPS: 70.664656587922, Tokens per sec: 175114.8033794097, Loss: 2.458264112472534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21285 , TFLOPS: 67.87773580424968, Tokens per sec: 168208.5066728015, Loss: 2.4746973514556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21286 , TFLOPS: 69.70912866910584, Tokens per sec: 172746.89993649357, Loss: 2.4712257385253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21287 , TFLOPS: 71.55268411158514, Tokens per sec: 177315.43340735434, Loss: 2.480435371398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21288 , TFLOPS: 68.17256761152987, Tokens per sec: 168939.13236964654, Loss: 2.506408452987671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21289 , TFLOPS: 69.9283821976956, Tokens per sec: 173290.23433310902, Loss: 2.4823198318481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21290 , TFLOPS: 70.71896012248195, Tokens per sec: 175249.37352007744, Loss: 2.4958009719848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21291 , TFLOPS: 69.71491875333479, Tokens per sec: 172761.24840304334, Loss: 2.4935860633850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21292 , TFLOPS: 69.65354255850802, Tokens per sec: 172609.1514311166, Loss: 2.4584310054779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21293 , TFLOPS: 70.58506058811096, Tokens per sec: 174917.55572365478, Loss: 2.4986870288848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21294 , TFLOPS: 70.54276040845869, Tokens per sec: 174812.73121872696, Loss: 2.500359535217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21295 , TFLOPS: 69.13636996132436, Tokens per sec: 171327.5407640308, Loss: 2.469926595687866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21296 , TFLOPS: 69.78179967037309, Tokens per sec: 172926.98668300544, Loss: 2.494201898574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21297 , TFLOPS: 69.86258583158165, Tokens per sec: 173127.1836898106, Loss: 2.489819049835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21298 , TFLOPS: 69.84321725650392, Tokens per sec: 173079.1862271433, Loss: 2.4709506034851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21299 , TFLOPS: 69.06490585820822, Tokens per sec: 171150.44484408817, Loss: 2.4886083602905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21300 , TFLOPS: 70.62167930557521, Tokens per sec: 175008.30093941593, Loss: 2.498539924621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21301 , TFLOPS: 69.81334966619288, Tokens per sec: 173005.17104816582, Loss: 2.5054750442504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21302 , TFLOPS: 69.01011158328528, Tokens per sec: 171014.65859474146, Loss: 2.501946210861206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21303 , TFLOPS: 69.66086080128986, Tokens per sec: 172627.286842901, Loss: 2.4731664657592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21304 , TFLOPS: 70.66226108579666, Tokens per sec: 175108.86706125675, Loss: 2.4799890518188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21305 , TFLOPS: 68.8739126862076, Tokens per sec: 170677.14272423586, Loss: 2.4951894283294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21306 , TFLOPS: 70.67815814902316, Tokens per sec: 175148.26173513813, Loss: 2.49590802192688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21307 , TFLOPS: 69.83776385732138, Tokens per sec: 173065.67210322744, Loss: 2.4861502647399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21308 , TFLOPS: 69.01260724687194, Tokens per sec: 171020.84312402923, Loss: 2.515627384185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21309 , TFLOPS: 69.93918317336512, Tokens per sec: 173317.0003406438, Loss: 2.4952328205108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21310 , TFLOPS: 70.63008132989182, Tokens per sec: 175029.12208123185, Loss: 2.4896745681762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21311 , TFLOPS: 69.6051620561417, Tokens per sec: 172489.25921670056, Loss: 2.481069326400757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21312 , TFLOPS: 69.77746152156516, Tokens per sec: 172916.2362723729, Loss: 2.4772708415985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21313 , TFLOPS: 69.86747271870848, Tokens per sec: 173139.29393444594, Loss: 2.4946603775024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21314 , TFLOPS: 69.57008823574876, Tokens per sec: 172402.34242606655, Loss: 2.4790854454040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21315 , TFLOPS: 69.79669980170864, Tokens per sec: 172963.91084983994, Loss: 2.500577449798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21316 , TFLOPS: 70.18942476725067, Tokens per sec: 173937.1265766794, Loss: 2.4813826084136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21317 , TFLOPS: 69.88224504421653, Tokens per sec: 173175.90138436088, Loss: 2.4783425331115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21318 , TFLOPS: 68.81001098665524, Tokens per sec: 170518.7872733918, Loss: 2.502624034881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21319 , TFLOPS: 69.75041405891622, Tokens per sec: 172849.2096804048, Loss: 2.514439582824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21320 , TFLOPS: 70.56300572004113, Tokens per sec: 174862.90133103344, Loss: 2.4739058017730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21321 , TFLOPS: 69.68407424461623, Tokens per sec: 172684.8123126356, Loss: 2.490593910217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21322 , TFLOPS: 67.8186599830398, Tokens per sec: 168062.11028010445, Loss: 2.50105881690979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21323 , TFLOPS: 71.53154900721893, Tokens per sec: 177263.0582905109, Loss: 2.4798481464385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21324 , TFLOPS: 70.17526487269231, Tokens per sec: 173902.03679812266, Loss: 2.4904110431671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21325 , TFLOPS: 68.97914974337604, Tokens per sec: 170937.9317447181, Loss: 2.510702610015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21326 , TFLOPS: 69.78618185001815, Tokens per sec: 172937.84620690288, Loss: 2.4757306575775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21327 , TFLOPS: 70.64419785065971, Tokens per sec: 175064.10437475782, Loss: 2.5031659603118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21328 , TFLOPS: 68.92905589941758, Tokens per sec: 170813.79368109658, Loss: 2.4859538078308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21329 , TFLOPS: 69.41404604385845, Tokens per sec: 172015.6526850959, Loss: 2.5049123764038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21330 , TFLOPS: 70.67718232280342, Tokens per sec: 175145.84353026949, Loss: 2.4799857139587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21331 , TFLOPS: 70.63810722557855, Tokens per sec: 175049.01113486898, Loss: 2.499413013458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21332 , TFLOPS: 69.5141621448112, Tokens per sec: 172263.75141195577, Loss: 2.4842231273651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21333 , TFLOPS: 68.65188098238706, Tokens per sec: 170126.92370335726, Loss: 2.4637794494628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21334 , TFLOPS: 70.52943961890054, Tokens per sec: 174779.72083479576, Loss: 2.4819765090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21335 , TFLOPS: 69.72217188129548, Tokens per sec: 172779.22245312796, Loss: 2.4955899715423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21336 , TFLOPS: 68.77671146247023, Tokens per sec: 170436.26738421593, Loss: 2.500504493713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21337 , TFLOPS: 70.65346425784696, Tokens per sec: 175087.06755254685, Loss: 2.487078905105591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21338 , TFLOPS: 69.17599226744363, Tokens per sec: 171425.72920335172, Loss: 2.502760410308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21339 , TFLOPS: 69.87369370402843, Tokens per sec: 173154.71022136687, Loss: 2.515277624130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21340 , TFLOPS: 69.57338893072853, Tokens per sec: 172410.52191182802, Loss: 2.486069679260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21341 , TFLOPS: 69.83369605925513, Tokens per sec: 173055.59165151435, Loss: 2.5057373046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21342 , TFLOPS: 68.96578609914242, Tokens per sec: 170904.81516218028, Loss: 2.4992835521698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21343 , TFLOPS: 71.54741821480495, Tokens per sec: 177302.3839350466, Loss: 2.4909205436706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21344 , TFLOPS: 68.77510242614933, Tokens per sec: 170432.28001496277, Loss: 2.4873030185699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21345 , TFLOPS: 68.72130053524384, Tokens per sec: 170298.9530025314, Loss: 2.500944137573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21346 , TFLOPS: 70.66722285623192, Tokens per sec: 175121.1628749795, Loss: 2.4857630729675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21347 , TFLOPS: 70.64417194743947, Tokens per sec: 175064.0401837246, Loss: 2.474531888961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21348 , TFLOPS: 68.84237633031064, Tokens per sec: 170598.99216030684, Loss: 2.4977877140045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21349 , TFLOPS: 69.27781647767124, Tokens per sec: 171678.06081315843, Loss: 2.4794602394104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21350 , TFLOPS: 70.71871186706696, Tokens per sec: 175248.7583158115, Loss: 2.484639883041382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21351 , TFLOPS: 69.62624897826929, Tokens per sec: 172541.51493264895, Loss: 2.48281192779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21352 , TFLOPS: 69.764261873863, Tokens per sec: 172883.52609130568, Loss: 2.4914040565490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21353 , TFLOPS: 70.06548828398675, Tokens per sec: 173629.99831842075, Loss: 2.480553150177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21354 , TFLOPS: 70.91549053911471, Tokens per sec: 175736.39754210607, Loss: 2.4961485862731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21355 , TFLOPS: 68.20603735480813, Tokens per sec: 169022.07408048667, Loss: 2.4887499809265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21356 , TFLOPS: 69.82666739523776, Tokens per sec: 173038.17384780807, Loss: 2.4834697246551514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21357 , TFLOPS: 70.62397457910593, Tokens per sec: 175013.9888800703, Loss: 2.494720935821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21358 , TFLOPS: 68.80260461948427, Tokens per sec: 170500.43347966342, Loss: 2.504240036010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21359 , TFLOPS: 68.82096111745888, Tokens per sec: 170545.9229037797, Loss: 2.482891798019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21360 , TFLOPS: 70.70030052527555, Tokens per sec: 175203.13298267568, Loss: 2.4972755908966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21361 , TFLOPS: 70.70462977151286, Tokens per sec: 175213.86133175754, Loss: 2.4902806282043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21362 , TFLOPS: 69.04790305473591, Tokens per sec: 171108.31002407035, Loss: 2.4810216426849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21363 , TFLOPS: 70.9880788258739, Tokens per sec: 175916.27931295522, Loss: 2.5086681842803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21364 , TFLOPS: 69.73385388881402, Tokens per sec: 172808.17175463826, Loss: 2.4928293228149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21365 , TFLOPS: 69.78581082312802, Tokens per sec: 172936.92676139713, Loss: 2.48341965675354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21366 , TFLOPS: 69.16041753330785, Tokens per sec: 171387.13329646466, Loss: 2.491898775100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21367 , TFLOPS: 70.65074855468289, Tokens per sec: 175080.33774094685, Loss: 2.4759910106658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21368 , TFLOPS: 69.85057048171676, Tokens per sec: 173097.40832924534, Loss: 2.457831859588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21369 , TFLOPS: 68.65256548846115, Tokens per sec: 170128.61998481344, Loss: 2.492713689804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21370 , TFLOPS: 68.8120368999986, Tokens per sec: 170523.80770982942, Loss: 2.5085747241973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21371 , TFLOPS: 70.55923775628747, Tokens per sec: 174853.5639017764, Loss: 2.5004096031188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21372 , TFLOPS: 69.70560368833031, Tokens per sec: 172738.16464582313, Loss: 2.5229687690734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21373 , TFLOPS: 69.69571267881898, Tokens per sec: 172713.6536346694, Loss: 2.482642412185669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21374 , TFLOPS: 70.62071058642222, Tokens per sec: 175005.90034664157, Loss: 2.4714741706848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21375 , TFLOPS: 68.93361592879056, Tokens per sec: 170825.09393621155, Loss: 2.4978928565979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21376 , TFLOPS: 69.73326413175997, Tokens per sec: 172806.71027169118, Loss: 2.4956982135772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21377 , TFLOPS: 69.55418015376682, Tokens per sec: 172362.92044649547, Loss: 2.4929208755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21378 , TFLOPS: 69.76135794672875, Tokens per sec: 172876.32983997304, Loss: 2.482750654220581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21379 , TFLOPS: 70.17246027249182, Tokens per sec: 173895.08668987363, Loss: 2.4922308921813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21380 , TFLOPS: 70.21223125509852, Tokens per sec: 173993.64356590842, Loss: 2.4873030185699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21381 , TFLOPS: 69.66854890103335, Tokens per sec: 172646.33880097946, Loss: 2.4888722896575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21382 , TFLOPS: 69.65388366253512, Tokens per sec: 172609.99672447148, Loss: 2.489743947982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21383 , TFLOPS: 70.03435166028763, Tokens per sec: 173552.83833491206, Loss: 2.4759275913238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21384 , TFLOPS: 69.6661119553049, Tokens per sec: 172640.2997810126, Loss: 2.486671209335327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21385 , TFLOPS: 68.81415994156882, Tokens per sec: 170529.06884071277, Loss: 2.4734911918640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21386 , TFLOPS: 69.89898501348338, Tokens per sec: 173217.38487226397, Loss: 2.473837375640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21387 , TFLOPS: 70.71331071795845, Tokens per sec: 175235.37367333495, Loss: 2.479665994644165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21388 , TFLOPS: 68.74152003513203, Tokens per sec: 170349.0591506161, Loss: 2.5061397552490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21389 , TFLOPS: 68.90914738116714, Tokens per sec: 170764.45817976768, Loss: 2.510282278060913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21390 , TFLOPS: 71.52889711212647, Tokens per sec: 177256.48660234144, Loss: 2.490767002105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21391 , TFLOPS: 69.92086130632995, Tokens per sec: 173271.59673581037, Loss: 2.490575075149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21392 , TFLOPS: 68.73649181853231, Tokens per sec: 170336.59867597822, Loss: 2.499152660369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21393 , TFLOPS: 70.51499372358663, Tokens per sec: 174743.92231486118, Loss: 2.4918365478515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21394 , TFLOPS: 70.70073881917247, Tokens per sec: 175204.2191232343, Loss: 2.4901366233825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21395 , TFLOPS: 68.88074867428351, Tokens per sec: 170694.0830557342, Loss: 2.513676166534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21396 , TFLOPS: 68.86138513787215, Tokens per sec: 170646.09807943727, Loss: 2.457881450653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21397 , TFLOPS: 70.59107811341673, Tokens per sec: 174932.46781424756, Loss: 2.4881227016448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21398 , TFLOPS: 70.62144390363974, Tokens per sec: 175007.71758748812, Loss: 2.5105671882629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21399 , TFLOPS: 69.23532493799321, Tokens per sec: 171572.76209700678, Loss: 2.4836671352386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21400 , TFLOPS: 68.85876542573314, Tokens per sec: 170639.60614417164, Loss: 2.4917049407958984 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/model.pt +[rank0]:[2024-08-27 23:14:00,332] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063971679774113, 'preprocessing_with_comm': 0.001248078013304621, 'state_converting': 0.9845289830118418, : 0.9935313159949146}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021400, took 4.37s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019400 + eval ppl=9.812799453735352, eval loss=2.2836875915527344 +------------------------------------------------------------------ +iteration: 21401 , TFLOPS: 66.06670191677374, Tokens per sec: 163720.56519778408, Loss: 2.5002388954162598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21402 , TFLOPS: 68.86776840817961, Tokens per sec: 170661.91652643442, Loss: 2.4768290519714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21403 , TFLOPS: 69.16131327514015, Tokens per sec: 171389.35304339917, Loss: 2.49869704246521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21404 , TFLOPS: 69.70705038759861, Tokens per sec: 172741.74972597708, Loss: 2.5020782947540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21405 , TFLOPS: 69.7000837499829, Tokens per sec: 172724.48563052682, Loss: 2.4881832599639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21406 , TFLOPS: 68.76157628946221, Tokens per sec: 170398.76075822514, Loss: 2.506110429763794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21407 , TFLOPS: 70.75095555823238, Tokens per sec: 175328.6617344552, Loss: 2.4767889976501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21408 , TFLOPS: 69.88356848640825, Tokens per sec: 173179.18101990115, Loss: 2.5043632984161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21409 , TFLOPS: 69.24858988602914, Tokens per sec: 171605.63409949446, Loss: 2.477219581604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21410 , TFLOPS: 70.56524491642617, Tokens per sec: 174868.4503063434, Loss: 2.5011777877807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21411 , TFLOPS: 69.40910891613021, Tokens per sec: 172003.41793871584, Loss: 2.4716036319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21412 , TFLOPS: 68.85210977411205, Tokens per sec: 170623.11270627403, Loss: 2.497990131378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21413 , TFLOPS: 70.67549038111531, Tokens per sec: 175141.6507123836, Loss: 2.49784779548645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21414 , TFLOPS: 69.82364313603206, Tokens per sec: 173030.67940034665, Loss: 2.4809389114379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21415 , TFLOPS: 68.61815566225185, Tokens per sec: 170043.34864491152, Loss: 2.519263744354248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21416 , TFLOPS: 69.77622077654992, Tokens per sec: 172913.16156954577, Loss: 2.479438304901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21417 , TFLOPS: 70.57991052593945, Tokens per sec: 174904.79330226756, Loss: 2.4970812797546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21418 , TFLOPS: 68.61711280087223, Tokens per sec: 170040.76431953217, Loss: 2.493091344833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21419 , TFLOPS: 69.06707455378621, Tokens per sec: 171155.81911063197, Loss: 2.472616195678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21420 , TFLOPS: 70.65817613267787, Tokens per sec: 175098.7440974337, Loss: 2.4984123706817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21421 , TFLOPS: 69.63557465433401, Tokens per sec: 172564.62498524535, Loss: 2.5140912532806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21422 , TFLOPS: 68.85273295711454, Tokens per sec: 170624.6570224035, Loss: 2.50807523727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21423 , TFLOPS: 70.86839646531772, Tokens per sec: 175619.69323939658, Loss: 2.459327220916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21424 , TFLOPS: 69.7341547690263, Tokens per sec: 172808.91736894898, Loss: 2.4921154975891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21425 , TFLOPS: 69.48510846777017, Tokens per sec: 172191.75319972142, Loss: 2.500598192214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21426 , TFLOPS: 70.24315303202675, Tokens per sec: 174070.2711354515, Loss: 2.478013038635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21427 , TFLOPS: 70.69318798937921, Tokens per sec: 175185.50733521403, Loss: 2.4915003776550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21428 , TFLOPS: 69.71223492779144, Tokens per sec: 172754.59758769916, Loss: 2.4766199588775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21429 , TFLOPS: 67.8736603702285, Tokens per sec: 168198.4072983509, Loss: 2.490443468093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21430 , TFLOPS: 70.70371285077748, Tokens per sec: 175211.58910116812, Loss: 2.4758718013763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21431 , TFLOPS: 70.67607233206176, Tokens per sec: 175143.09285093608, Loss: 2.495978355407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21432 , TFLOPS: 69.07177387637606, Tokens per sec: 171167.46454968274, Loss: 2.4816341400146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21433 , TFLOPS: 70.37561763893598, Tokens per sec: 174398.53302355367, Loss: 2.5006442070007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21434 , TFLOPS: 70.56908556400984, Tokens per sec: 174877.967854138, Loss: 2.4873809814453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21435 , TFLOPS: 68.87087191434166, Tokens per sec: 170669.6073565834, Loss: 2.51153564453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21436 , TFLOPS: 68.74406755682818, Tokens per sec: 170355.37219001187, Loss: 2.5002946853637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21437 , TFLOPS: 70.65654246073724, Tokens per sec: 175094.695678401, Loss: 2.4963784217834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21438 , TFLOPS: 69.75176409679551, Tokens per sec: 172852.55522298865, Loss: 2.4902262687683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21439 , TFLOPS: 69.84664452528254, Tokens per sec: 173087.67937672156, Loss: 2.474897861480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21440 , TFLOPS: 68.81829904719105, Tokens per sec: 170539.32600040475, Loss: 2.4718568325042725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21441 , TFLOPS: 70.60290616900745, Tokens per sec: 174961.7790389693, Loss: 2.4893150329589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21442 , TFLOPS: 69.75604085523769, Tokens per sec: 172863.15350153222, Loss: 2.50885009765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21443 , TFLOPS: 70.08098992892188, Tokens per sec: 173668.41310221714, Loss: 2.4861397743225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21444 , TFLOPS: 68.79975880203808, Tokens per sec: 170493.38123053912, Loss: 2.494593381881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21445 , TFLOPS: 69.73551753302044, Tokens per sec: 172812.2944482473, Loss: 2.490215301513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21446 , TFLOPS: 69.8941079456049, Tokens per sec: 173205.29896080773, Loss: 2.4911763668060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21447 , TFLOPS: 70.56460282897417, Tokens per sec: 174866.8591429056, Loss: 2.4815359115600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21448 , TFLOPS: 69.54043115138481, Tokens per sec: 172328.84890401515, Loss: 2.4929771423339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21449 , TFLOPS: 68.43998996793036, Tokens per sec: 169601.8344278114, Loss: 2.4567484855651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21450 , TFLOPS: 71.47861804231965, Tokens per sec: 177131.8895286634, Loss: 2.4877772331237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21451 , TFLOPS: 68.91160248828464, Tokens per sec: 170770.54220565723, Loss: 2.511669874191284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21452 , TFLOPS: 68.97776724065875, Tokens per sec: 170934.50575068942, Loss: 2.501460075378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21453 , TFLOPS: 70.50205432220669, Tokens per sec: 174711.85705284894, Loss: 2.477506637573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21454 , TFLOPS: 70.52129280647812, Tokens per sec: 174759.53213616752, Loss: 2.485884428024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21455 , TFLOPS: 67.7168912053589, Tokens per sec: 167809.91603825486, Loss: 2.4968152046203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21456 , TFLOPS: 69.77456873463609, Tokens per sec: 172909.06762769585, Loss: 2.495547294616699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21457 , TFLOPS: 70.67364731947733, Tokens per sec: 175137.08340261466, Loss: 2.504030704498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21458 , TFLOPS: 69.57237715351987, Tokens per sec: 172408.0146164485, Loss: 2.4975714683532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21459 , TFLOPS: 69.2305289395504, Tokens per sec: 171560.87708454262, Loss: 2.4815263748168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21460 , TFLOPS: 69.83205352690314, Tokens per sec: 173051.52127540624, Loss: 2.4966845512390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21461 , TFLOPS: 70.59331307709702, Tokens per sec: 174938.0063004489, Loss: 2.468301296234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21462 , TFLOPS: 68.69530143665703, Tokens per sec: 170234.52437219615, Loss: 2.4645895957946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21463 , TFLOPS: 69.52900379667622, Tokens per sec: 172300.53066022028, Loss: 2.4883522987365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21464 , TFLOPS: 70.64375447009229, Tokens per sec: 175063.00562886934, Loss: 2.5046820640563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21465 , TFLOPS: 70.0922297618578, Tokens per sec: 173696.26664640213, Loss: 2.446256637573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21466 , TFLOPS: 68.35310229590938, Tokens per sec: 169386.5172050762, Loss: 2.4998483657836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21467 , TFLOPS: 70.6268125518862, Tokens per sec: 175021.02168924833, Loss: 2.502455949783325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21468 , TFLOPS: 70.6249444900551, Tokens per sec: 175016.3924262468, Loss: 2.495670795440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21469 , TFLOPS: 68.85052351364644, Tokens per sec: 170619.18177809924, Loss: 2.4503021240234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21470 , TFLOPS: 69.89996599260319, Tokens per sec: 173219.8158465866, Loss: 2.5007498264312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21471 , TFLOPS: 69.8683656808689, Tokens per sec: 173141.5067930474, Loss: 2.481706142425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21472 , TFLOPS: 69.71543725669873, Tokens per sec: 172762.53331149704, Loss: 2.481090545654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21473 , TFLOPS: 69.10051377261699, Tokens per sec: 171238.68517852869, Loss: 2.4821524620056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21474 , TFLOPS: 70.58591328289994, Tokens per sec: 174919.66879527393, Loss: 2.4873671531677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21475 , TFLOPS: 69.72933901609123, Tokens per sec: 172796.98340267685, Loss: 2.486708164215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21476 , TFLOPS: 69.29282271956319, Tokens per sec: 171715.24793364073, Loss: 2.4854071140289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21477 , TFLOPS: 69.66239630569923, Tokens per sec: 172631.09199197718, Loss: 2.5145890712738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21478 , TFLOPS: 70.70470508411495, Tokens per sec: 175214.04796468193, Loss: 2.4731075763702393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21479 , TFLOPS: 68.98531427236216, Tokens per sec: 170953.20812662464, Loss: 2.5023117065429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21480 , TFLOPS: 69.6693064545259, Tokens per sec: 172648.2161019854, Loss: 2.5091893672943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21481 , TFLOPS: 70.40353150537098, Tokens per sec: 174467.70665954583, Loss: 2.4928388595581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21482 , TFLOPS: 69.15875857622625, Tokens per sec: 171383.02221806723, Loss: 2.501922130584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21483 , TFLOPS: 69.88808203367095, Tokens per sec: 173190.3660872264, Loss: 2.5056841373443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21484 , TFLOPS: 70.66098543154408, Tokens per sec: 175105.705849494, Loss: 2.4985547065734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21485 , TFLOPS: 68.81819996026057, Tokens per sec: 170539.08045207572, Loss: 2.5138888359069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21486 , TFLOPS: 68.9278437517535, Tokens per sec: 170810.78984565593, Loss: 2.514582395553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21487 , TFLOPS: 70.59791035714338, Tokens per sec: 174949.39886683598, Loss: 2.4955596923828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21488 , TFLOPS: 69.62756638633833, Tokens per sec: 172544.7796149685, Loss: 2.4949302673339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21489 , TFLOPS: 69.06691238162622, Tokens per sec: 171155.4172301548, Loss: 2.5060243606567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21490 , TFLOPS: 70.72841383172735, Tokens per sec: 175272.80085299962, Loss: 2.4930500984191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21491 , TFLOPS: 69.87623505746456, Tokens per sec: 173161.0079751348, Loss: 2.4824085235595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21492 , TFLOPS: 68.59179822283579, Tokens per sec: 169978.03200652686, Loss: 2.4920315742492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21493 , TFLOPS: 69.78517137092862, Tokens per sec: 172935.34212840558, Loss: 2.4978129863739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21494 , TFLOPS: 70.59019845091571, Tokens per sec: 174930.2879136672, Loss: 2.4517323970794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21495 , TFLOPS: 69.53104101201882, Tokens per sec: 172305.57910425696, Loss: 2.484039068222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21496 , TFLOPS: 69.04022999128917, Tokens per sec: 171089.2953275916, Loss: 2.467421531677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21497 , TFLOPS: 70.64688712609765, Tokens per sec: 175070.7686955412, Loss: 2.487849235534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21498 , TFLOPS: 70.74253677680223, Tokens per sec: 175307.799066666, Loss: 2.499871253967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21499 , TFLOPS: 67.94513988690206, Tokens per sec: 168375.54141478674, Loss: 2.4922378063201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21500 , TFLOPS: 70.59050173070868, Tokens per sec: 174931.03947440325, Loss: 2.4925599098205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21501 , TFLOPS: 70.90033489173551, Tokens per sec: 175698.840178368, Loss: 2.4748713970184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21502 , TFLOPS: 68.82148358507641, Tokens per sec: 170547.2176360905, Loss: 2.473956346511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21503 , TFLOPS: 69.72307012235183, Tokens per sec: 172781.448393415, Loss: 2.496744394302368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21504 , TFLOPS: 70.62411773880034, Tokens per sec: 175014.34364556338, Loss: 2.4843032360076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21505 , TFLOPS: 69.84634413863617, Tokens per sec: 173086.9349855215, Loss: 2.497128486633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21506 , TFLOPS: 69.73229492561366, Tokens per sec: 172804.30847209378, Loss: 2.4944140911102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21507 , TFLOPS: 69.08020876349093, Tokens per sec: 171188.3671291329, Loss: 2.4674017429351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21508 , TFLOPS: 70.56062909457475, Tokens per sec: 174857.01179132084, Loss: 2.501957893371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21509 , TFLOPS: 69.12089782529836, Tokens per sec: 171289.19910656186, Loss: 2.486983060836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21510 , TFLOPS: 69.09023882469887, Tokens per sec: 171213.22272570865, Loss: 2.4905943870544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21511 , TFLOPS: 70.72018389334885, Tokens per sec: 175252.4061590951, Loss: 2.4876677989959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21512 , TFLOPS: 69.2809916661449, Tokens per sec: 171685.92927997204, Loss: 2.491665840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21513 , TFLOPS: 69.82484832735712, Tokens per sec: 173033.66599721293, Loss: 2.502426862716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21514 , TFLOPS: 69.57877995882967, Tokens per sec: 172423.88147333343, Loss: 2.5057430267333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21515 , TFLOPS: 70.56064706573143, Tokens per sec: 174857.0563258268, Loss: 2.4809072017669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21516 , TFLOPS: 68.43668020750057, Tokens per sec: 169593.6324768662, Loss: 2.489347457885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21517 , TFLOPS: 71.48513079553537, Tokens per sec: 177148.02884857092, Loss: 2.483290433883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21518 , TFLOPS: 68.95194225560184, Tokens per sec: 170870.50859286144, Loss: 2.4882266521453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21519 , TFLOPS: 68.93098688582064, Tokens per sec: 170818.57887811912, Loss: 2.493007183074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21520 , TFLOPS: 70.66387902609, Tokens per sec: 175112.87649553453, Loss: 2.513042449951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21521 , TFLOPS: 70.62340005182948, Tokens per sec: 175012.56513818982, Loss: 2.4880850315093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21522 , TFLOPS: 68.0222361538842, Tokens per sec: 168566.59445722337, Loss: 2.490327835083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21523 , TFLOPS: 69.87086943977846, Tokens per sec: 173147.71138343707, Loss: 2.491652727127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21524 , TFLOPS: 70.67788741271423, Tokens per sec: 175147.59082074053, Loss: 2.4952259063720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21525 , TFLOPS: 69.60634903735487, Tokens per sec: 172492.20068690355, Loss: 2.5099434852600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21526 , TFLOPS: 69.04984372276888, Tokens per sec: 171113.1192132383, Loss: 2.4936842918395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21527 , TFLOPS: 69.64441946364136, Tokens per sec: 172586.54339704514, Loss: 2.4894704818725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21528 , TFLOPS: 70.60042511163097, Tokens per sec: 174955.63070547933, Loss: 2.484638214111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21529 , TFLOPS: 69.67838126941855, Tokens per sec: 172670.7044929628, Loss: 2.47744083404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21530 , TFLOPS: 69.8813245450613, Tokens per sec: 173173.62028605287, Loss: 2.473466157913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21531 , TFLOPS: 70.67370206588718, Tokens per sec: 175137.21907024912, Loss: 2.4921984672546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21532 , TFLOPS: 68.7320907283228, Tokens per sec: 170325.69228961895, Loss: 2.4757986068725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21533 , TFLOPS: 69.02794095459805, Tokens per sec: 171058.84174092187, Loss: 2.5109026432037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21534 , TFLOPS: 70.58918921234756, Tokens per sec: 174927.78690931856, Loss: 2.50567626953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21535 , TFLOPS: 70.56323320903095, Tokens per sec: 174863.4650738102, Loss: 2.4789962768554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21536 , TFLOPS: 67.8810709118697, Tokens per sec: 168216.77143687633, Loss: 2.5070719718933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21537 , TFLOPS: 71.50712886001996, Tokens per sec: 177202.54247565058, Loss: 2.4851763248443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21538 , TFLOPS: 69.80050546443022, Tokens per sec: 172973.34170129235, Loss: 2.4895992279052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21539 , TFLOPS: 69.04914233874442, Tokens per sec: 171111.38110636183, Loss: 2.4872231483459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21540 , TFLOPS: 69.40428188808279, Tokens per sec: 171991.45603147327, Loss: 2.48897123336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21541 , TFLOPS: 70.73487451693927, Tokens per sec: 175288.81114265794, Loss: 2.4729204177856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21542 , TFLOPS: 69.76491929202626, Tokens per sec: 172885.15524593485, Loss: 2.4872257709503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21543 , TFLOPS: 69.23016682107531, Tokens per sec: 171559.97971507075, Loss: 2.463364362716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21544 , TFLOPS: 69.59432347846159, Tokens per sec: 172462.40002724133, Loss: 2.481510877609253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21545 , TFLOPS: 70.58021077214813, Tokens per sec: 174905.53734544755, Loss: 2.4970033168792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21546 , TFLOPS: 68.95750653250533, Tokens per sec: 170884.29748398403, Loss: 2.4998137950897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21547 , TFLOPS: 68.82358797411713, Tokens per sec: 170552.432543951, Loss: 2.513458728790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21548 , TFLOPS: 71.42226897817518, Tokens per sec: 176992.25034035154, Loss: 2.4898324012756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21549 , TFLOPS: 69.33825757080406, Tokens per sec: 171827.84050007764, Loss: 2.468679428100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21550 , TFLOPS: 69.90721698044734, Tokens per sec: 173237.7845932267, Loss: 2.4933571815490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21551 , TFLOPS: 69.63226172830936, Tokens per sec: 172556.41518960736, Loss: 2.4817678928375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21552 , TFLOPS: 69.59425999913486, Tokens per sec: 172462.2427184769, Loss: 2.492685317993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21553 , TFLOPS: 68.92879572285419, Tokens per sec: 170813.1489349117, Loss: 2.4950785636901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21554 , TFLOPS: 70.678335430175, Tokens per sec: 175148.70105736153, Loss: 2.473773956298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21555 , TFLOPS: 69.71468843021556, Tokens per sec: 172760.6776369817, Loss: 2.50278377532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21556 , TFLOPS: 68.93339346788805, Tokens per sec: 170824.54265358904, Loss: 2.5044960975646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21557 , TFLOPS: 70.90172664938785, Tokens per sec: 175702.28910714528, Loss: 2.4941625595092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21558 , TFLOPS: 69.66690320977254, Tokens per sec: 172642.2605967473, Loss: 2.491621971130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21559 , TFLOPS: 68.37679515604002, Tokens per sec: 169445.2307224639, Loss: 2.4914190769195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21560 , TFLOPS: 70.18970509982468, Tokens per sec: 173937.82127167901, Loss: 2.493248701095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21561 , TFLOPS: 70.74340833910954, Tokens per sec: 175309.95889407402, Loss: 2.4944326877593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21562 , TFLOPS: 69.63179865834971, Tokens per sec: 172555.26765123598, Loss: 2.479729652404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21563 , TFLOPS: 68.16776297646405, Tokens per sec: 168927.22595467916, Loss: 2.480386257171631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21564 , TFLOPS: 70.62195511288685, Tokens per sec: 175008.98442031682, Loss: 2.4689793586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21565 , TFLOPS: 70.64883942372036, Tokens per sec: 175075.6067041136, Loss: 2.518465280532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21566 , TFLOPS: 68.83104887390587, Tokens per sec: 170570.92147551343, Loss: 2.507392406463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21567 , TFLOPS: 70.69602705283512, Tokens per sec: 175192.5428472064, Loss: 2.482452869415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21568 , TFLOPS: 71.00216279199138, Tokens per sec: 175951.18093247127, Loss: 2.4989492893218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21569 , TFLOPS: 67.44850501926545, Tokens per sec: 167144.8255039362, Loss: 2.5091397762298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21570 , TFLOPS: 69.7157793709167, Tokens per sec: 172763.38110821618, Loss: 2.4888062477111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21571 , TFLOPS: 70.59442908265969, Tokens per sec: 174940.77188517136, Loss: 2.479379415512085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21572 , TFLOPS: 70.53558084248276, Tokens per sec: 174794.93946334414, Loss: 2.4828789234161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21573 , TFLOPS: 68.21588597037008, Tokens per sec: 169046.48003476372, Loss: 2.48661470413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21574 , TFLOPS: 70.05583970534335, Tokens per sec: 173606.0880776637, Loss: 2.497579336166382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21575 , TFLOPS: 70.60917614428774, Tokens per sec: 174977.31672840862, Loss: 2.491908073425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21576 , TFLOPS: 69.6671148181686, Tokens per sec: 172642.78498566334, Loss: 2.5090153217315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21577 , TFLOPS: 68.55192087335159, Tokens per sec: 169879.21154165236, Loss: 2.4871597290039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21578 , TFLOPS: 69.62001507592332, Tokens per sec: 172526.0666359146, Loss: 2.4736404418945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21579 , TFLOPS: 69.92031587270328, Tokens per sec: 173270.24509119903, Loss: 2.484349012374878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21580 , TFLOPS: 69.90606956852442, Tokens per sec: 173234.9411800838, Loss: 2.503394365310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21581 , TFLOPS: 69.68053731613458, Tokens per sec: 172676.04741423324, Loss: 2.497544765472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21582 , TFLOPS: 70.69735037242451, Tokens per sec: 175195.8221789246, Loss: 2.4903454780578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21583 , TFLOPS: 67.87228456707844, Tokens per sec: 168194.99790658779, Loss: 2.4795444011688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21584 , TFLOPS: 70.06032470944587, Tokens per sec: 173617.20241189376, Loss: 2.473534107208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21585 , TFLOPS: 69.89261336253368, Tokens per sec: 173201.59521931535, Loss: 2.490159034729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21586 , TFLOPS: 69.07619836204081, Tokens per sec: 171178.42891255728, Loss: 2.49786376953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21587 , TFLOPS: 70.65942107482643, Tokens per sec: 175101.82920121952, Loss: 2.4743456840515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21588 , TFLOPS: 69.76924493119893, Tokens per sec: 172895.8746563132, Loss: 2.5064523220062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21589 , TFLOPS: 69.00280496565536, Tokens per sec: 170996.5519913641, Loss: 2.4936389923095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21590 , TFLOPS: 69.72149397583195, Tokens per sec: 172777.54252871184, Loss: 2.4798038005828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21591 , TFLOPS: 70.59631832033472, Tokens per sec: 174945.4536242468, Loss: 2.487640619277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21592 , TFLOPS: 69.45920787617055, Tokens per sec: 172127.56868055183, Loss: 2.4978861808776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21593 , TFLOPS: 68.83218518692603, Tokens per sec: 170573.7373843526, Loss: 2.4947876930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21594 , TFLOPS: 69.77403383878628, Tokens per sec: 172907.74209685638, Loss: 2.4928410053253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21595 , TFLOPS: 70.67892307972906, Tokens per sec: 175150.15731769142, Loss: 2.4796180725097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21596 , TFLOPS: 68.71605378772814, Tokens per sec: 170285.95098421056, Loss: 2.4805314540863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21597 , TFLOPS: 69.77234785122049, Tokens per sec: 172903.56403394148, Loss: 2.502962112426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21598 , TFLOPS: 70.54168026702456, Tokens per sec: 174810.05450926578, Loss: 2.499295234680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21599 , TFLOPS: 69.06036435934433, Tokens per sec: 171139.19050961605, Loss: 2.4919114112854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21600 , TFLOPS: 69.28987248280613, Tokens per sec: 171707.93692196129, Loss: 2.4791386127471924 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/model.pt +[rank0]:[2024-08-27 23:39:26,521] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0063566679891664535, 'preprocessing_with_comm': 0.001347801007796079, 'state_converting': 0.999778786994284, : 1.008849197998643}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021600, took 4.56s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019600 + eval ppl=10.085031509399414, eval loss=2.3110523223876953 +------------------------------------------------------------------ +iteration: 21601 , TFLOPS: 65.42627463497942, Tokens per sec: 162133.5158446087, Loss: 2.47206449508667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21602 , TFLOPS: 70.63634172313645, Tokens per sec: 175044.6360253313, Loss: 2.4804301261901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21603 , TFLOPS: 68.41811326725208, Tokens per sec: 169547.6215536131, Loss: 2.505305290222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21604 , TFLOPS: 69.78810213718238, Tokens per sec: 172942.6048900329, Loss: 2.4999237060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21605 , TFLOPS: 70.59179440015663, Tokens per sec: 174934.24285169455, Loss: 2.4855971336364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21606 , TFLOPS: 69.0336846958304, Tokens per sec: 171073.07536442098, Loss: 2.497725486755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21607 , TFLOPS: 69.4310903772141, Tokens per sec: 172057.89042073957, Loss: 2.486851215362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21608 , TFLOPS: 70.62075208164673, Tokens per sec: 175006.003176379, Loss: 2.4880616664886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21609 , TFLOPS: 69.03331605539195, Tokens per sec: 171072.1618328054, Loss: 2.506635904312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21610 , TFLOPS: 68.75814245755225, Tokens per sec: 170390.25134448442, Loss: 2.5053224563598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21611 , TFLOPS: 70.64601981648259, Tokens per sec: 175068.61940677304, Loss: 2.479468822479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21612 , TFLOPS: 70.57279839975484, Tokens per sec: 174887.16867011675, Loss: 2.5039844512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21613 , TFLOPS: 69.72016538163321, Tokens per sec: 172774.25012593064, Loss: 2.5165882110595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21614 , TFLOPS: 69.75698139285029, Tokens per sec: 172865.484257345, Loss: 2.4970860481262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21615 , TFLOPS: 69.98874102309506, Tokens per sec: 173439.8101515224, Loss: 2.4840588569641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21616 , TFLOPS: 69.85187490061416, Tokens per sec: 173100.6408229667, Loss: 2.4924733638763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21617 , TFLOPS: 70.08591964780781, Tokens per sec: 173680.62948866986, Loss: 2.4915454387664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21618 , TFLOPS: 69.55161805850992, Tokens per sec: 172356.57129221075, Loss: 2.4834811687469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21619 , TFLOPS: 69.57545785980335, Tokens per sec: 172415.64894598676, Loss: 2.4699463844299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21620 , TFLOPS: 68.96981219858492, Tokens per sec: 170914.79228011024, Loss: 2.4774818420410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21621 , TFLOPS: 69.472273998414, Tokens per sec: 172159.94797082327, Loss: 2.4855735301971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21622 , TFLOPS: 70.6815178817318, Tokens per sec: 175156.58752289528, Loss: 2.48656964302063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21623 , TFLOPS: 69.17004966758272, Tokens per sec: 171411.00278626496, Loss: 2.484781503677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21624 , TFLOPS: 69.96576581333588, Tokens per sec: 173382.87505081086, Loss: 2.47393536567688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21625 , TFLOPS: 69.52192018842882, Tokens per sec: 172282.97669865933, Loss: 2.48285174369812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21626 , TFLOPS: 69.02750927838348, Tokens per sec: 171057.7719997091, Loss: 2.4995594024658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21627 , TFLOPS: 70.54242597646574, Tokens per sec: 174811.9024594077, Loss: 2.4835808277130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21628 , TFLOPS: 70.57801993628796, Tokens per sec: 174900.10821284555, Loss: 2.481053590774536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21629 , TFLOPS: 69.00930014236543, Tokens per sec: 171012.64775475446, Loss: 2.504092216491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21630 , TFLOPS: 70.00383442992306, Tokens per sec: 173477.21327631822, Loss: 2.4748997688293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21631 , TFLOPS: 70.13766994656139, Tokens per sec: 173808.87242974722, Loss: 2.506854295730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21632 , TFLOPS: 69.52524817793277, Tokens per sec: 172291.2238232584, Loss: 2.4939095973968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21633 , TFLOPS: 69.77572000900152, Tokens per sec: 172911.92061239673, Loss: 2.464395046234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21634 , TFLOPS: 70.29075802444228, Tokens per sec: 174188.24155078005, Loss: 2.5074949264526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21635 , TFLOPS: 69.71458858114946, Tokens per sec: 172760.4301999966, Loss: 2.481090545654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21636 , TFLOPS: 69.53564021759075, Tokens per sec: 172316.97644230857, Loss: 2.4734575748443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21637 , TFLOPS: 69.79823807072273, Tokens per sec: 172967.72284991117, Loss: 2.5028254985809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21638 , TFLOPS: 70.66284751561797, Tokens per sec: 175110.32029895458, Loss: 2.4787333011627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21639 , TFLOPS: 70.55598753004848, Tokens per sec: 174845.5094831142, Loss: 2.4792909622192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21640 , TFLOPS: 68.17636572537344, Tokens per sec: 168948.5445141477, Loss: 2.475550889968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21641 , TFLOPS: 70.51404294634317, Tokens per sec: 174741.56618411507, Loss: 2.492518186569214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21642 , TFLOPS: 70.65629372894072, Tokens per sec: 175094.07929360916, Loss: 2.509350299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21643 , TFLOPS: 68.7641151972297, Tokens per sec: 170405.05245135672, Loss: 2.4937922954559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21644 , TFLOPS: 68.60971569444399, Tokens per sec: 170022.4334749444, Loss: 2.501896858215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21645 , TFLOPS: 71.49024895268893, Tokens per sec: 177160.712205809, Loss: 2.481362819671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21646 , TFLOPS: 69.2331590539687, Tokens per sec: 171567.39479780223, Loss: 2.4948902130126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21647 , TFLOPS: 69.89267860510864, Tokens per sec: 173201.75689760313, Loss: 2.4897818565368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21648 , TFLOPS: 70.56151983200647, Tokens per sec: 174859.21913679945, Loss: 2.4823455810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21649 , TFLOPS: 69.77478972445364, Tokens per sec: 172909.61526480777, Loss: 2.4904584884643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21650 , TFLOPS: 69.7416153719468, Tokens per sec: 172827.40556483925, Loss: 2.5040531158447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21651 , TFLOPS: 68.68023082012547, Tokens per sec: 170197.1777242655, Loss: 2.5030312538146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21652 , TFLOPS: 70.61745051233308, Tokens per sec: 174997.82152392188, Loss: 2.48942494392395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21653 , TFLOPS: 69.61044512352407, Tokens per sec: 172502.351239652, Loss: 2.5228514671325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21654 , TFLOPS: 70.6672279971563, Tokens per sec: 175121.17561475642, Loss: 2.5005226135253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21655 , TFLOPS: 68.62931685756047, Tokens per sec: 170071.00731639343, Loss: 2.479130744934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21656 , TFLOPS: 69.71942434948137, Tokens per sec: 172772.41376662697, Loss: 2.5042684078216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21657 , TFLOPS: 69.68686418375535, Tokens per sec: 172691.72608915952, Loss: 2.489419937133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21658 , TFLOPS: 70.71342277966468, Tokens per sec: 175235.65137458756, Loss: 2.489367723464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21659 , TFLOPS: 69.52957843409041, Tokens per sec: 172301.95467503413, Loss: 2.4693500995635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21660 , TFLOPS: 68.25326953711172, Tokens per sec: 169139.12063128932, Loss: 2.510150671005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21661 , TFLOPS: 70.58243236358736, Tokens per sec: 174911.04269376243, Loss: 2.49574613571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21662 , TFLOPS: 69.5989899820608, Tokens per sec: 172473.964137218, Loss: 2.489180088043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21663 , TFLOPS: 68.97898047971213, Tokens per sec: 170937.51229071303, Loss: 2.495265007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21664 , TFLOPS: 70.58746589772755, Tokens per sec: 174923.51634585139, Loss: 2.49511456489563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21665 , TFLOPS: 71.00056721054939, Tokens per sec: 175947.22690589022, Loss: 2.484347343444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21666 , TFLOPS: 68.6021529679782, Tokens per sec: 170003.69220565908, Loss: 2.5144736766815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21667 , TFLOPS: 69.19322149198887, Tokens per sec: 171468.42511973056, Loss: 2.4873135089874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21668 , TFLOPS: 70.63639434780873, Tokens per sec: 175044.76643506627, Loss: 2.4758856296539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21669 , TFLOPS: 68.71329154397623, Tokens per sec: 170279.10583990702, Loss: 2.483121395111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21670 , TFLOPS: 69.93284883647965, Tokens per sec: 173301.3031560568, Loss: 2.488413095474243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21671 , TFLOPS: 69.91234792854952, Tokens per sec: 173250.4996478445, Loss: 2.465045928955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21672 , TFLOPS: 70.5891264465125, Tokens per sec: 174927.63136866505, Loss: 2.4886515140533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21673 , TFLOPS: 68.73392766272946, Tokens per sec: 170330.24441543367, Loss: 2.4949913024902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21674 , TFLOPS: 70.53940806451399, Tokens per sec: 174804.42374114084, Loss: 2.4752659797668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21675 , TFLOPS: 70.65259032804745, Tokens per sec: 175084.90185823236, Loss: 2.4965062141418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21676 , TFLOPS: 69.08180832919503, Tokens per sec: 171192.33102915436, Loss: 2.475700616836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21677 , TFLOPS: 68.86867904414902, Tokens per sec: 170664.17318268144, Loss: 2.4743876457214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21678 , TFLOPS: 70.72341382806769, Tokens per sec: 175260.41029313506, Loss: 2.4970879554748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21679 , TFLOPS: 70.6746088118267, Tokens per sec: 175139.46608657355, Loss: 2.4871749877929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21680 , TFLOPS: 68.95988250475214, Tokens per sec: 170890.18540494656, Loss: 2.49995493888855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21681 , TFLOPS: 69.90324106216461, Tokens per sec: 173227.93182973855, Loss: 2.4797723293304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21682 , TFLOPS: 69.98423353139866, Tokens per sec: 173428.6400905575, Loss: 2.475567102432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21683 , TFLOPS: 69.70661815108517, Tokens per sec: 172740.67859628194, Loss: 2.489177942276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21684 , TFLOPS: 69.75527324391824, Tokens per sec: 172861.25127612325, Loss: 2.494560718536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21685 , TFLOPS: 70.6950672148057, Tokens per sec: 175190.16426283465, Loss: 2.4987967014312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21686 , TFLOPS: 69.7313959144021, Tokens per sec: 172802.08062327723, Loss: 2.46419620513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21687 , TFLOPS: 69.14951966826024, Tokens per sec: 171360.1271863777, Loss: 2.4824576377868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21688 , TFLOPS: 69.62960190906979, Tokens per sec: 172549.82386452841, Loss: 2.4905076026916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21689 , TFLOPS: 70.61989797935784, Tokens per sec: 175003.88661682, Loss: 2.4852495193481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21690 , TFLOPS: 69.28636243595189, Tokens per sec: 171699.23863919292, Loss: 2.479008674621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21691 , TFLOPS: 69.71659572355684, Tokens per sec: 172765.4041199871, Loss: 2.4854516983032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21692 , TFLOPS: 69.64800403527434, Tokens per sec: 172595.42636042484, Loss: 2.52287220954895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21693 , TFLOPS: 69.69774070051982, Tokens per sec: 172718.6792958491, Loss: 2.4965786933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21694 , TFLOPS: 70.6407061663755, Tokens per sec: 175055.45159646077, Loss: 2.4895853996276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21695 , TFLOPS: 69.53649440530002, Tokens per sec: 172319.09321354865, Loss: 2.4740006923675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21696 , TFLOPS: 69.6553194000469, Tokens per sec: 172613.5546401852, Loss: 2.477921485900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21697 , TFLOPS: 70.1344663848586, Tokens per sec: 173800.93365094627, Loss: 2.491718292236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21698 , TFLOPS: 69.96476856331859, Tokens per sec: 173380.40375541185, Loss: 2.4696409702301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21699 , TFLOPS: 69.70123554008791, Tokens per sec: 172727.33989328722, Loss: 2.475463390350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21700 , TFLOPS: 68.91518614203848, Tokens per sec: 170779.42289443125, Loss: 2.4888527393341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21701 , TFLOPS: 70.6576429740893, Tokens per sec: 175097.42287171897, Loss: 2.4827966690063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21702 , TFLOPS: 69.75807351076057, Tokens per sec: 172868.1906458334, Loss: 2.484807252883911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21703 , TFLOPS: 69.43973168179316, Tokens per sec: 172079.30452540473, Loss: 2.494731903076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21704 , TFLOPS: 69.69152318098752, Tokens per sec: 172703.27159753174, Loss: 2.499847173690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21705 , TFLOPS: 70.69077906553356, Tokens per sec: 175179.53775655964, Loss: 2.488657236099243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21706 , TFLOPS: 69.66198383438369, Tokens per sec: 172630.06984261982, Loss: 2.4842753410339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21707 , TFLOPS: 68.41806932469501, Tokens per sec: 169547.51265911604, Loss: 2.4986157417297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21708 , TFLOPS: 70.5606649427472, Tokens per sec: 174857.10062704113, Loss: 2.497908115386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21709 , TFLOPS: 70.59359915836933, Tokens per sec: 174938.71524135614, Loss: 2.4802582263946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21710 , TFLOPS: 68.05281201103469, Tokens per sec: 168642.36479945105, Loss: 2.495821714401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21711 , TFLOPS: 69.89219720102834, Tokens per sec: 173200.5639252611, Loss: 2.469351291656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21712 , TFLOPS: 71.52605530168172, Tokens per sec: 177249.4442830081, Loss: 2.481627941131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21713 , TFLOPS: 69.99104348030951, Tokens per sec: 173445.51589413642, Loss: 2.47800874710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21714 , TFLOPS: 69.02843173961749, Tokens per sec: 171060.0579602645, Loss: 2.5071089267730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21715 , TFLOPS: 70.63727070405561, Tokens per sec: 175046.93814238466, Loss: 2.471160411834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21716 , TFLOPS: 69.64111348219525, Tokens per sec: 172578.35081083776, Loss: 2.485045909881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21717 , TFLOPS: 69.84517827800975, Tokens per sec: 173084.0458544591, Loss: 2.4871206283569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21718 , TFLOPS: 68.81303948175437, Tokens per sec: 170526.29221786372, Loss: 2.4770195484161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21719 , TFLOPS: 70.58709632441189, Tokens per sec: 174922.60050246332, Loss: 2.484201431274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21720 , TFLOPS: 68.8858593789709, Tokens per sec: 170706.7479449402, Loss: 2.4916858673095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21721 , TFLOPS: 69.51874150006634, Tokens per sec: 172275.09955873567, Loss: 2.5258686542510986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21722 , TFLOPS: 69.56402896359911, Tokens per sec: 172387.32688219583, Loss: 2.4981391429901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21723 , TFLOPS: 69.81250016119843, Tokens per sec: 173003.06588120892, Loss: 2.48443865776062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21724 , TFLOPS: 69.87661656408066, Tokens per sec: 173161.95339055595, Loss: 2.4700024127960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21725 , TFLOPS: 69.65868240825876, Tokens per sec: 172621.8885450001, Loss: 2.4792065620422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21726 , TFLOPS: 70.65949382019191, Tokens per sec: 175102.00947224876, Loss: 2.4956207275390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21727 , TFLOPS: 68.92694533398047, Tokens per sec: 170808.56346744546, Loss: 2.5044662952423096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21728 , TFLOPS: 69.92925884526595, Tokens per sec: 173292.40676235908, Loss: 2.5095088481903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21729 , TFLOPS: 69.45206475663024, Tokens per sec: 172109.86724345133, Loss: 2.499593734741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21730 , TFLOPS: 69.18057805003674, Tokens per sec: 171437.09327778307, Loss: 2.480929136276245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21731 , TFLOPS: 70.6316578248236, Tokens per sec: 175033.02880933805, Loss: 2.496870756149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21732 , TFLOPS: 70.84798205405673, Tokens per sec: 175569.1040794588, Loss: 2.470675468444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21733 , TFLOPS: 68.79226707937711, Tokens per sec: 170474.81591650384, Loss: 2.4875307083129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21734 , TFLOPS: 69.69749775563189, Tokens per sec: 172718.07725165423, Loss: 2.500509023666382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21735 , TFLOPS: 70.68933083940445, Tokens per sec: 175175.94889267642, Loss: 2.488053321838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21736 , TFLOPS: 69.51262748251399, Tokens per sec: 172259.9483497263, Loss: 2.4799578189849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21737 , TFLOPS: 69.30431574060592, Tokens per sec: 171743.72890584538, Loss: 2.4923806190490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21738 , TFLOPS: 69.64368851489527, Tokens per sec: 172584.73202553167, Loss: 2.4817795753479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21739 , TFLOPS: 70.59543369837668, Tokens per sec: 174943.26143358566, Loss: 2.5003228187561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21740 , TFLOPS: 67.9346383382452, Tokens per sec: 168349.51742037394, Loss: 2.492161750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21741 , TFLOPS: 70.62027335986271, Tokens per sec: 175004.8168510626, Loss: 2.4999325275421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21742 , TFLOPS: 69.76614764670998, Tokens per sec: 172888.19924415552, Loss: 2.5024704933166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21743 , TFLOPS: 69.63407559306123, Tokens per sec: 172560.91014627618, Loss: 2.48573637008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21744 , TFLOPS: 69.36165897330399, Tokens per sec: 171885.83175334978, Loss: 2.488422155380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21745 , TFLOPS: 70.6521562895235, Tokens per sec: 175083.82626295672, Loss: 2.495143175125122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21746 , TFLOPS: 70.64839561722792, Tokens per sec: 175074.506902736, Loss: 2.477863073348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21747 , TFLOPS: 67.8820540859976, Tokens per sec: 168219.20785067082, Loss: 2.486783742904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21748 , TFLOPS: 70.63302649312065, Tokens per sec: 175036.4205201494, Loss: 2.48337984085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21749 , TFLOPS: 70.09748515115955, Tokens per sec: 173709.29008002082, Loss: 2.4757978916168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21750 , TFLOPS: 69.86617180146757, Tokens per sec: 173136.0701182151, Loss: 2.488713264465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21751 , TFLOPS: 68.93604299779808, Tokens per sec: 170831.1084805759, Loss: 2.4958043098449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21752 , TFLOPS: 70.6141659974652, Tokens per sec: 174989.68213426007, Loss: 2.4779889583587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21753 , TFLOPS: 69.6820395711158, Tokens per sec: 172679.77016756375, Loss: 2.488564968109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21754 , TFLOPS: 69.74631390660707, Tokens per sec: 172839.0490513137, Loss: 2.4965312480926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21755 , TFLOPS: 69.72214950513211, Tokens per sec: 172779.16700253016, Loss: 2.497997760772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21756 , TFLOPS: 70.47803089641717, Tokens per sec: 174652.3243573437, Loss: 2.50575590133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21757 , TFLOPS: 68.9256401003821, Tokens per sec: 170805.32895480533, Loss: 2.487977981567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21758 , TFLOPS: 68.76358293671676, Tokens per sec: 170403.7334511723, Loss: 2.4951984882354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21759 , TFLOPS: 70.61411547788231, Tokens per sec: 174989.5569411685, Loss: 2.4984750747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21760 , TFLOPS: 69.12531630568336, Tokens per sec: 171300.14858769168, Loss: 2.498819351196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21761 , TFLOPS: 70.73226538241424, Tokens per sec: 175282.34541988504, Loss: 2.503274440765381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21762 , TFLOPS: 69.630841291472, Tokens per sec: 172552.89519065034, Loss: 2.487717866897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21763 , TFLOPS: 69.90243075251976, Tokens per sec: 173225.92379317564, Loss: 2.484072208404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21764 , TFLOPS: 69.69785988894078, Tokens per sec: 172718.97465788596, Loss: 2.4847424030303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21765 , TFLOPS: 70.00865749469212, Tokens per sec: 173489.16536212037, Loss: 2.474672317504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21766 , TFLOPS: 69.58032324076045, Tokens per sec: 172427.70589596443, Loss: 2.5031423568725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21767 , TFLOPS: 69.08749705609152, Tokens per sec: 171206.42832106887, Loss: 2.476943254470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21768 , TFLOPS: 70.59941469562284, Tokens per sec: 174953.12678330467, Loss: 2.4864730834960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21769 , TFLOPS: 68.37952002529218, Tokens per sec: 169451.9832486399, Loss: 2.4903833866119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21770 , TFLOPS: 69.82672740345781, Tokens per sec: 173038.32255478777, Loss: 2.498164176940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21771 , TFLOPS: 69.64972885176995, Tokens per sec: 172599.70064570516, Loss: 2.4861032962799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21772 , TFLOPS: 70.56632582699136, Tokens per sec: 174871.12892179604, Loss: 2.4897801876068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21773 , TFLOPS: 69.66404266361337, Tokens per sec: 172635.17184824974, Loss: 2.490734577178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21774 , TFLOPS: 69.18493104127388, Tokens per sec: 171447.88046959025, Loss: 2.5005366802215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21775 , TFLOPS: 70.76447564886482, Tokens per sec: 175362.16600840332, Loss: 2.4958102703094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21776 , TFLOPS: 70.56681991053965, Tokens per sec: 174872.35331525654, Loss: 2.4885573387145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21777 , TFLOPS: 68.72101886665611, Tokens per sec: 170298.25499674262, Loss: 2.4860403537750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21778 , TFLOPS: 70.6933575583795, Tokens per sec: 175185.92754587633, Loss: 2.4835329055786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21779 , TFLOPS: 70.87859133049618, Tokens per sec: 175644.95723836595, Loss: 2.484616756439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21780 , TFLOPS: 68.12135836910012, Tokens per sec: 168812.23022574186, Loss: 2.4859471321105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21781 , TFLOPS: 70.11592899889288, Tokens per sec: 173754.99596646157, Loss: 2.49757719039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21782 , TFLOPS: 70.62131381068897, Tokens per sec: 175007.39520282522, Loss: 2.5026650428771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21783 , TFLOPS: 69.74011797904544, Tokens per sec: 172823.6948602782, Loss: 2.5042967796325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21784 , TFLOPS: 68.83567271612316, Tokens per sec: 170582.37986588615, Loss: 2.4972007274627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21785 , TFLOPS: 69.79239774211315, Tokens per sec: 172953.24987225133, Loss: 2.491434097290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21786 , TFLOPS: 70.56406507154233, Tokens per sec: 174865.52652075063, Loss: 2.495696783065796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21787 , TFLOPS: 68.99239869594011, Tokens per sec: 170970.76410866447, Loss: 2.4732398986816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21788 , TFLOPS: 69.47516163858866, Tokens per sec: 172167.10386127591, Loss: 2.4636902809143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21789 , TFLOPS: 69.86759935736252, Tokens per sec: 173139.60775898106, Loss: 2.4689226150512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21790 , TFLOPS: 69.10522693737403, Tokens per sec: 171250.3649199946, Loss: 2.4949543476104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21791 , TFLOPS: 69.83135943674817, Tokens per sec: 173049.80124354188, Loss: 2.491262674331665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21792 , TFLOPS: 69.62648018256486, Tokens per sec: 172542.08788236266, Loss: 2.4805710315704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21793 , TFLOPS: 70.60341610287205, Tokens per sec: 174963.0427112596, Loss: 2.4986765384674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21794 , TFLOPS: 68.99077110061474, Tokens per sec: 170966.73074815382, Loss: 2.467036247253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21795 , TFLOPS: 69.57933373831307, Tokens per sec: 172425.25379989712, Loss: 2.475294351577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21796 , TFLOPS: 70.61004353834933, Tokens per sec: 174979.46622644458, Loss: 2.4970147609710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21797 , TFLOPS: 68.75306689952131, Tokens per sec: 170377.6735525649, Loss: 2.4698853492736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21798 , TFLOPS: 70.7166437375459, Tokens per sec: 175243.63326303504, Loss: 2.478644847869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21799 , TFLOPS: 69.61993185772045, Tokens per sec: 172525.8604120406, Loss: 2.491849660873413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21800 , TFLOPS: 69.64333873458143, Tokens per sec: 172583.86523138252, Loss: 2.501675605773926 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/model.pt +[rank0]:[2024-08-28 00:04:52,011] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006518975016660988, 'preprocessing_with_comm': 0.0014623120077885687, 'state_converting': 0.9951841680158395, : 1.0045425330172293}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021800, took 4.46s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0019800 + eval ppl=9.021954536437988, eval loss=2.1996610164642334 +------------------------------------------------------------------ +iteration: 21801 , TFLOPS: 64.28525544698996, Tokens per sec: 159305.94460313098, Loss: 2.4741969108581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21802 , TFLOPS: 70.66876392950635, Tokens per sec: 175124.9818243168, Loss: 2.486595869064331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21803 , TFLOPS: 69.21632420127668, Tokens per sec: 171525.67617831926, Loss: 2.504272222518921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21804 , TFLOPS: 69.60298504107917, Tokens per sec: 172483.86433355758, Loss: 2.49062442779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21805 , TFLOPS: 69.74506430966349, Tokens per sec: 172835.95241243302, Loss: 2.493225574493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21806 , TFLOPS: 69.46371955035373, Tokens per sec: 172138.7491061797, Loss: 2.4760658740997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21807 , TFLOPS: 68.9062216794884, Tokens per sec: 170757.2079687151, Loss: 2.5104188919067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21808 , TFLOPS: 70.68539302520833, Tokens per sec: 175166.19055531238, Loss: 2.4620413780212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21809 , TFLOPS: 69.75573155139074, Tokens per sec: 172862.38701252683, Loss: 2.493913173675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21810 , TFLOPS: 69.48895737657162, Tokens per sec: 172201.29121972367, Loss: 2.4707491397857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21811 , TFLOPS: 69.01067280934014, Tokens per sec: 171016.04937472954, Loss: 2.5055816173553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21812 , TFLOPS: 70.67764762592611, Tokens per sec: 175146.9966026649, Loss: 2.4792282581329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21813 , TFLOPS: 70.51466456067948, Tokens per sec: 174743.10661291695, Loss: 2.460853338241577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21814 , TFLOPS: 67.05095589716063, Tokens per sec: 166159.65498570944, Loss: 2.5023322105407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21815 , TFLOPS: 71.4401458081083, Tokens per sec: 177036.55109422118, Loss: 2.486565589904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21816 , TFLOPS: 69.75570110270671, Tokens per sec: 172862.31155733363, Loss: 2.50296688079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21817 , TFLOPS: 69.0062960088874, Tokens per sec: 171005.20318106326, Loss: 2.4861693382263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21818 , TFLOPS: 69.56203307297957, Tokens per sec: 172382.38084537507, Loss: 2.468366861343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21819 , TFLOPS: 70.60670001321468, Tokens per sec: 174971.18060284122, Loss: 2.5063047409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21820 , TFLOPS: 69.75792778331702, Tokens per sec: 172867.82951717507, Loss: 2.467329740524292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21821 , TFLOPS: 68.16700525107048, Tokens per sec: 168925.34822768354, Loss: 2.483341932296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21822 , TFLOPS: 70.5963427202741, Tokens per sec: 174945.51408998447, Loss: 2.4756035804748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21823 , TFLOPS: 69.73361878395657, Tokens per sec: 172807.58913890255, Loss: 2.493149757385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21824 , TFLOPS: 69.85839596501485, Tokens per sec: 173116.80073890617, Loss: 2.4974467754364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21825 , TFLOPS: 69.44629872931046, Tokens per sec: 172095.5783925726, Loss: 2.4855918884277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21826 , TFLOPS: 70.00157583551264, Tokens per sec: 173471.61623056448, Loss: 2.505324602127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21827 , TFLOPS: 69.81648774918126, Tokens per sec: 173012.94756349945, Loss: 2.4799249172210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21828 , TFLOPS: 70.18536086837518, Tokens per sec: 173927.05578759057, Loss: 2.502241373062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21829 , TFLOPS: 69.42323438528395, Tokens per sec: 172038.4224073277, Loss: 2.482156753540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21830 , TFLOPS: 69.91362944220283, Tokens per sec: 173253.6753798478, Loss: 2.4835290908813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21831 , TFLOPS: 68.92629979993157, Tokens per sec: 170806.96376296083, Loss: 2.4967641830444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21832 , TFLOPS: 70.74823694114234, Tokens per sec: 175321.9247018257, Loss: 2.500941038131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21833 , TFLOPS: 69.39196993275768, Tokens per sec: 171960.94564990382, Loss: 2.480630397796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21834 , TFLOPS: 68.95283349448982, Tokens per sec: 170872.7171810039, Loss: 2.4950294494628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21835 , TFLOPS: 70.83993487204707, Tokens per sec: 175549.16227596859, Loss: 2.486804246902466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21836 , TFLOPS: 69.52142769263685, Tokens per sec: 172281.75623983392, Loss: 2.5081844329833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21837 , TFLOPS: 69.75275999864499, Tokens per sec: 172855.0231774793, Loss: 2.495857000350952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21838 , TFLOPS: 70.13633497686988, Tokens per sec: 173805.5642277926, Loss: 2.4902660846710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21839 , TFLOPS: 70.66737770854841, Tokens per sec: 175121.54661607815, Loss: 2.4847800731658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21840 , TFLOPS: 68.57957247821456, Tokens per sec: 169947.7352645785, Loss: 2.493368148803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21841 , TFLOPS: 69.01777406672616, Tokens per sec: 171033.64707280017, Loss: 2.5050747394561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21842 , TFLOPS: 70.62395095162931, Tokens per sec: 175013.93032858043, Loss: 2.481560707092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21843 , TFLOPS: 70.62617998907922, Tokens per sec: 175019.45412893005, Loss: 2.499396324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21844 , TFLOPS: 68.53772573557067, Tokens per sec: 169844.0344265077, Loss: 2.5064616203308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21845 , TFLOPS: 70.68313140997549, Tokens per sec: 175160.5860236279, Loss: 2.4785964488983154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21846 , TFLOPS: 69.76004979164894, Tokens per sec: 172873.08808758008, Loss: 2.496394157409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21847 , TFLOPS: 69.83081677131443, Tokens per sec: 173048.45645881756, Loss: 2.4734206199645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21848 , TFLOPS: 68.66328642780738, Tokens per sec: 170155.18765352236, Loss: 2.4688146114349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21849 , TFLOPS: 70.57545493994549, Tokens per sec: 174893.75186935122, Loss: 2.4803102016448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21850 , TFLOPS: 69.65886567402508, Tokens per sec: 172622.34269775744, Loss: 2.498765230178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21851 , TFLOPS: 69.4701940272778, Tokens per sec: 172154.79357322003, Loss: 2.4792566299438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21852 , TFLOPS: 70.12732181784787, Tokens per sec: 173783.22862685466, Loss: 2.4872002601623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21853 , TFLOPS: 70.5642548347744, Tokens per sec: 174865.99677494384, Loss: 2.4996111392974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21854 , TFLOPS: 69.06524781624165, Tokens per sec: 171151.29225376478, Loss: 2.4905271530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21855 , TFLOPS: 69.54826284006592, Tokens per sec: 172348.25669129868, Loss: 2.4820284843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21856 , TFLOPS: 69.7857700735914, Tokens per sec: 172936.82577955647, Loss: 2.4988317489624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21857 , TFLOPS: 69.81523819557572, Tokens per sec: 173009.8510320152, Loss: 2.5010223388671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21858 , TFLOPS: 68.35877572419166, Tokens per sec: 169400.57658533938, Loss: 2.464473009109497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21859 , TFLOPS: 70.69648636407119, Tokens per sec: 175193.6810710467, Loss: 2.4875648021698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21860 , TFLOPS: 70.61295601704872, Tokens per sec: 174986.6836694977, Loss: 2.491891860961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21861 , TFLOPS: 68.38826817130048, Tokens per sec: 169473.66211813633, Loss: 2.491250991821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21862 , TFLOPS: 70.37555199666876, Tokens per sec: 174398.3703547844, Loss: 2.5142486095428467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21863 , TFLOPS: 70.03036366754999, Tokens per sec: 173542.9556495945, Loss: 2.4955198764801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21864 , TFLOPS: 69.0154313940337, Tokens per sec: 171027.84167180155, Loss: 2.4787769317626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21865 , TFLOPS: 70.60328500298395, Tokens per sec: 174962.71783129507, Loss: 2.4947948455810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21866 , TFLOPS: 70.32570771268581, Tokens per sec: 174274.85072827348, Loss: 2.4911880493164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21867 , TFLOPS: 70.01767833428738, Tokens per sec: 173511.51999636495, Loss: 2.4886837005615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21868 , TFLOPS: 69.28574464401954, Tokens per sec: 171697.70768272917, Loss: 2.487187385559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21869 , TFLOPS: 70.70519110794831, Tokens per sec: 175215.2523852809, Loss: 2.466032028198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21870 , TFLOPS: 68.33592706010808, Tokens per sec: 169343.95507875216, Loss: 2.491712808609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21871 , TFLOPS: 69.76985930346024, Tokens per sec: 172897.39713845542, Loss: 2.4834887981414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21872 , TFLOPS: 69.71126168909053, Tokens per sec: 172752.1857949872, Loss: 2.5005812644958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21873 , TFLOPS: 69.45177881506872, Tokens per sec: 172109.15864876288, Loss: 2.5120317935943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21874 , TFLOPS: 68.74977635506418, Tokens per sec: 170369.51922091673, Loss: 2.5025689601898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21875 , TFLOPS: 70.72783242590042, Tokens per sec: 175271.36006531332, Loss: 2.487215995788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21876 , TFLOPS: 69.81055119207818, Tokens per sec: 172998.23612103227, Loss: 2.4668095111846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21877 , TFLOPS: 69.47054181557345, Tokens per sec: 172155.65543092875, Loss: 2.512850284576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21878 , TFLOPS: 69.23277052652634, Tokens per sec: 171566.43198400023, Loss: 2.5157876014709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21879 , TFLOPS: 70.55916801914734, Tokens per sec: 174853.391085461, Loss: 2.499528646469116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21880 , TFLOPS: 70.67491716670015, Tokens per sec: 175140.23022391807, Loss: 2.504749059677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21881 , TFLOPS: 67.94002047678808, Tokens per sec: 168362.8549525747, Loss: 2.4916696548461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21882 , TFLOPS: 70.98450348989279, Tokens per sec: 175907.41923653826, Loss: 2.4734840393066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21883 , TFLOPS: 69.7951136540156, Tokens per sec: 172959.98020112794, Loss: 2.512964963912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21884 , TFLOPS: 68.60675001026085, Tokens per sec: 170015.08418284153, Loss: 2.485835313796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21885 , TFLOPS: 70.50468494817588, Tokens per sec: 174718.37603378796, Loss: 2.4895710945129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21886 , TFLOPS: 70.56597961203529, Tokens per sec: 174870.2709629962, Loss: 2.4868030548095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21887 , TFLOPS: 69.79175178446651, Tokens per sec: 172951.64911804485, Loss: 2.4680898189544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21888 , TFLOPS: 68.19109153757258, Tokens per sec: 168985.03669895927, Loss: 2.5151150226593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21889 , TFLOPS: 70.70780744303467, Tokens per sec: 175221.73595183803, Loss: 2.4783239364624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21890 , TFLOPS: 70.61154652961999, Tokens per sec: 174983.19080438142, Loss: 2.47687029838562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21891 , TFLOPS: 69.24355241336204, Tokens per sec: 171593.15068730197, Loss: 2.4842758178710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21892 , TFLOPS: 68.47801260826603, Tokens per sec: 169696.0587191028, Loss: 2.5036303997039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21893 , TFLOPS: 70.63584602459629, Tokens per sec: 175043.40762974307, Loss: 2.504263401031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21894 , TFLOPS: 69.637799525305, Tokens per sec: 172570.13846060124, Loss: 2.4951255321502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21895 , TFLOPS: 68.44183498968462, Tokens per sec: 169606.40659496438, Loss: 2.4793059825897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21896 , TFLOPS: 69.73718178941883, Tokens per sec: 172816.41865893546, Loss: 2.4882304668426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21897 , TFLOPS: 70.60247568776204, Tokens per sec: 174960.71225902173, Loss: 2.509054183959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21898 , TFLOPS: 68.96327969949436, Tokens per sec: 170898.60402774968, Loss: 2.4854135513305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21899 , TFLOPS: 69.6141075259367, Tokens per sec: 172511.4270762774, Loss: 2.4863314628601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21900 , TFLOPS: 70.69175961865375, Tokens per sec: 175181.9676752083, Loss: 2.5107321739196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21901 , TFLOPS: 69.04203431918371, Tokens per sec: 171093.76665087743, Loss: 2.5092217922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21902 , TFLOPS: 70.64783754866876, Tokens per sec: 175073.12394737007, Loss: 2.5289247035980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21903 , TFLOPS: 69.63209985847557, Tokens per sec: 172556.0140583278, Loss: 2.479992389678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21904 , TFLOPS: 69.85526731273613, Tokens per sec: 173109.04759390999, Loss: 2.490413188934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21905 , TFLOPS: 70.03108351538546, Tokens per sec: 173544.7395118286, Loss: 2.4736039638519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21906 , TFLOPS: 70.59455108448826, Tokens per sec: 174941.07421914217, Loss: 2.4953670501708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21907 , TFLOPS: 69.61750600171426, Tokens per sec: 172519.84887362723, Loss: 2.4653334617614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21908 , TFLOPS: 68.87768392948172, Tokens per sec: 170686.48828050587, Loss: 2.5062224864959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21909 , TFLOPS: 70.70245124037748, Tokens per sec: 175208.4626916191, Loss: 2.492109537124634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21910 , TFLOPS: 69.47503609936751, Tokens per sec: 172166.79276125663, Loss: 2.4802167415618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21911 , TFLOPS: 69.04648716036097, Tokens per sec: 171104.80128183562, Loss: 2.4915542602539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21912 , TFLOPS: 70.5788167137514, Tokens per sec: 174902.0827151722, Loss: 2.5028622150421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21913 , TFLOPS: 70.0044737120104, Tokens per sec: 173478.79748775336, Loss: 2.5017571449279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21914 , TFLOPS: 68.88388207482267, Tokens per sec: 170701.84796744282, Loss: 2.5030360221862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21915 , TFLOPS: 69.7129553553024, Tokens per sec: 172756.38288643296, Loss: 2.499220132827759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21916 , TFLOPS: 70.70024887256902, Tokens per sec: 175203.00498157888, Loss: 2.485576868057251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21917 , TFLOPS: 69.91614716588582, Tokens per sec: 173259.91457648436, Loss: 2.4758036136627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21918 , TFLOPS: 68.7889524129437, Tokens per sec: 170466.60180794122, Loss: 2.4751100540161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21919 , TFLOPS: 70.05636989255018, Tokens per sec: 173607.40193996718, Loss: 2.4899587631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21920 , TFLOPS: 70.68675625523522, Tokens per sec: 175169.56878949123, Loss: 2.510331630706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21921 , TFLOPS: 68.96531748676384, Tokens per sec: 170903.65388908423, Loss: 2.4939324855804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21922 , TFLOPS: 69.54564427633997, Tokens per sec: 172341.7676019272, Loss: 2.480501174926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21923 , TFLOPS: 69.79151144959202, Tokens per sec: 172951.05354175076, Loss: 2.48606538772583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21924 , TFLOPS: 69.88408707532331, Tokens per sec: 173180.46614035996, Loss: 2.5022857189178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21925 , TFLOPS: 68.71128981985873, Tokens per sec: 170274.14534703578, Loss: 2.4675300121307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21926 , TFLOPS: 70.54993258982613, Tokens per sec: 174830.50467423646, Loss: 2.4911158084869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21927 , TFLOPS: 70.60745388741128, Tokens per sec: 174973.04878614654, Loss: 2.4613311290740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21928 , TFLOPS: 68.25751269110188, Tokens per sec: 169149.63563429896, Loss: 2.496614694595337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21929 , TFLOPS: 70.54778700713868, Tokens per sec: 174825.18768398152, Loss: 2.480912208557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21930 , TFLOPS: 69.97242232526035, Tokens per sec: 173399.37062063505, Loss: 2.502340316772461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21931 , TFLOPS: 69.00900886668454, Tokens per sec: 171011.92594153056, Loss: 2.4720258712768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21932 , TFLOPS: 69.57919286230613, Tokens per sec: 172424.90469363314, Loss: 2.4694528579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21933 , TFLOPS: 70.61203578426098, Tokens per sec: 174984.40323127765, Loss: 2.506470203399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21934 , TFLOPS: 69.0397934614901, Tokens per sec: 171088.2135586618, Loss: 2.481783151626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21935 , TFLOPS: 69.713562971833, Tokens per sec: 172757.88862713025, Loss: 2.485261917114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21936 , TFLOPS: 69.63919979162965, Tokens per sec: 172573.6084748058, Loss: 2.488471746444702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21937 , TFLOPS: 69.67524081738863, Tokens per sec: 172662.92210688328, Loss: 2.496257781982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21938 , TFLOPS: 69.90091383921845, Tokens per sec: 173222.16471491317, Loss: 2.502582550048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21939 , TFLOPS: 69.8327472343742, Tokens per sec: 173053.2403589377, Loss: 2.4916584491729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21940 , TFLOPS: 69.61296885541081, Tokens per sec: 172508.60532527935, Loss: 2.4893417358398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21941 , TFLOPS: 68.9093552187002, Tokens per sec: 170764.97322406978, Loss: 2.472841739654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21942 , TFLOPS: 70.57883053547114, Tokens per sec: 174902.11696691636, Loss: 2.507568597793579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21943 , TFLOPS: 69.83165014306277, Tokens per sec: 173050.52164581342, Loss: 2.48799729347229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21944 , TFLOPS: 69.55697364666781, Tokens per sec: 172369.84302963273, Loss: 2.4938650131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21945 , TFLOPS: 69.37566186250153, Tokens per sec: 171920.53245532684, Loss: 2.4750924110412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21946 , TFLOPS: 70.58321742167145, Tokens per sec: 174912.9881541761, Loss: 2.4771060943603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21947 , TFLOPS: 69.68598710950464, Tokens per sec: 172689.55260254856, Loss: 2.4990711212158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21948 , TFLOPS: 68.04739845796489, Tokens per sec: 168628.94941859195, Loss: 2.476686954498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21949 , TFLOPS: 71.49335894106032, Tokens per sec: 177168.41909958678, Loss: 2.485708713531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21950 , TFLOPS: 69.72041700491123, Tokens per sec: 172774.87367613218, Loss: 2.4774818420410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21951 , TFLOPS: 67.168499227371, Tokens per sec: 166450.94030643746, Loss: 2.509699821472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21952 , TFLOPS: 70.57032754774049, Tokens per sec: 174881.04562663837, Loss: 2.4668421745300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21953 , TFLOPS: 70.57922350695966, Tokens per sec: 174903.09079355426, Loss: 2.5165109634399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21954 , TFLOPS: 69.64605938364568, Tokens per sec: 172590.60729946796, Loss: 2.4880852699279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21955 , TFLOPS: 68.54682637865987, Tokens per sec: 169866.58682260133, Loss: 2.496107339859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21956 , TFLOPS: 70.69817432649661, Tokens per sec: 175197.86402788133, Loss: 2.454178810119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21957 , TFLOPS: 69.6215634197388, Tokens per sec: 172529.90360245353, Loss: 2.508150815963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21958 , TFLOPS: 68.9041415584729, Tokens per sec: 170752.05319969446, Loss: 2.4835963249206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21959 , TFLOPS: 69.66894794562528, Tokens per sec: 172647.32767743664, Loss: 2.472339153289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21960 , TFLOPS: 70.63040029254499, Tokens per sec: 175029.91250582304, Loss: 2.5068278312683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21961 , TFLOPS: 69.61736971315909, Tokens per sec: 172519.51113557408, Loss: 2.4887588024139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21962 , TFLOPS: 69.85871288145839, Tokens per sec: 173117.58609276448, Loss: 2.456348419189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21963 , TFLOPS: 69.7595018730449, Tokens per sec: 172871.73028492107, Loss: 2.4842817783355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21964 , TFLOPS: 70.55721742063135, Tokens per sec: 174848.55728746217, Loss: 2.5006892681121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21965 , TFLOPS: 68.94287161972048, Tokens per sec: 170848.03055793443, Loss: 2.4719762802124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21966 , TFLOPS: 69.50423854892213, Tokens per sec: 172239.15964816837, Loss: 2.465749502182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21967 , TFLOPS: 70.53335275551251, Tokens per sec: 174789.4180183878, Loss: 2.4856526851654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21968 , TFLOPS: 69.05468771935666, Tokens per sec: 171125.12317039308, Loss: 2.479464054107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21969 , TFLOPS: 69.95524566348091, Tokens per sec: 173356.80496058008, Loss: 2.4858171939849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21970 , TFLOPS: 69.75368609279457, Tokens per sec: 172857.31814079967, Loss: 2.4962947368621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21971 , TFLOPS: 69.59044836305365, Tokens per sec: 172452.79706438124, Loss: 2.5068578720092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21972 , TFLOPS: 69.94087265998883, Tokens per sec: 173321.18707460968, Loss: 2.5139243602752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21973 , TFLOPS: 69.72507588003432, Tokens per sec: 172786.41888190468, Loss: 2.483248710632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21974 , TFLOPS: 69.66150772960903, Tokens per sec: 172628.89000254092, Loss: 2.5108888149261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21975 , TFLOPS: 69.13724313243465, Tokens per sec: 171329.70457822984, Loss: 2.4778196811676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21976 , TFLOPS: 70.51019610788374, Tokens per sec: 174732.0332946484, Loss: 2.5065274238586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21977 , TFLOPS: 69.61014522758872, Tokens per sec: 172501.6080644879, Loss: 2.5020010471343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21978 , TFLOPS: 69.22791367149694, Tokens per sec: 171554.3961622106, Loss: 2.4714245796203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21979 , TFLOPS: 70.60526694350632, Tokens per sec: 174967.62929823826, Loss: 2.5000505447387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21980 , TFLOPS: 69.98223682245897, Tokens per sec: 173423.69202584933, Loss: 2.5015697479248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21981 , TFLOPS: 68.86060840320171, Tokens per sec: 170644.17324535953, Loss: 2.4865200519561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21982 , TFLOPS: 69.82180805598139, Tokens per sec: 173026.13186983037, Loss: 2.504873514175415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21983 , TFLOPS: 70.52736013709529, Tokens per sec: 174774.56764980854, Loss: 2.5115556716918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21984 , TFLOPS: 69.0976170335741, Tokens per sec: 171231.50674007824, Loss: 2.4929451942443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21985 , TFLOPS: 69.74537812174185, Tokens per sec: 172836.73007333247, Loss: 2.4819486141204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21986 , TFLOPS: 70.08266515887397, Tokens per sec: 173672.5645065802, Loss: 2.4727964401245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21987 , TFLOPS: 70.51285905576741, Tokens per sec: 174738.63237285204, Loss: 2.476985216140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21988 , TFLOPS: 68.05477171228084, Tokens per sec: 168647.2211550179, Loss: 2.5091984272003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21989 , TFLOPS: 70.70067723310063, Tokens per sec: 175204.066506164, Loss: 2.4810304641723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21990 , TFLOPS: 69.73321695914335, Tokens per sec: 172806.59337275074, Loss: 2.4953842163085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21991 , TFLOPS: 70.62460887339056, Tokens per sec: 175015.56073118086, Loss: 2.4908034801483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21992 , TFLOPS: 68.70486448106277, Tokens per sec: 170258.22264968988, Loss: 2.464252471923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21993 , TFLOPS: 70.66355882744641, Tokens per sec: 175112.08300802292, Loss: 2.472569227218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21994 , TFLOPS: 70.49430880819804, Tokens per sec: 174692.6628158967, Loss: 2.480103015899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21995 , TFLOPS: 68.42345516301884, Tokens per sec: 169560.85935978155, Loss: 2.4758434295654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21996 , TFLOPS: 70.42338254974271, Tokens per sec: 174516.899734272, Loss: 2.488210678100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21997 , TFLOPS: 70.02899386626945, Tokens per sec: 173539.56113112526, Loss: 2.490607500076294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21998 , TFLOPS: 68.90326176490017, Tokens per sec: 170749.872974304, Loss: 2.497769832611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21999 , TFLOPS: 70.36641419160311, Tokens per sec: 174375.72586721936, Loss: 2.474484443664551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22000 , TFLOPS: 70.74330671275965, Tokens per sec: 175309.70705278392, Loss: 2.498819351196289 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/model.pt +[rank0]:[2024-08-28 00:30:17,659] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006403515988495201, 'preprocessing_with_comm': 0.0012567749945446849, 'state_converting': 0.9914448100025766, : 1.0004650769988075}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022000, took 4.36s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020000 + eval ppl=10.547050476074219, eval loss=2.3558461666107178 +------------------------------------------------------------------ +iteration: 22001 , TFLOPS: 63.98437771532008, Tokens per sec: 158560.3364396347, Loss: 2.485893726348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22002 , TFLOPS: 69.29052186127859, Tokens per sec: 171709.54615335102, Loss: 2.4882330894470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22003 , TFLOPS: 70.60706820041658, Tokens per sec: 174972.0930112867, Loss: 2.484776258468628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22004 , TFLOPS: 69.59457291329628, Tokens per sec: 172463.01815423922, Loss: 2.4915952682495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22005 , TFLOPS: 69.7766672112328, Tokens per sec: 172914.26788386877, Loss: 2.487873077392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22006 , TFLOPS: 69.46016379168377, Tokens per sec: 172129.93754449638, Loss: 2.4859611988067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22007 , TFLOPS: 69.84349368781032, Tokens per sec: 173079.87125437197, Loss: 2.483564853668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22008 , TFLOPS: 69.78113630502867, Tokens per sec: 172925.34279060626, Loss: 2.476706027984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22009 , TFLOPS: 70.12472906525781, Tokens per sec: 173776.8035003212, Loss: 2.4872286319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22010 , TFLOPS: 68.62001744447903, Tokens per sec: 170047.96234636268, Loss: 2.5009119510650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22011 , TFLOPS: 69.69542911278016, Tokens per sec: 172712.95092678783, Loss: 2.4903929233551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22012 , TFLOPS: 69.77509141781302, Tokens per sec: 172910.36289418658, Loss: 2.477776288986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22013 , TFLOPS: 70.67055358911668, Tokens per sec: 175129.41679797845, Loss: 2.51043438911438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22014 , TFLOPS: 69.57558140319733, Tokens per sec: 172415.95510012648, Loss: 2.4729411602020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22015 , TFLOPS: 69.90100922088442, Tokens per sec: 173222.40108118858, Loss: 2.4760332107543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22016 , TFLOPS: 70.94222939186126, Tokens per sec: 175802.65936473975, Loss: 2.4770309925079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22017 , TFLOPS: 68.55851126171832, Tokens per sec: 169895.54325001632, Loss: 2.4746694564819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22018 , TFLOPS: 69.0425108354819, Tokens per sec: 171094.94751075693, Loss: 2.4945151805877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22019 , TFLOPS: 70.58739352628795, Tokens per sec: 174923.33700145167, Loss: 2.5026655197143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22020 , TFLOPS: 70.7454579938573, Tokens per sec: 175315.03816432707, Loss: 2.4731295108795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22021 , TFLOPS: 68.47417154782345, Tokens per sec: 169686.54014819834, Loss: 2.4895405769348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22022 , TFLOPS: 69.82898850717822, Tokens per sec: 173043.9258188881, Loss: 2.4935007095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22023 , TFLOPS: 70.65302187368435, Tokens per sec: 175085.97127585928, Loss: 2.487412452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22024 , TFLOPS: 69.77054830962422, Tokens per sec: 172899.10457163004, Loss: 2.474248170852661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22025 , TFLOPS: 68.91375403182998, Tokens per sec: 170775.87396757468, Loss: 2.4869682788848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22026 , TFLOPS: 70.60025490559165, Tokens per sec: 174955.20891616418, Loss: 2.4812655448913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22027 , TFLOPS: 70.05886605347143, Tokens per sec: 173613.58770170488, Loss: 2.4940543174743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22028 , TFLOPS: 68.48397826184646, Tokens per sec: 169710.8422658461, Loss: 2.4713857173919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22029 , TFLOPS: 69.82404616928098, Tokens per sec: 173031.67816113526, Loss: 2.474118709564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22030 , TFLOPS: 70.58217693546631, Tokens per sec: 174910.40971474073, Loss: 2.496673583984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22031 , TFLOPS: 70.11890899094567, Tokens per sec: 173762.38071504163, Loss: 2.503469705581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22032 , TFLOPS: 68.94015790861778, Tokens per sec: 170841.30568288197, Loss: 2.495619058609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22033 , TFLOPS: 69.98110223742346, Tokens per sec: 173420.88039914647, Loss: 2.4739677906036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22034 , TFLOPS: 70.53926287957485, Tokens per sec: 174804.06395686822, Loss: 2.483996629714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22035 , TFLOPS: 69.70976001894927, Tokens per sec: 172748.4644909545, Loss: 2.5092034339904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22036 , TFLOPS: 69.49048640659008, Tokens per sec: 172205.080324546, Loss: 2.491607427597046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22037 , TFLOPS: 69.7524033046612, Tokens per sec: 172854.13925049437, Loss: 2.482839822769165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22038 , TFLOPS: 69.72479871420524, Tokens per sec: 172785.73203444786, Loss: 2.4742348194122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22039 , TFLOPS: 69.07970220789186, Tokens per sec: 171187.1118285567, Loss: 2.4851009845733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22040 , TFLOPS: 70.72336930252999, Tokens per sec: 175260.29995394778, Loss: 2.4973227977752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22041 , TFLOPS: 69.4866475517311, Tokens per sec: 172195.56721932066, Loss: 2.5033953189849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22042 , TFLOPS: 69.74813904775343, Tokens per sec: 172843.57195213024, Loss: 2.499415874481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22043 , TFLOPS: 70.49195494080661, Tokens per sec: 174686.82967320085, Loss: 2.494818925857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22044 , TFLOPS: 70.0441621327455, Tokens per sec: 173577.14976637543, Loss: 2.4910309314727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22045 , TFLOPS: 69.2669579722477, Tokens per sec: 171651.1522405558, Loss: 2.4929444789886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22046 , TFLOPS: 70.62311229749297, Tokens per sec: 175011.8520512452, Loss: 2.4895665645599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22047 , TFLOPS: 69.50744723843263, Tokens per sec: 172247.1111342417, Loss: 2.4742870330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22048 , TFLOPS: 69.07510912341617, Tokens per sec: 171175.7296592559, Loss: 2.502007246017456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22049 , TFLOPS: 69.80290338583579, Tokens per sec: 172979.28401468822, Loss: 2.4710590839385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22050 , TFLOPS: 70.73205814898763, Tokens per sec: 175281.83187262528, Loss: 2.4796485900878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22051 , TFLOPS: 68.72350112180173, Tokens per sec: 170304.4062984365, Loss: 2.5031991004943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22052 , TFLOPS: 69.73225878484635, Tokens per sec: 172804.21891129116, Loss: 2.4820358753204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22053 , TFLOPS: 69.6410646935091, Tokens per sec: 172578.22990709895, Loss: 2.495471954345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22054 , TFLOPS: 69.51129557430875, Tokens per sec: 172256.64773447192, Loss: 2.4745700359344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22055 , TFLOPS: 69.84800183853646, Tokens per sec: 173091.04294848515, Loss: 2.4824821949005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22056 , TFLOPS: 70.65381969485955, Tokens per sec: 175087.94836461864, Loss: 2.4996984004974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22057 , TFLOPS: 69.83895388400266, Tokens per sec: 173068.62112043594, Loss: 2.475770950317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22058 , TFLOPS: 68.91354246300996, Tokens per sec: 170775.3496767324, Loss: 2.5037336349487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22059 , TFLOPS: 69.72338086269058, Tokens per sec: 172782.21844220528, Loss: 2.503774642944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22060 , TFLOPS: 70.57259469654275, Tokens per sec: 174886.66387111743, Loss: 2.489147186279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22061 , TFLOPS: 69.50911998237713, Tokens per sec: 172251.25637800412, Loss: 2.4806408882141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22062 , TFLOPS: 68.88112077746926, Tokens per sec: 170695.00516841907, Loss: 2.4831390380859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22063 , TFLOPS: 71.52041064667974, Tokens per sec: 177235.4562061031, Loss: 2.4876489639282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22064 , TFLOPS: 69.86559616917148, Tokens per sec: 173134.64363797425, Loss: 2.4793059825897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22065 , TFLOPS: 68.98016155058487, Tokens per sec: 170940.43911444137, Loss: 2.5086092948913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22066 , TFLOPS: 69.67563662839534, Tokens per sec: 172663.90297016007, Loss: 2.490342617034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22067 , TFLOPS: 70.59611611909892, Tokens per sec: 174944.95254731018, Loss: 2.5069026947021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22068 , TFLOPS: 68.84846713116025, Tokens per sec: 170614.08583576925, Loss: 2.4686176776885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22069 , TFLOPS: 68.81526490122711, Tokens per sec: 170531.80705246734, Loss: 2.4826927185058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22070 , TFLOPS: 70.74392421809141, Tokens per sec: 175311.23729901985, Loss: 2.477804183959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22071 , TFLOPS: 69.7516610722172, Tokens per sec: 172852.29991673448, Loss: 2.4851129055023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22072 , TFLOPS: 69.83612504841113, Tokens per sec: 173061.61095421817, Loss: 2.4910435676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22073 , TFLOPS: 69.33440172050835, Tokens per sec: 171818.2852782879, Loss: 2.481663465499878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22074 , TFLOPS: 69.92751698814062, Tokens per sec: 173288.09024852113, Loss: 2.5039620399475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22075 , TFLOPS: 69.83982555426009, Tokens per sec: 173070.78121535623, Loss: 2.4856693744659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22076 , TFLOPS: 69.0792993573499, Tokens per sec: 171186.11352053605, Loss: 2.4688613414764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22077 , TFLOPS: 70.66057675250345, Tokens per sec: 175104.69309781157, Loss: 2.5059714317321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22078 , TFLOPS: 69.20428020523576, Tokens per sec: 171495.8298293742, Loss: 2.4961063861846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22079 , TFLOPS: 69.83733681520047, Tokens per sec: 173064.61384580928, Loss: 2.5164480209350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22080 , TFLOPS: 69.66674896170217, Tokens per sec: 172641.87835303711, Loss: 2.497995615005493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22081 , TFLOPS: 70.60409717131763, Tokens per sec: 174964.73047389367, Loss: 2.4946775436401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22082 , TFLOPS: 69.45853921187272, Tokens per sec: 172125.91165676236, Loss: 2.4967024326324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22083 , TFLOPS: 70.88201903737341, Tokens per sec: 175653.4514736008, Loss: 2.4760403633117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22084 , TFLOPS: 69.59697791299466, Tokens per sec: 172468.9780084244, Loss: 2.4920265674591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22085 , TFLOPS: 68.91599530879047, Tokens per sec: 170781.42809877978, Loss: 2.492973566055298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22086 , TFLOPS: 70.75684197900283, Tokens per sec: 175343.24893356714, Loss: 2.4893250465393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22087 , TFLOPS: 70.56459650844515, Tokens per sec: 174866.8434799384, Loss: 2.504703998565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22088 , TFLOPS: 67.8863916932014, Tokens per sec: 168229.95691914874, Loss: 2.4824225902557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22089 , TFLOPS: 69.78046278018442, Tokens per sec: 172923.67372184733, Loss: 2.506619930267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22090 , TFLOPS: 70.7167611369128, Tokens per sec: 175243.9241915988, Loss: 2.4631574153900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22091 , TFLOPS: 67.96787587111729, Tokens per sec: 168431.8836882475, Loss: 2.501193046569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22092 , TFLOPS: 69.73683963968621, Tokens per sec: 172815.57077420712, Loss: 2.494770050048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22093 , TFLOPS: 70.68414483674279, Tokens per sec: 175163.09740679548, Loss: 2.494558095932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22094 , TFLOPS: 69.72592219371411, Tokens per sec: 172788.51614043233, Loss: 2.4808645248413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22095 , TFLOPS: 69.48324185101632, Tokens per sec: 172187.12751778055, Loss: 2.4732260704040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22096 , TFLOPS: 69.8196549147649, Tokens per sec: 173020.7961487078, Loss: 2.4969682693481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22097 , TFLOPS: 70.71004485597203, Tokens per sec: 175227.28050756818, Loss: 2.4770987033843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22098 , TFLOPS: 69.74786142282338, Tokens per sec: 172842.88396697066, Loss: 2.4798245429992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22099 , TFLOPS: 69.1347942196855, Tokens per sec: 171323.63590266745, Loss: 2.483741521835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22100 , TFLOPS: 70.06535979568305, Tokens per sec: 173629.67991025, Loss: 2.481717586517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22101 , TFLOPS: 70.57990454168349, Tokens per sec: 174904.77847262207, Loss: 2.498131275177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22102 , TFLOPS: 68.84375513922178, Tokens per sec: 170602.40900067694, Loss: 2.482703447341919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22103 , TFLOPS: 70.54715397955927, Tokens per sec: 174823.61897190602, Loss: 2.4788613319396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22104 , TFLOPS: 69.82383466537051, Tokens per sec: 173031.15403114606, Loss: 2.493177890777588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22105 , TFLOPS: 68.73196460337888, Tokens per sec: 170325.3797381141, Loss: 2.4901866912841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22106 , TFLOPS: 69.47231750938525, Tokens per sec: 172160.0557958032, Loss: 2.490703582763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22107 , TFLOPS: 70.59128436131938, Tokens per sec: 174932.97891927045, Loss: 2.466987133026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22108 , TFLOPS: 68.89352763331422, Tokens per sec: 170725.75072393086, Loss: 2.492832660675049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22109 , TFLOPS: 69.75981906001111, Tokens per sec: 172872.51630916438, Loss: 2.4543731212615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22110 , TFLOPS: 70.49172044031363, Tokens per sec: 174686.24855514686, Loss: 2.4759368896484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22111 , TFLOPS: 69.24926366230108, Tokens per sec: 171607.3037913189, Loss: 2.4906585216522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22112 , TFLOPS: 69.81816175600082, Tokens per sec: 173017.0959368054, Loss: 2.4648563861846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22113 , TFLOPS: 69.62172374816592, Tokens per sec: 172530.30091395756, Loss: 2.503911256790161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22114 , TFLOPS: 70.54135750938987, Tokens per sec: 174809.25468029233, Loss: 2.501016616821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22115 , TFLOPS: 69.84299356659993, Tokens per sec: 173078.63189891973, Loss: 2.4905152320861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22116 , TFLOPS: 69.709411327539, Tokens per sec: 172747.60039522834, Loss: 2.4897797107696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22117 , TFLOPS: 70.46407255925976, Tokens per sec: 174617.73406022985, Loss: 2.497596263885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22118 , TFLOPS: 69.70234099029044, Tokens per sec: 172730.0793206645, Loss: 2.47175931930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22119 , TFLOPS: 69.83359382434794, Tokens per sec: 173055.33830215223, Loss: 2.4936163425445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22120 , TFLOPS: 69.78438021474855, Tokens per sec: 172933.38155623828, Loss: 2.5095882415771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22121 , TFLOPS: 69.58941875644831, Tokens per sec: 172450.24558579273, Loss: 2.49261736869812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22122 , TFLOPS: 69.65108710884701, Tokens per sec: 172603.0665563665, Loss: 2.490220308303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22123 , TFLOPS: 70.61906345305647, Tokens per sec: 175001.81856871452, Loss: 2.4947800636291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22124 , TFLOPS: 69.76146956698088, Tokens per sec: 172876.6064472538, Loss: 2.4964041709899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22125 , TFLOPS: 68.78706488300872, Tokens per sec: 170461.92430083372, Loss: 2.48235821723938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22126 , TFLOPS: 70.238863534664, Tokens per sec: 174059.6412884595, Loss: 2.4822075366973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22127 , TFLOPS: 70.62330735230948, Tokens per sec: 175012.33541856759, Loss: 2.4788568019866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22128 , TFLOPS: 68.6881700638755, Tokens per sec: 170216.85204485795, Loss: 2.456328868865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22129 , TFLOPS: 69.97779744765144, Tokens per sec: 173412.69076603878, Loss: 2.4885971546173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22130 , TFLOPS: 70.84599403904058, Tokens per sec: 175564.17755925085, Loss: 2.5112364292144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22131 , TFLOPS: 69.77609938895341, Tokens per sec: 172912.86075770968, Loss: 2.4910643100738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22132 , TFLOPS: 69.17392740574381, Tokens per sec: 171420.61224859627, Loss: 2.486910104751587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22133 , TFLOPS: 70.60645169771122, Tokens per sec: 174970.56524966948, Loss: 2.4794535636901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22134 , TFLOPS: 70.57854879854816, Tokens per sec: 174901.41879178528, Loss: 2.4896671772003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22135 , TFLOPS: 69.22838298976657, Tokens per sec: 171555.55918458247, Loss: 2.501055955886841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22136 , TFLOPS: 68.90256852949015, Tokens per sec: 170748.15506059173, Loss: 2.4859108924865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22137 , TFLOPS: 70.64173119598036, Tokens per sec: 175057.99173273792, Loss: 2.4791157245635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22138 , TFLOPS: 69.60403956402723, Tokens per sec: 172486.4775575882, Loss: 2.5021941661834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22139 , TFLOPS: 69.85708238019112, Tokens per sec: 173113.54553100964, Loss: 2.499962329864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22140 , TFLOPS: 68.71621417364453, Tokens per sec: 170286.34843817932, Loss: 2.4788658618927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22141 , TFLOPS: 70.29422365463006, Tokens per sec: 174196.82976415605, Loss: 2.5068626403808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22142 , TFLOPS: 70.15910376395854, Tokens per sec: 173861.9877903873, Loss: 2.489915132522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22143 , TFLOPS: 69.1434731554572, Tokens per sec: 171345.14326157342, Loss: 2.4961323738098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22144 , TFLOPS: 69.74496640420804, Tokens per sec: 172835.70979192926, Loss: 2.491922378540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22145 , TFLOPS: 69.58116596158477, Tokens per sec: 172429.79425100045, Loss: 2.4811155796051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22146 , TFLOPS: 69.85540927160359, Tokens per sec: 173109.39938362164, Loss: 2.4913644790649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22147 , TFLOPS: 69.60117361001379, Tokens per sec: 172479.37540783198, Loss: 2.506828546524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22148 , TFLOPS: 70.64343892189018, Tokens per sec: 175062.223665664, Loss: 2.4931020736694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22149 , TFLOPS: 69.14361721210066, Tokens per sec: 171345.50024980507, Loss: 2.488929033279419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22150 , TFLOPS: 69.67384870118427, Tokens per sec: 172659.47228957462, Loss: 2.4791617393493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22151 , TFLOPS: 69.75834969115381, Tokens per sec: 172868.87505127164, Loss: 2.489561080932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22152 , TFLOPS: 68.83177287770917, Tokens per sec: 170572.71563669364, Loss: 2.492173671722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22153 , TFLOPS: 70.52557169113135, Tokens per sec: 174770.13568369628, Loss: 2.4917588233947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22154 , TFLOPS: 69.5214737300046, Tokens per sec: 172281.87032550265, Loss: 2.4948184490203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22155 , TFLOPS: 69.63677821657232, Tokens per sec: 172567.60754505522, Loss: 2.4896318912506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22156 , TFLOPS: 69.90019329621852, Tokens per sec: 173220.37912998497, Loss: 2.4713807106018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22157 , TFLOPS: 70.68855274381139, Tokens per sec: 175174.0206860825, Loss: 2.487109899520874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22158 , TFLOPS: 68.66553484636347, Tokens per sec: 170160.7594823881, Loss: 2.4769845008850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22159 , TFLOPS: 69.82187579060036, Tokens per sec: 173026.29972367772, Loss: 2.4931423664093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22160 , TFLOPS: 70.54365979341699, Tokens per sec: 174814.95999372905, Loss: 2.4901182651519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22161 , TFLOPS: 70.11092962950548, Tokens per sec: 173742.6069783936, Loss: 2.474168062210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22162 , TFLOPS: 69.40947649405852, Tokens per sec: 172004.3288373142, Loss: 2.5035157203674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22163 , TFLOPS: 69.82851453770164, Tokens per sec: 173042.75127031308, Loss: 2.4870572090148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22164 , TFLOPS: 70.49573384758541, Tokens per sec: 174696.19422047967, Loss: 2.492650032043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22165 , TFLOPS: 69.74663468589152, Tokens per sec: 172839.8439777173, Loss: 2.510735511779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22166 , TFLOPS: 70.10877994533288, Tokens per sec: 173737.27982420663, Loss: 2.4868974685668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22167 , TFLOPS: 70.12391317602756, Tokens per sec: 173774.78163693103, Loss: 2.478916645050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22168 , TFLOPS: 70.61079599595976, Tokens per sec: 174981.33089929327, Loss: 2.474820613861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22169 , TFLOPS: 67.87398375090704, Tokens per sec: 168199.2086712956, Loss: 2.514228582382202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22170 , TFLOPS: 70.57424914587772, Tokens per sec: 174890.7637788221, Loss: 2.491452693939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22171 , TFLOPS: 69.67999658870636, Tokens per sec: 172674.70743210005, Loss: 2.4975178241729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22172 , TFLOPS: 68.97869462777103, Tokens per sec: 170936.80391811376, Loss: 2.4623894691467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22173 , TFLOPS: 69.53594168203024, Tokens per sec: 172317.72350439866, Loss: 2.490387201309204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22174 , TFLOPS: 70.66992944019593, Tokens per sec: 175127.87008819715, Loss: 2.5100698471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22175 , TFLOPS: 69.15016722246067, Tokens per sec: 171361.73189702036, Loss: 2.482564687728882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22176 , TFLOPS: 68.56556874399122, Tokens per sec: 169913.0324685344, Loss: 2.4864253997802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22177 , TFLOPS: 71.50562927105018, Tokens per sec: 177198.82632899014, Loss: 2.501897096633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22178 , TFLOPS: 68.97109897910207, Tokens per sec: 170917.9810639823, Loss: 2.5006704330444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22179 , TFLOPS: 69.73057179406717, Tokens per sec: 172800.03836230296, Loss: 2.48054575920105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22180 , TFLOPS: 70.05917630841404, Tokens per sec: 173614.35654763004, Loss: 2.496347188949585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22181 , TFLOPS: 70.72223343071823, Tokens per sec: 175257.48513847162, Loss: 2.4790966510772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22182 , TFLOPS: 69.83770559913603, Tokens per sec: 173065.5277330265, Loss: 2.4961705207824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22183 , TFLOPS: 69.97216693782043, Tokens per sec: 173398.73774242567, Loss: 2.4925596714019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22184 , TFLOPS: 69.63643666641701, Tokens per sec: 172566.7611461455, Loss: 2.496885061264038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22185 , TFLOPS: 69.63635412335522, Tokens per sec: 172566.55659534552, Loss: 2.494431257247925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22186 , TFLOPS: 69.8542682788513, Tokens per sec: 173106.5718778907, Loss: 2.483159065246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22187 , TFLOPS: 68.7508025497335, Tokens per sec: 170372.06224435195, Loss: 2.513766288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22188 , TFLOPS: 70.59797105798403, Tokens per sec: 174949.5492902059, Loss: 2.505201578140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22189 , TFLOPS: 68.98056480960295, Tokens per sec: 170941.43843471084, Loss: 2.483999490737915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22190 , TFLOPS: 70.58561117176305, Tokens per sec: 174918.92013059644, Loss: 2.496936559677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22191 , TFLOPS: 68.77475383825268, Tokens per sec: 170431.4161757547, Loss: 2.4860763549804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22192 , TFLOPS: 69.78228924914002, Tokens per sec: 172928.19991312147, Loss: 2.491050958633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22193 , TFLOPS: 70.09908310418375, Tokens per sec: 173713.24998364385, Loss: 2.5088956356048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22194 , TFLOPS: 70.6194578426316, Tokens per sec: 175002.79590952728, Loss: 2.487278699874878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22195 , TFLOPS: 68.62321394413647, Tokens per sec: 170055.88362463735, Loss: 2.498927354812622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22196 , TFLOPS: 68.94908549910312, Tokens per sec: 170863.42923555992, Loss: 2.4773569107055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22197 , TFLOPS: 71.47456112893958, Tokens per sec: 177121.83605040165, Loss: 2.470163345336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22198 , TFLOPS: 68.6716928899963, Tokens per sec: 170176.01979287472, Loss: 2.487105369567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22199 , TFLOPS: 69.25103273617646, Tokens per sec: 171611.68775126166, Loss: 2.494377851486206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22200 , TFLOPS: 70.72939892530617, Tokens per sec: 175275.24202340492, Loss: 2.4804210662841797 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/model.pt +[rank0]:[2024-08-28 00:55:42,852] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006406166998203844, 'preprocessing_with_comm': 0.0012786799925379455, 'state_converting': 1.0131680339982267, : 1.0222315250139218}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022200, took 4.43s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020200 + eval ppl=9.6315279006958, eval loss=2.2650418281555176 +------------------------------------------------------------------ +iteration: 22201 , TFLOPS: 66.79714829650621, Tokens per sec: 165530.69179207247, Loss: 2.476369857788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22202 , TFLOPS: 67.43337367797777, Tokens per sec: 167107.3283733705, Loss: 2.489154577255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22203 , TFLOPS: 69.81887327960662, Tokens per sec: 173018.85917068136, Loss: 2.4638094902038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22204 , TFLOPS: 70.61789883743417, Tokens per sec: 174998.93252290945, Loss: 2.485325813293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22205 , TFLOPS: 68.9507102479379, Tokens per sec: 170867.4555421534, Loss: 2.4809212684631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22206 , TFLOPS: 69.60355265735804, Tokens per sec: 172485.2709492247, Loss: 2.485269784927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22207 , TFLOPS: 69.92631320070777, Tokens per sec: 173285.10713065433, Loss: 2.4729089736938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22208 , TFLOPS: 70.63650197211096, Tokens per sec: 175045.033139943, Loss: 2.4876725673675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22209 , TFLOPS: 68.9722495805602, Tokens per sec: 170920.83238114478, Loss: 2.4828031063079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22210 , TFLOPS: 70.06129052060682, Tokens per sec: 173619.59579834345, Loss: 2.470108985900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22211 , TFLOPS: 69.37987613044116, Tokens per sec: 171930.97587552117, Loss: 2.4988250732421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22212 , TFLOPS: 70.5267303601711, Tokens per sec: 174773.00699321472, Loss: 2.486074209213257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22213 , TFLOPS: 69.82280853713762, Tokens per sec: 173028.61117234756, Loss: 2.5024490356445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22214 , TFLOPS: 69.54625907470123, Tokens per sec: 172343.291139992, Loss: 2.4868216514587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22215 , TFLOPS: 69.77624085173336, Tokens per sec: 172913.2113180618, Loss: 2.500152111053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22216 , TFLOPS: 68.84293495650354, Tokens per sec: 170600.37649755058, Loss: 2.4914402961730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22217 , TFLOPS: 69.4433473139856, Tokens per sec: 172088.26446027236, Loss: 2.4641175270080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22218 , TFLOPS: 70.68772354019607, Tokens per sec: 175171.96582817944, Loss: 2.474905252456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22219 , TFLOPS: 68.92536181487341, Tokens per sec: 170804.63933265914, Loss: 2.4810385704040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22220 , TFLOPS: 70.63997981972362, Tokens per sec: 175053.65162944372, Loss: 2.486551284790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22221 , TFLOPS: 69.90619810107596, Tokens per sec: 173235.25969790557, Loss: 2.491209030151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22222 , TFLOPS: 68.93956530584296, Tokens per sec: 170839.83714792537, Loss: 2.5023555755615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22223 , TFLOPS: 69.65441223685131, Tokens per sec: 172611.30658985424, Loss: 2.4953625202178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22224 , TFLOPS: 70.71974715360558, Tokens per sec: 175251.32386990046, Loss: 2.4848413467407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22225 , TFLOPS: 68.88985998068912, Tokens per sec: 170716.66187670061, Loss: 2.4792799949645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22226 , TFLOPS: 70.15731537031454, Tokens per sec: 173857.5559539296, Loss: 2.482823371887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22227 , TFLOPS: 69.71602987030953, Tokens per sec: 172764.00187330638, Loss: 2.4928975105285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22228 , TFLOPS: 70.60669621845373, Tokens per sec: 174971.1711990056, Loss: 2.4863812923431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22229 , TFLOPS: 68.66787642870968, Tokens per sec: 170166.56218138855, Loss: 2.496833562850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22230 , TFLOPS: 70.07723533805797, Tokens per sec: 173659.10881245407, Loss: 2.4797191619873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22231 , TFLOPS: 70.7378450318516, Tokens per sec: 175296.17240583996, Loss: 2.507364511489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22232 , TFLOPS: 68.87808425275638, Tokens per sec: 170687.48032567973, Loss: 2.5101897716522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22233 , TFLOPS: 69.80732223527863, Tokens per sec: 172990.23441038403, Loss: 2.472740650177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22234 , TFLOPS: 70.55552851919693, Tokens per sec: 174844.3720036598, Loss: 2.5028133392333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22235 , TFLOPS: 69.61893987358793, Tokens per sec: 172523.4021660842, Loss: 2.4815304279327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22236 , TFLOPS: 68.67699123424767, Tokens per sec: 170189.14967359044, Loss: 2.510805368423462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22237 , TFLOPS: 70.91428594014194, Tokens per sec: 175733.412413154, Loss: 2.514698028564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22238 , TFLOPS: 70.15873737944983, Tokens per sec: 173861.07984921412, Loss: 2.486391305923462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22239 , TFLOPS: 68.93181817864584, Tokens per sec: 170820.6389133144, Loss: 2.4910030364990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22240 , TFLOPS: 70.57355008292843, Tokens per sec: 174889.03142382565, Loss: 2.4983811378479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22241 , TFLOPS: 70.5945025133898, Tokens per sec: 174940.95385460966, Loss: 2.493680477142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22242 , TFLOPS: 68.79878229234096, Tokens per sec: 170490.9613319382, Loss: 2.4985203742980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22243 , TFLOPS: 69.50391611777547, Tokens per sec: 172238.36062826822, Loss: 2.467494487762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22244 , TFLOPS: 70.67103661522563, Tokens per sec: 175130.61378988615, Loss: 2.4807300567626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22245 , TFLOPS: 70.60977383202675, Tokens per sec: 174978.79786446661, Loss: 2.515256643295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22246 , TFLOPS: 69.13886967309328, Tokens per sec: 171333.73532516017, Loss: 2.5113778114318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22247 , TFLOPS: 68.60342767495806, Tokens per sec: 170006.85106997573, Loss: 2.502021074295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22248 , TFLOPS: 70.53626128876436, Tokens per sec: 174796.6256841873, Loss: 2.4962005615234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22249 , TFLOPS: 69.68326184480934, Tokens per sec: 172682.7990964209, Loss: 2.5048458576202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22250 , TFLOPS: 69.81205534824068, Tokens per sec: 173001.9635856994, Loss: 2.490467071533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22251 , TFLOPS: 69.67442502919405, Tokens per sec: 172660.9004938706, Loss: 2.489715576171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22252 , TFLOPS: 69.62212103880974, Tokens per sec: 172531.2854439381, Loss: 2.498797655105591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22253 , TFLOPS: 70.73743054895724, Tokens per sec: 175295.14527156862, Loss: 2.5057952404022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22254 , TFLOPS: 69.7722442509465, Tokens per sec: 172903.30730105002, Loss: 2.4901163578033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22255 , TFLOPS: 69.67291274211401, Tokens per sec: 172657.1528798941, Loss: 2.486459255218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22256 , TFLOPS: 69.74503704915807, Tokens per sec: 172835.88485789762, Loss: 2.493757724761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22257 , TFLOPS: 70.71208017719053, Tokens per sec: 175232.3242577568, Loss: 2.476201057434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22258 , TFLOPS: 68.79613225374933, Tokens per sec: 170484.39424438233, Loss: 2.4872114658355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22259 , TFLOPS: 69.05121713871083, Tokens per sec: 171116.52268923668, Loss: 2.4920756816864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22260 , TFLOPS: 70.72090233984909, Tokens per sec: 175254.18654866613, Loss: 2.4893405437469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22261 , TFLOPS: 70.61797634872867, Tokens per sec: 174999.1246044358, Loss: 2.477141857147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22262 , TFLOPS: 68.66700938233362, Tokens per sec: 170164.41354495566, Loss: 2.5021603107452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22263 , TFLOPS: 68.93915545249682, Tokens per sec: 170838.82148618455, Loss: 2.487514019012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22264 , TFLOPS: 70.64072217997122, Tokens per sec: 175055.491279915, Loss: 2.485344171524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22265 , TFLOPS: 69.66834002463388, Tokens per sec: 172645.82118225194, Loss: 2.4903597831726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22266 , TFLOPS: 69.09403129819843, Tokens per sec: 171222.620892816, Loss: 2.476238965988159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22267 , TFLOPS: 69.74134100501703, Tokens per sec: 172826.72565336325, Loss: 2.51035213470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22268 , TFLOPS: 71.54408774171235, Tokens per sec: 177294.13065584123, Loss: 2.473587989807129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22269 , TFLOPS: 68.9341257205884, Tokens per sec: 170826.35725644478, Loss: 2.47776460647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22270 , TFLOPS: 69.74994354452777, Tokens per sec: 172848.0436939188, Loss: 2.50166654586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22271 , TFLOPS: 70.69997943950806, Tokens per sec: 175202.3372967733, Loss: 2.5127029418945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22272 , TFLOPS: 69.68257957833131, Tokens per sec: 172681.1083649305, Loss: 2.4902594089508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22273 , TFLOPS: 70.49054575557774, Tokens per sec: 174683.3375569691, Loss: 2.488370180130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22274 , TFLOPS: 69.70110342007064, Tokens per sec: 172727.01248533017, Loss: 2.50311541557312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22275 , TFLOPS: 70.57460931790541, Tokens per sec: 174891.65632478296, Loss: 2.485626697540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22276 , TFLOPS: 68.15126802008376, Tokens per sec: 168886.3496356994, Loss: 2.482848882675171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22277 , TFLOPS: 70.60830877126247, Tokens per sec: 174975.16728250298, Loss: 2.4766204357147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22278 , TFLOPS: 69.81903435273796, Tokens per sec: 173019.2583276445, Loss: 2.486755132675171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22279 , TFLOPS: 69.76277007183938, Tokens per sec: 172879.82924155562, Loss: 2.472748041152954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22280 , TFLOPS: 68.97968224596971, Tokens per sec: 170939.25134480526, Loss: 2.49652361869812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22281 , TFLOPS: 70.57614066244136, Tokens per sec: 174895.45116523458, Loss: 2.4773941040039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22282 , TFLOPS: 69.7380642317589, Tokens per sec: 172818.6054482632, Loss: 2.487685203552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22283 , TFLOPS: 69.76094308009829, Tokens per sec: 172875.3017547614, Loss: 2.4787991046905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22284 , TFLOPS: 68.99765081785502, Tokens per sec: 170983.77944533862, Loss: 2.493211507797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22285 , TFLOPS: 70.70729772853667, Tokens per sec: 175220.4728231623, Loss: 2.497211456298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22286 , TFLOPS: 69.1129338171427, Tokens per sec: 171269.46341704423, Loss: 2.496640682220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22287 , TFLOPS: 70.62237739135107, Tokens per sec: 175010.03087286899, Loss: 2.4859731197357178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22288 , TFLOPS: 69.90511436398894, Tokens per sec: 173232.57407802047, Loss: 2.479539394378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22289 , TFLOPS: 68.91510953435979, Tokens per sec: 170779.23305216446, Loss: 2.4790854454040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22290 , TFLOPS: 69.84938724720489, Tokens per sec: 173094.4761437809, Loss: 2.504321575164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22291 , TFLOPS: 70.4338293757377, Tokens per sec: 174542.78811990054, Loss: 2.4939675331115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22292 , TFLOPS: 70.60166436643227, Tokens per sec: 174958.70171539215, Loss: 2.5058815479278564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22293 , TFLOPS: 69.94954756439637, Tokens per sec: 173342.68444335117, Loss: 2.504420042037964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22294 , TFLOPS: 69.70145476287347, Tokens per sec: 172727.8831514992, Loss: 2.4845709800720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22295 , TFLOPS: 69.59515033667046, Tokens per sec: 172464.44907296888, Loss: 2.506636142730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22296 , TFLOPS: 69.6986985409087, Tokens per sec: 172721.0529298476, Loss: 2.4982614517211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22297 , TFLOPS: 70.17532520246642, Tokens per sec: 173902.1863019488, Loss: 2.5035006999969482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22298 , TFLOPS: 69.7694758723673, Tokens per sec: 172896.44695396867, Loss: 2.51436185836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22299 , TFLOPS: 68.79087453851656, Tokens per sec: 170471.36504685038, Loss: 2.4985451698303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22300 , TFLOPS: 69.76279423876063, Tokens per sec: 172879.88912984869, Loss: 2.4763686656951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22301 , TFLOPS: 70.63423956401677, Tokens per sec: 175039.42664346076, Loss: 2.4912190437316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22302 , TFLOPS: 69.78534632518972, Tokens per sec: 172935.7756843374, Loss: 2.463428258895874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22303 , TFLOPS: 68.75335725058093, Tokens per sec: 170378.3930744754, Loss: 2.460536241531372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22304 , TFLOPS: 70.62775176861643, Tokens per sec: 175023.34917176905, Loss: 2.464001178741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22305 , TFLOPS: 70.09386049369667, Tokens per sec: 173700.30777953882, Loss: 2.486405611038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22306 , TFLOPS: 68.83720271150163, Tokens per sec: 170586.17136297692, Loss: 2.479942798614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22307 , TFLOPS: 70.66936323016917, Tokens per sec: 175126.46695737768, Loss: 2.4857583045959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22308 , TFLOPS: 70.63677440825448, Tokens per sec: 175045.70826671796, Loss: 2.5095012187957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22309 , TFLOPS: 68.86761001033891, Tokens per sec: 170661.52399913617, Loss: 2.494689702987671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22310 , TFLOPS: 68.76949344993197, Tokens per sec: 170418.38035401644, Loss: 2.4878172874450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22311 , TFLOPS: 70.65197839495234, Tokens per sec: 175083.38542061267, Loss: 2.476533889770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22312 , TFLOPS: 70.63732691147902, Tokens per sec: 175047.07743057163, Loss: 2.499973773956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22313 , TFLOPS: 68.70524201507743, Tokens per sec: 170259.158220567, Loss: 2.4880712032318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22314 , TFLOPS: 69.79916588816047, Tokens per sec: 172970.02208372945, Loss: 2.4731509685516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22315 , TFLOPS: 70.58775603466621, Tokens per sec: 174924.23533714665, Loss: 2.475844383239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22316 , TFLOPS: 69.84508126156997, Tokens per sec: 173083.80543703408, Loss: 2.4817569255828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22317 , TFLOPS: 68.7384922627727, Tokens per sec: 170341.555997173, Loss: 2.4927897453308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22318 , TFLOPS: 70.67597355955928, Tokens per sec: 175142.84808179428, Loss: 2.515047073364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22319 , TFLOPS: 69.01075659221117, Tokens per sec: 171016.25699791338, Loss: 2.4445786476135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22320 , TFLOPS: 70.68356168667333, Tokens per sec: 175161.65229668407, Loss: 2.478783130645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22321 , TFLOPS: 69.07485551265029, Tokens per sec: 171175.10118384054, Loss: 2.453810453414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22322 , TFLOPS: 69.88913254601887, Tokens per sec: 173192.96937254805, Loss: 2.503121852874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22323 , TFLOPS: 69.76080286526619, Tokens per sec: 172874.95428696153, Loss: 2.4723193645477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22324 , TFLOPS: 70.55910053918363, Tokens per sec: 174853.22386267738, Loss: 2.48829984664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22325 , TFLOPS: 69.55659518354835, Tokens per sec: 172368.90515633154, Loss: 2.476984977722168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22326 , TFLOPS: 68.95174235085145, Tokens per sec: 170870.01320686864, Loss: 2.4720659255981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22327 , TFLOPS: 70.6438256096026, Tokens per sec: 175063.18192041255, Loss: 2.4805707931518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22328 , TFLOPS: 69.65016837288091, Tokens per sec: 172600.7898274354, Loss: 2.5198416709899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22329 , TFLOPS: 69.8980765220445, Tokens per sec: 173215.13353039985, Loss: 2.501983404159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22330 , TFLOPS: 69.28966250668569, Tokens per sec: 171707.41657800405, Loss: 2.4957833290100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22331 , TFLOPS: 70.70529770477722, Tokens per sec: 175215.51654396558, Loss: 2.5228052139282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22332 , TFLOPS: 69.45962557680268, Tokens per sec: 172128.60378873177, Loss: 2.4823343753814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22333 , TFLOPS: 69.60657088506552, Tokens per sec: 172492.75044996908, Loss: 2.4740707874298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22334 , TFLOPS: 70.61177203346675, Tokens per sec: 174983.74962775502, Loss: 2.489006519317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22335 , TFLOPS: 70.98992686317659, Tokens per sec: 175920.8589529694, Loss: 2.4692838191986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22336 , TFLOPS: 68.7582329151133, Tokens per sec: 170390.47550828542, Loss: 2.4934754371643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22337 , TFLOPS: 69.90686020029871, Tokens per sec: 173236.9004527158, Loss: 2.4752204418182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22338 , TFLOPS: 70.57247646749411, Tokens per sec: 174886.3708865109, Loss: 2.4869275093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22339 , TFLOPS: 68.64715724804059, Tokens per sec: 170115.2177692844, Loss: 2.507700204849243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22340 , TFLOPS: 70.62616935256638, Tokens per sec: 175019.42777047952, Loss: 2.484908103942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22341 , TFLOPS: 69.77993622740883, Tokens per sec: 172922.36886606485, Loss: 2.5106348991394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22342 , TFLOPS: 70.63147038454355, Tokens per sec: 175032.56431167584, Loss: 2.4778285026550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22343 , TFLOPS: 69.08942719320451, Tokens per sec: 171211.211413457, Loss: 2.490562915802002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22344 , TFLOPS: 70.64664217627971, Tokens per sec: 175070.1616829089, Loss: 2.4781479835510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22345 , TFLOPS: 69.6231572536579, Tokens per sec: 172533.85329848033, Loss: 2.4692461490631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22346 , TFLOPS: 69.76189209659769, Tokens per sec: 172877.65352218974, Loss: 2.4980454444885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22347 , TFLOPS: 68.9561270162617, Tokens per sec: 170880.87889076603, Loss: 2.485848903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22348 , TFLOPS: 70.6799932557173, Tokens per sec: 175152.80933167995, Loss: 2.4967663288116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22349 , TFLOPS: 69.75149385005652, Tokens per sec: 172851.8855217993, Loss: 2.4931235313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22350 , TFLOPS: 68.01255890093573, Tokens per sec: 168542.61315837968, Loss: 2.4659206867218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22351 , TFLOPS: 70.53960741308072, Tokens per sec: 174804.91774884923, Loss: 2.503373384475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22352 , TFLOPS: 70.52665145195816, Tokens per sec: 174772.8114499706, Loss: 2.4732205867767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22353 , TFLOPS: 69.4354317692423, Tokens per sec: 172068.6488684292, Loss: 2.5059285163879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22354 , TFLOPS: 69.56070368546285, Tokens per sec: 172379.08647666452, Loss: 2.495798110961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22355 , TFLOPS: 70.69881990464077, Tokens per sec: 175199.46384163885, Loss: 2.4884605407714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22356 , TFLOPS: 69.00909643280104, Tokens per sec: 171012.14294001344, Loss: 2.4852375984191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22357 , TFLOPS: 69.83372420890517, Tokens per sec: 173055.66140944808, Loss: 2.503804922103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22358 , TFLOPS: 69.53642687637526, Tokens per sec: 172318.9258694342, Loss: 2.4809346199035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22359 , TFLOPS: 69.61905936015935, Tokens per sec: 172523.69826697072, Loss: 2.4821314811706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22360 , TFLOPS: 70.0968066655618, Tokens per sec: 173707.60871796825, Loss: 2.4912936687469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22361 , TFLOPS: 69.83571148480158, Tokens per sec: 173060.58609803586, Loss: 2.5078086853027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22362 , TFLOPS: 69.53534776066144, Tokens per sec: 172316.25170182093, Loss: 2.4992847442626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22363 , TFLOPS: 69.72842515785022, Tokens per sec: 172794.71876128585, Loss: 2.500283718109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22364 , TFLOPS: 70.03897739781657, Tokens per sec: 173564.3014220758, Loss: 2.482987642288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22365 , TFLOPS: 68.73911118630677, Tokens per sec: 170343.08975787053, Loss: 2.4804928302764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22366 , TFLOPS: 69.69259680091064, Tokens per sec: 172705.9321459702, Loss: 2.4816627502441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22367 , TFLOPS: 69.80853164050446, Tokens per sec: 172993.2314497606, Loss: 2.5058441162109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22368 , TFLOPS: 70.67455293193389, Tokens per sec: 175139.3276100435, Loss: 2.49696683883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22369 , TFLOPS: 69.50736908898425, Tokens per sec: 172246.91747129976, Loss: 2.5125579833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22370 , TFLOPS: 69.71610287115287, Tokens per sec: 172764.18277743788, Loss: 2.5112762451171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22371 , TFLOPS: 70.93613400301909, Tokens per sec: 175787.55431972636, Loss: 2.4836418628692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22372 , TFLOPS: 69.89406799194894, Tokens per sec: 173205.199951247, Loss: 2.490631341934204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22373 , TFLOPS: 68.80686387877653, Tokens per sec: 170510.98839338578, Loss: 2.4719083309173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22374 , TFLOPS: 70.65817069145093, Tokens per sec: 175098.73061347395, Loss: 2.5040078163146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22375 , TFLOPS: 70.6700973030975, Tokens per sec: 175128.2860709589, Loss: 2.48880934715271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22376 , TFLOPS: 68.81596374246084, Tokens per sec: 170533.53885802825, Loss: 2.484128713607788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22377 , TFLOPS: 68.60067139381911, Tokens per sec: 170000.020701684, Loss: 2.511063575744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22378 , TFLOPS: 70.5405460411616, Tokens per sec: 174807.24377263206, Loss: 2.4752821922302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22379 , TFLOPS: 70.31939978142856, Tokens per sec: 174259.218979742, Loss: 2.477339029312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22380 , TFLOPS: 68.42774624887116, Tokens per sec: 169571.49314322573, Loss: 2.492345094680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22381 , TFLOPS: 69.9459468024131, Tokens per sec: 173333.76135849993, Loss: 2.4797542095184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22382 , TFLOPS: 70.61846211329566, Tokens per sec: 175000.3283825442, Loss: 2.49859356880188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22383 , TFLOPS: 69.72589594930598, Tokens per sec: 172788.4511038979, Loss: 2.4898340702056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22384 , TFLOPS: 68.6556252872963, Tokens per sec: 170136.20250339157, Loss: 2.484229326248169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22385 , TFLOPS: 70.62865941826931, Tokens per sec: 175025.59842759478, Loss: 2.4945027828216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22386 , TFLOPS: 69.22136570445285, Tokens per sec: 171538.16957855725, Loss: 2.508434534072876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22387 , TFLOPS: 69.76310979183671, Tokens per sec: 172880.67110513223, Loss: 2.466973066329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22388 , TFLOPS: 70.08932058857712, Tokens per sec: 173689.05739454078, Loss: 2.4970996379852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22389 , TFLOPS: 70.64942967543982, Tokens per sec: 175077.069412896, Loss: 2.5001749992370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22390 , TFLOPS: 69.15442712396896, Tokens per sec: 171372.28840222477, Loss: 2.4852538108825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22391 , TFLOPS: 69.53691587641666, Tokens per sec: 172320.1376654046, Loss: 2.497015953063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22392 , TFLOPS: 69.61787014197277, Tokens per sec: 172520.75125330116, Loss: 2.4978673458099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22393 , TFLOPS: 70.49695058074063, Tokens per sec: 174699.2094192725, Loss: 2.4953596591949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22394 , TFLOPS: 70.12383236184087, Tokens per sec: 173774.581370474, Loss: 2.5012354850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22395 , TFLOPS: 69.68177992473588, Tokens per sec: 172679.12673523193, Loss: 2.493828773498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22396 , TFLOPS: 69.61661780182645, Tokens per sec: 172517.64781646192, Loss: 2.488071918487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22397 , TFLOPS: 69.14933784419758, Tokens per sec: 171359.67660632086, Loss: 2.4883813858032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22398 , TFLOPS: 70.59942530572103, Tokens per sec: 174953.15307629676, Loss: 2.4811785221099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22399 , TFLOPS: 69.54490694033787, Tokens per sec: 172339.9404020897, Loss: 2.48647403717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22400 , TFLOPS: 69.06417676159293, Tokens per sec: 171148.63806235886, Loss: 2.4884531497955322 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/model.pt +[rank0]:[2024-08-28 01:21:07,208] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006373818003339693, 'preprocessing_with_comm': 0.0012602010101545602, 'state_converting': 0.9822418480180204, : 0.9912434149882756}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022400, took 4.28s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020400 + eval ppl=9.81480884552002, eval loss=2.2838923931121826 +------------------------------------------------------------------ +iteration: 22401 , TFLOPS: 65.02842059417031, Tokens per sec: 161147.5897042431, Loss: 2.5049586296081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22402 , TFLOPS: 69.29964188823119, Tokens per sec: 171732.14658479343, Loss: 2.4824399948120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22403 , TFLOPS: 69.58418657383206, Tokens per sec: 172437.27966089713, Loss: 2.5037362575531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22404 , TFLOPS: 69.96090002901954, Tokens per sec: 173370.8171012646, Loss: 2.51053524017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22405 , TFLOPS: 70.62067112604561, Tokens per sec: 175005.80255948138, Loss: 2.4899237155914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22406 , TFLOPS: 68.70273976183043, Tokens per sec: 170252.95736137498, Loss: 2.4723386764526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22407 , TFLOPS: 70.67747646244608, Tokens per sec: 175146.57244070616, Loss: 2.487205743789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22408 , TFLOPS: 69.87799303666188, Tokens per sec: 173165.36444124315, Loss: 2.485806941986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22409 , TFLOPS: 68.69028637804706, Tokens per sec: 170222.0965045063, Loss: 2.492724657058716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22410 , TFLOPS: 69.90135414622202, Tokens per sec: 173223.25584417183, Loss: 2.4650957584381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22411 , TFLOPS: 70.57286194559613, Tokens per sec: 174887.3261437116, Loss: 2.4986469745635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22412 , TFLOPS: 69.53299224046789, Tokens per sec: 172310.41446329938, Loss: 2.5059053897857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22413 , TFLOPS: 68.56422280109707, Tokens per sec: 169909.69707377485, Loss: 2.5039196014404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22414 , TFLOPS: 70.65473926321881, Tokens per sec: 175090.22715631174, Loss: 2.489807367324829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22415 , TFLOPS: 70.58549010437815, Tokens per sec: 174918.62011227998, Loss: 2.500595808029175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22416 , TFLOPS: 68.88898462146874, Tokens per sec: 170714.49264012367, Loss: 2.4776792526245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22417 , TFLOPS: 69.5084354821143, Tokens per sec: 172249.5601109489, Loss: 2.5093460083007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22418 , TFLOPS: 70.19998517630347, Tokens per sec: 173963.29643363738, Loss: 2.491223096847534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22419 , TFLOPS: 69.78845442186103, Tokens per sec: 172943.47789027396, Loss: 2.499992847442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22420 , TFLOPS: 69.0751626466764, Tokens per sec: 171175.86229579084, Loss: 2.4578042030334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22421 , TFLOPS: 69.6852262306821, Tokens per sec: 172687.66706100857, Loss: 2.490481376647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22422 , TFLOPS: 70.55342963494412, Tokens per sec: 174839.17073727018, Loss: 2.499447822570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22423 , TFLOPS: 69.62004977328063, Tokens per sec: 172526.15261978825, Loss: 2.4880056381225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22424 , TFLOPS: 69.82213111364248, Tokens per sec: 173026.9324423023, Loss: 2.499208688735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22425 , TFLOPS: 69.85723311244271, Tokens per sec: 173113.91906213362, Loss: 2.4867188930511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22426 , TFLOPS: 69.8533957966039, Tokens per sec: 173104.40977076982, Loss: 2.4827041625976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22427 , TFLOPS: 70.78957651216095, Tokens per sec: 175424.3687127395, Loss: 2.4748148918151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22428 , TFLOPS: 68.67485587433335, Tokens per sec: 170183.85801649344, Loss: 2.4767343997955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22429 , TFLOPS: 69.30509332706451, Tokens per sec: 171745.65585074798, Loss: 2.479982852935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22430 , TFLOPS: 69.80514896632529, Tokens per sec: 172984.8488105123, Loss: 2.4699866771698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22431 , TFLOPS: 69.93059292660188, Tokens per sec: 173295.71276287068, Loss: 2.4919495582580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22432 , TFLOPS: 69.52493738443812, Tokens per sec: 172290.45364274192, Loss: 2.485201597213745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22433 , TFLOPS: 69.76102891558935, Tokens per sec: 172875.51446456384, Loss: 2.5027735233306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22434 , TFLOPS: 71.55937163221242, Tokens per sec: 177332.00582015992, Loss: 2.50120210647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22435 , TFLOPS: 68.95291124588336, Tokens per sec: 170872.9098575221, Loss: 2.490786075592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22436 , TFLOPS: 68.87005673861461, Tokens per sec: 170667.58726133272, Loss: 2.479614496231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22437 , TFLOPS: 70.61435505991442, Tokens per sec: 174990.15065183616, Loss: 2.488632917404175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22438 , TFLOPS: 69.93138176655526, Tokens per sec: 173297.6675951731, Loss: 2.492988109588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22439 , TFLOPS: 69.53912462864754, Tokens per sec: 172325.61119674685, Loss: 2.4950883388519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22440 , TFLOPS: 69.25288232318508, Tokens per sec: 171616.27123161787, Loss: 2.4880452156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22441 , TFLOPS: 70.60189165263519, Tokens per sec: 174959.2649556406, Loss: 2.490755081176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22442 , TFLOPS: 69.4612294476473, Tokens per sec: 172132.57835736568, Loss: 2.484414577484131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22443 , TFLOPS: 68.17527973066974, Tokens per sec: 168945.8532996397, Loss: 2.474221706390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22444 , TFLOPS: 69.80122426705088, Tokens per sec: 172975.12297336903, Loss: 2.515068292617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22445 , TFLOPS: 70.77020059611144, Tokens per sec: 175376.3530583355, Loss: 2.474679708480835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22446 , TFLOPS: 69.13273791930892, Tokens per sec: 171318.5401638141, Loss: 2.4946014881134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22447 , TFLOPS: 69.75833465398794, Tokens per sec: 172868.83778751813, Loss: 2.5024523735046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22448 , TFLOPS: 69.72320642101909, Tokens per sec: 172781.786156527, Loss: 2.4887094497680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22449 , TFLOPS: 70.64276371105548, Tokens per sec: 175060.55041883502, Loss: 2.4864227771759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22450 , TFLOPS: 68.03115247522948, Tokens per sec: 168588.69008373088, Loss: 2.4831273555755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22451 , TFLOPS: 70.653067791127, Tokens per sec: 175086.08506434038, Loss: 2.4980337619781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22452 , TFLOPS: 69.8150782910239, Tokens per sec: 173009.45477092083, Loss: 2.4931399822235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22453 , TFLOPS: 70.54245759510214, Tokens per sec: 174811.98081387178, Loss: 2.501497745513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22454 , TFLOPS: 69.97388534662521, Tokens per sec: 173402.99614874227, Loss: 2.4646077156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22455 , TFLOPS: 69.8114705550451, Tokens per sec: 173000.5144037405, Loss: 2.4964330196380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22456 , TFLOPS: 69.6389466292638, Tokens per sec: 172572.98111057509, Loss: 2.487607002258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22457 , TFLOPS: 68.90018802213196, Tokens per sec: 170742.25590112363, Loss: 2.480262279510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22458 , TFLOPS: 70.55390997058568, Tokens per sec: 174840.3610619033, Loss: 2.5045361518859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22459 , TFLOPS: 70.09927773484442, Tokens per sec: 173713.73229986135, Loss: 2.492191791534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22460 , TFLOPS: 69.08434642963569, Tokens per sec: 171198.62072164114, Loss: 2.4774744510650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22461 , TFLOPS: 70.43423900618033, Tokens per sec: 174543.80322926186, Loss: 2.4867172241210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22462 , TFLOPS: 70.53718989500595, Tokens per sec: 174798.9268727484, Loss: 2.487616539001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22463 , TFLOPS: 69.15506431915676, Tokens per sec: 171373.86744209303, Loss: 2.473426342010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22464 , TFLOPS: 69.72256842772029, Tokens per sec: 172780.2051388518, Loss: 2.492140769958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22465 , TFLOPS: 69.67042653291426, Tokens per sec: 172650.9917796186, Loss: 2.4908595085144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22466 , TFLOPS: 68.96660723506379, Tokens per sec: 170906.850027449, Loss: 2.497659683227539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22467 , TFLOPS: 70.75346191977374, Tokens per sec: 175334.87277445325, Loss: 2.4733805656433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22468 , TFLOPS: 69.7679900166138, Tokens per sec: 172892.76483973098, Loss: 2.4919304847717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22469 , TFLOPS: 69.64971910254633, Tokens per sec: 172599.67648605508, Loss: 2.499446153640747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22470 , TFLOPS: 68.84017066608958, Tokens per sec: 170593.52628139302, Loss: 2.4771766662597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22471 , TFLOPS: 70.63660320707844, Tokens per sec: 175045.28401134437, Loss: 2.487713575363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22472 , TFLOPS: 68.82869440531002, Tokens per sec: 170565.08684296682, Loss: 2.480283737182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22473 , TFLOPS: 70.53736809516248, Tokens per sec: 174799.36847236662, Loss: 2.462348699569702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22474 , TFLOPS: 70.7211048312132, Tokens per sec: 175254.68834457264, Loss: 2.4618191719055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22475 , TFLOPS: 69.62625043411091, Tokens per sec: 172541.51854038486, Loss: 2.512995719909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22476 , TFLOPS: 68.82803542749839, Tokens per sec: 170563.4538233572, Loss: 2.48327374458313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22477 , TFLOPS: 69.78359968239666, Tokens per sec: 172931.44731108754, Loss: 2.4830424785614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22478 , TFLOPS: 70.63020757284578, Tokens per sec: 175029.43492517856, Loss: 2.474684715270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22479 , TFLOPS: 69.82159932805722, Tokens per sec: 173025.61461904095, Loss: 2.489384651184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22480 , TFLOPS: 68.01249864130511, Tokens per sec: 168542.46382837687, Loss: 2.4790561199188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22481 , TFLOPS: 70.50104201628358, Tokens per sec: 174709.34844725678, Loss: 2.494266986846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22482 , TFLOPS: 70.59131510144768, Tokens per sec: 174933.05509669465, Loss: 2.495934009552002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22483 , TFLOPS: 67.9492382375852, Tokens per sec: 168385.69757925018, Loss: 2.5076546669006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22484 , TFLOPS: 70.67018763566536, Tokens per sec: 175128.509925013, Loss: 2.495624303817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22485 , TFLOPS: 70.14502647879361, Tokens per sec: 173827.1027270076, Loss: 2.4930806159973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22486 , TFLOPS: 69.91072670237955, Tokens per sec: 173246.4820708025, Loss: 2.488121747970581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22487 , TFLOPS: 69.99429436942401, Tokens per sec: 173453.57195547715, Loss: 2.5043935775756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22488 , TFLOPS: 70.5575861168342, Tokens per sec: 174849.47095726812, Loss: 2.4940030574798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22489 , TFLOPS: 69.84113685023058, Tokens per sec: 173074.03075122228, Loss: 2.4991862773895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22490 , TFLOPS: 69.42673604496481, Tokens per sec: 172047.09990575654, Loss: 2.4857687950134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22491 , TFLOPS: 69.04017931296615, Tokens per sec: 171089.16974112447, Loss: 2.48391056060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22492 , TFLOPS: 70.69424535378971, Tokens per sec: 175188.12760070155, Loss: 2.487090587615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22493 , TFLOPS: 69.37189804825033, Tokens per sec: 171911.20530899402, Loss: 2.4847474098205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22494 , TFLOPS: 69.76555100938229, Tokens per sec: 172886.7207111325, Loss: 2.491114377975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22495 , TFLOPS: 69.676676628454, Tokens per sec: 172666.48020487092, Loss: 2.4817540645599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22496 , TFLOPS: 69.70217830813564, Tokens per sec: 172729.67617636395, Loss: 2.4876255989074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22497 , TFLOPS: 69.85111947242905, Tokens per sec: 173098.76878870671, Loss: 2.4856455326080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22498 , TFLOPS: 69.6832012394304, Tokens per sec: 172682.64890961567, Loss: 2.5074260234832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22499 , TFLOPS: 69.89565788398207, Tokens per sec: 173209.13987884615, Loss: 2.5094995498657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22500 , TFLOPS: 68.86594639378902, Tokens per sec: 170657.40137406305, Loss: 2.488616466522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22501 , TFLOPS: 71.54594425839805, Tokens per sec: 177298.7313087006, Loss: 2.4947357177734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22502 , TFLOPS: 69.08403827627197, Tokens per sec: 171197.85708365997, Loss: 2.4933021068573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22503 , TFLOPS: 68.82935099747748, Tokens per sec: 170566.71395068735, Loss: 2.4794719219207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22504 , TFLOPS: 70.7790996342944, Tokens per sec: 175398.4058552628, Loss: 2.486997127532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22505 , TFLOPS: 69.89535495348593, Tokens per sec: 173208.3891837062, Loss: 2.4770164489746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22506 , TFLOPS: 69.63645613903756, Tokens per sec: 172566.80940144433, Loss: 2.4863357543945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22507 , TFLOPS: 69.3029282284421, Tokens per sec: 171740.29049785633, Loss: 2.499056339263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22508 , TFLOPS: 70.63372342701403, Tokens per sec: 175038.1475991108, Loss: 2.5158851146698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22509 , TFLOPS: 68.69612454958744, Tokens per sec: 170236.56413671104, Loss: 2.474144697189331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22510 , TFLOPS: 69.78554025708699, Tokens per sec: 172936.25626894218, Loss: 2.474653959274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22511 , TFLOPS: 69.90674344341298, Tokens per sec: 173236.61111629114, Loss: 2.4729433059692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22512 , TFLOPS: 70.69783275490319, Tokens per sec: 175197.01757584562, Loss: 2.502249002456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22513 , TFLOPS: 68.72594145704963, Tokens per sec: 170310.45371800588, Loss: 2.4897348880767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22514 , TFLOPS: 70.65029161943927, Tokens per sec: 175079.20540507764, Loss: 2.4807326793670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22515 , TFLOPS: 69.78730349105511, Tokens per sec: 172940.6257569514, Loss: 2.448892831802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22516 , TFLOPS: 69.13611026710541, Tokens per sec: 171326.89721314856, Loss: 2.4781806468963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22517 , TFLOPS: 68.22434227973079, Tokens per sec: 169067.43570089893, Loss: 2.4784183502197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22518 , TFLOPS: 70.72744345788867, Tokens per sec: 175270.39615973184, Loss: 2.4786384105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22519 , TFLOPS: 69.73799380251359, Tokens per sec: 172818.4309168349, Loss: 2.482708215713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22520 , TFLOPS: 68.93372759966867, Tokens per sec: 170825.37066894906, Loss: 2.4849655628204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22521 , TFLOPS: 70.48318633096662, Tokens per sec: 174665.1000920752, Loss: 2.4866080284118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22522 , TFLOPS: 69.7945171282141, Tokens per sec: 172958.50194447913, Loss: 2.4706685543060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22523 , TFLOPS: 69.80110759573189, Tokens per sec: 172974.83384898817, Loss: 2.508019208908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22524 , TFLOPS: 69.00740092356259, Tokens per sec: 171007.94128134477, Loss: 2.4982283115386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22525 , TFLOPS: 70.70818480664641, Tokens per sec: 175222.67110043796, Loss: 2.4797983169555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22526 , TFLOPS: 69.95511907652701, Tokens per sec: 173356.49126416363, Loss: 2.486863613128662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22527 , TFLOPS: 68.93604387737282, Tokens per sec: 170831.110660259, Loss: 2.5183236598968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22528 , TFLOPS: 69.67648361311934, Tokens per sec: 172666.00189160922, Loss: 2.502621650695801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22529 , TFLOPS: 70.72463741294845, Tokens per sec: 175263.4424712586, Loss: 2.4661026000976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22530 , TFLOPS: 69.19536309177622, Tokens per sec: 171473.73223992027, Loss: 2.4775335788726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22531 , TFLOPS: 69.65088121434869, Tokens per sec: 172602.55632711854, Loss: 2.498522996902466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22532 , TFLOPS: 69.95658749366584, Tokens per sec: 173360.13016359313, Loss: 2.4912948608398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22533 , TFLOPS: 69.63936197866147, Tokens per sec: 172574.01039213702, Loss: 2.5005548000335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22534 , TFLOPS: 70.63042444263849, Tokens per sec: 175029.97235241506, Loss: 2.4799160957336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22535 , TFLOPS: 69.76587938457831, Tokens per sec: 172887.53446104174, Loss: 2.490410566329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22536 , TFLOPS: 70.59067995571881, Tokens per sec: 174931.48113561145, Loss: 2.4910717010498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22537 , TFLOPS: 69.10189925215583, Tokens per sec: 171242.1185494491, Loss: 2.4938156604766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22538 , TFLOPS: 70.58973342307152, Tokens per sec: 174929.1355234423, Loss: 2.4706130027770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22539 , TFLOPS: 68.76190128300163, Tokens per sec: 170399.5661280168, Loss: 2.4850356578826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22540 , TFLOPS: 69.65990772146924, Tokens per sec: 172624.92500611505, Loss: 2.489980936050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22541 , TFLOPS: 71.05804832000372, Tokens per sec: 176089.67142718498, Loss: 2.4713284969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22542 , TFLOPS: 69.95923235296766, Tokens per sec: 173366.68441629835, Loss: 2.4656474590301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22543 , TFLOPS: 68.8695423485268, Tokens per sec: 170666.3125460304, Loss: 2.5007967948913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22544 , TFLOPS: 69.79510341309845, Tokens per sec: 172959.95482300714, Loss: 2.4819860458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22545 , TFLOPS: 70.63303860197377, Tokens per sec: 175036.4505272213, Loss: 2.473299980163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22546 , TFLOPS: 68.77503692151991, Tokens per sec: 170432.11768727514, Loss: 2.485252618789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22547 , TFLOPS: 68.96801123929542, Tokens per sec: 170910.32930459882, Loss: 2.51672625541687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22548 , TFLOPS: 70.72704298550961, Tokens per sec: 175269.40374506084, Loss: 2.482579469680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22549 , TFLOPS: 70.72803751795249, Tokens per sec: 175271.8683060111, Loss: 2.4754064083099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22550 , TFLOPS: 67.92509955166057, Tokens per sec: 168325.87925643314, Loss: 2.4896140098571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22551 , TFLOPS: 70.63538123413169, Tokens per sec: 175042.25582777086, Loss: 2.4884915351867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22552 , TFLOPS: 69.47527822526747, Tokens per sec: 172167.3927759091, Loss: 2.489241600036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22553 , TFLOPS: 70.5828962690704, Tokens per sec: 174912.1923026528, Loss: 2.481398105621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22554 , TFLOPS: 69.57243880088733, Tokens per sec: 172408.16738541605, Loss: 2.5142221450805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22555 , TFLOPS: 69.71361926089386, Tokens per sec: 172758.0281176238, Loss: 2.4929277896881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22556 , TFLOPS: 70.62315262727142, Tokens per sec: 175011.9519928789, Loss: 2.5063672065734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22557 , TFLOPS: 68.83079888637285, Tokens per sec: 170570.30197886823, Loss: 2.4869580268859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22558 , TFLOPS: 70.0547126046039, Tokens per sec: 173603.29499787098, Loss: 2.4828031063079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22559 , TFLOPS: 69.66192029585666, Tokens per sec: 172629.91238715049, Loss: 2.494774103164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22560 , TFLOPS: 69.74749246637673, Tokens per sec: 172841.96965225198, Loss: 2.506920576095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22561 , TFLOPS: 69.71337318533729, Tokens per sec: 172757.41831528716, Loss: 2.496168851852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22562 , TFLOPS: 69.82820696725169, Tokens per sec: 173041.9890768564, Loss: 2.4815547466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22563 , TFLOPS: 69.87119556122876, Tokens per sec: 173148.51954831614, Loss: 2.4971718788146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22564 , TFLOPS: 69.2591489794023, Tokens per sec: 171631.80069605328, Loss: 2.471562147140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22565 , TFLOPS: 70.19813373338822, Tokens per sec: 173958.7083541401, Loss: 2.4917752742767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22566 , TFLOPS: 69.70314028978831, Tokens per sec: 172732.06007287028, Loss: 2.4753787517547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22567 , TFLOPS: 68.95929076662277, Tokens per sec: 170888.71901267665, Loss: 2.4812889099121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22568 , TFLOPS: 70.60597093852124, Tokens per sec: 174969.3738754367, Loss: 2.497110366821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22569 , TFLOPS: 70.17879273361, Tokens per sec: 173910.77922610263, Loss: 2.4875547885894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22570 , TFLOPS: 69.0400655683585, Tokens per sec: 171088.88786945675, Loss: 2.480327606201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22571 , TFLOPS: 70.58962865191933, Tokens per sec: 174928.87588898567, Loss: 2.494126319885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22572 , TFLOPS: 68.60106011253923, Tokens per sec: 170000.98398949334, Loss: 2.4827842712402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22573 , TFLOPS: 70.54902910586787, Tokens per sec: 174828.26574146096, Loss: 2.50228214263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22574 , TFLOPS: 69.72064035124146, Tokens per sec: 172775.4271529421, Loss: 2.495772361755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22575 , TFLOPS: 70.66942124296527, Tokens per sec: 175126.61071947703, Loss: 2.4886250495910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22576 , TFLOPS: 69.21316631098472, Tokens per sec: 171517.8505783055, Loss: 2.4900827407836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22577 , TFLOPS: 69.74528019722194, Tokens per sec: 172836.48740558486, Loss: 2.4879701137542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22578 , TFLOPS: 69.67949168962672, Tokens per sec: 172673.4562365615, Loss: 2.501100540161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22579 , TFLOPS: 70.57597951120812, Tokens per sec: 174895.0518147263, Loss: 2.5049667358398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22580 , TFLOPS: 68.0135478800203, Tokens per sec: 168545.06395749628, Loss: 2.476229667663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22581 , TFLOPS: 70.7984238081843, Tokens per sec: 175446.2932868955, Loss: 2.4790995121002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22582 , TFLOPS: 69.83776020026275, Tokens per sec: 173065.6630406333, Loss: 2.488030433654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22583 , TFLOPS: 69.59449201555361, Tokens per sec: 172462.81768072126, Loss: 2.4731619358062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22584 , TFLOPS: 69.80711365824484, Tokens per sec: 172989.71753351766, Loss: 2.4877874851226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22585 , TFLOPS: 70.6350240978322, Tokens per sec: 175041.37080467894, Loss: 2.490675210952759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22586 , TFLOPS: 69.66046770590098, Tokens per sec: 172626.31270922432, Loss: 2.506553888320923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22587 , TFLOPS: 68.39037833409814, Tokens per sec: 169478.89133400255, Loss: 2.488642454147339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22588 , TFLOPS: 71.516231063431, Tokens per sec: 177225.09873839357, Loss: 2.473020076751709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22589 , TFLOPS: 69.77169123073733, Tokens per sec: 172901.93685605153, Loss: 2.474024772644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22590 , TFLOPS: 69.89546035914105, Tokens per sec: 173208.65039053097, Loss: 2.472867965698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22591 , TFLOPS: 69.61396931957378, Tokens per sec: 172511.0845856854, Loss: 2.48745059967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22592 , TFLOPS: 70.69759380484476, Tokens per sec: 175196.4254312783, Loss: 2.4788379669189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22593 , TFLOPS: 69.92404723047355, Tokens per sec: 173279.49180679672, Loss: 2.480987548828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22594 , TFLOPS: 69.68149523411137, Tokens per sec: 172678.4212405031, Loss: 2.4793894290924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22595 , TFLOPS: 69.49425757249384, Tokens per sec: 172214.4256890837, Loss: 2.4964444637298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22596 , TFLOPS: 70.66148128481727, Tokens per sec: 175106.93462852767, Loss: 2.4824814796447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22597 , TFLOPS: 69.29053456622133, Tokens per sec: 171709.57763759865, Loss: 2.4996609687805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22598 , TFLOPS: 68.63382463399336, Tokens per sec: 170082.17808296668, Loss: 2.4811558723449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22599 , TFLOPS: 70.54715909345332, Tokens per sec: 174823.6316446988, Loss: 2.4963767528533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22600 , TFLOPS: 68.86714182651966, Tokens per sec: 170660.36378805735, Loss: 2.527012586593628 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/model.pt +[rank0]:[2024-08-28 01:46:32,046] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006344117980916053, 'preprocessing_with_comm': 0.001226224994752556, 'state_converting': 0.9856232210004237, : 0.9945845200272743}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022600, took 4.32s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020600 + eval ppl=10.571202278137207, eval loss=2.358133554458618 +------------------------------------------------------------------ +iteration: 22601 , TFLOPS: 66.21537245970927, Tokens per sec: 164088.98718059063, Loss: 2.490543842315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22602 , TFLOPS: 69.96125861804056, Tokens per sec: 173371.7057243605, Loss: 2.5025157928466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22603 , TFLOPS: 69.30908532960383, Tokens per sec: 171755.5484727958, Loss: 2.4786436557769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22604 , TFLOPS: 69.81024089857107, Tokens per sec: 172997.46717953996, Loss: 2.4812896251678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22605 , TFLOPS: 70.51788181678246, Tokens per sec: 174751.07932795014, Loss: 2.467297077178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22606 , TFLOPS: 68.04244454665135, Tokens per sec: 168616.67308063863, Loss: 2.5001025199890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22607 , TFLOPS: 68.9274010234459, Tokens per sec: 170809.6927161391, Loss: 2.480961322784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22608 , TFLOPS: 70.65793215131191, Tokens per sec: 175098.1394847321, Loss: 2.476125717163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22609 , TFLOPS: 68.7108456153576, Tokens per sec: 170273.0445593488, Loss: 2.498548984527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22610 , TFLOPS: 69.8394562417135, Tokens per sec: 173069.86601818269, Loss: 2.494483709335327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22611 , TFLOPS: 70.84817753762897, Tokens per sec: 175569.5885092851, Loss: 2.4950318336486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22612 , TFLOPS: 69.59950603944813, Tokens per sec: 172475.2429842722, Loss: 2.5040674209594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22613 , TFLOPS: 68.10732697529758, Tokens per sec: 168777.4588862135, Loss: 2.4734463691711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22614 , TFLOPS: 69.80747672928443, Tokens per sec: 172990.61726354933, Loss: 2.497013568878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22615 , TFLOPS: 70.68948668522202, Tokens per sec: 175176.3350957802, Loss: 2.4822356700897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22616 , TFLOPS: 69.4935202948624, Tokens per sec: 172212.5986338952, Loss: 2.489210605621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22617 , TFLOPS: 69.99617688577968, Tokens per sec: 173458.2370383828, Loss: 2.4924581050872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22618 , TFLOPS: 71.48557638399865, Tokens per sec: 177149.13306586855, Loss: 2.4965527057647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22619 , TFLOPS: 68.79387382620986, Tokens per sec: 170478.79761215317, Loss: 2.4942233562469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22620 , TFLOPS: 69.39596113504687, Tokens per sec: 171970.83628884368, Loss: 2.488783836364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22621 , TFLOPS: 70.62898961805175, Tokens per sec: 175026.41669903017, Loss: 2.5012128353118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22622 , TFLOPS: 70.62869755897708, Tokens per sec: 175025.69294447018, Loss: 2.4929723739624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22623 , TFLOPS: 69.93010489459964, Tokens per sec: 173294.50336580796, Loss: 2.482135772705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22624 , TFLOPS: 68.89945459858131, Tokens per sec: 170740.43839677187, Loss: 2.504528522491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22625 , TFLOPS: 70.58759014366383, Tokens per sec: 174923.82424096833, Loss: 2.4918525218963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22626 , TFLOPS: 69.86423766465269, Tokens per sec: 173131.27711412509, Loss: 2.4711389541625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22627 , TFLOPS: 69.8114714386155, Tokens per sec: 173000.51659332524, Loss: 2.4847514629364014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22628 , TFLOPS: 69.06338144545438, Tokens per sec: 171146.66718135623, Loss: 2.524684190750122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22629 , TFLOPS: 69.71639577645752, Tokens per sec: 172764.908629049, Loss: 2.491140365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22630 , TFLOPS: 70.62476529107548, Tokens per sec: 175015.94835143493, Loss: 2.4868009090423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22631 , TFLOPS: 69.34053006895579, Tokens per sec: 171833.47200083392, Loss: 2.4847421646118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22632 , TFLOPS: 69.84196108914279, Tokens per sec: 173076.07330604416, Loss: 2.4957022666931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22633 , TFLOPS: 70.56208333702575, Tokens per sec: 174860.61556431247, Loss: 2.4771406650543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22634 , TFLOPS: 69.27077511023455, Tokens per sec: 171660.61152897967, Loss: 2.502440929412842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22635 , TFLOPS: 69.54417996976395, Tokens per sec: 172338.1388889251, Loss: 2.470301628112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22636 , TFLOPS: 70.45980807261797, Tokens per sec: 174607.16619255953, Loss: 2.482506513595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22637 , TFLOPS: 69.7018815028733, Tokens per sec: 172728.94066022808, Loss: 2.472733736038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22638 , TFLOPS: 71.51642012760693, Tokens per sec: 177225.56726024873, Loss: 2.4830191135406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22639 , TFLOPS: 69.69635606175329, Tokens per sec: 172715.2480084551, Loss: 2.4849090576171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22640 , TFLOPS: 69.23884005642823, Tokens per sec: 171581.47294771104, Loss: 2.4879651069641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22641 , TFLOPS: 70.75330375892446, Tokens per sec: 175334.48083444586, Loss: 2.4891209602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22642 , TFLOPS: 69.47186041184, Tokens per sec: 172158.92305773258, Loss: 2.49936580657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22643 , TFLOPS: 69.64321349388979, Tokens per sec: 172583.55487115224, Loss: 2.511991500854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22644 , TFLOPS: 69.80211518678907, Tokens per sec: 172977.3307706231, Loss: 2.4870502948760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22645 , TFLOPS: 70.43624552076867, Tokens per sec: 174548.7755934476, Loss: 2.4781882762908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22646 , TFLOPS: 69.49000835637831, Tokens per sec: 172203.89566345964, Loss: 2.495018243789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22647 , TFLOPS: 69.60778698907491, Tokens per sec: 172495.76408966928, Loss: 2.494755268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22648 , TFLOPS: 70.76006146504916, Tokens per sec: 175351.22717464567, Loss: 2.4843854904174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22649 , TFLOPS: 69.13516946972581, Tokens per sec: 171324.56581360454, Loss: 2.4955344200134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22650 , TFLOPS: 69.47221395050839, Tokens per sec: 172159.79916549835, Loss: 2.4928364753723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22651 , TFLOPS: 70.64870631704804, Tokens per sec: 175075.27685111668, Loss: 2.487121105194092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22652 , TFLOPS: 70.58133900012635, Tokens per sec: 174908.33321866213, Loss: 2.476738929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22653 , TFLOPS: 70.11078322255338, Tokens per sec: 173742.2441658382, Loss: 2.4855215549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22654 , TFLOPS: 67.95147827319879, Tokens per sec: 168391.24863426093, Loss: 2.4778003692626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22655 , TFLOPS: 70.63194786333331, Tokens per sec: 175033.74755671548, Loss: 2.4853601455688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22656 , TFLOPS: 70.44878718368263, Tokens per sec: 174579.8552157266, Loss: 2.501833438873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22657 , TFLOPS: 68.75509174711868, Tokens per sec: 170382.69134796655, Loss: 2.4773740768432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22658 , TFLOPS: 71.49064002369505, Tokens per sec: 177161.6813228422, Loss: 2.4724698066711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22659 , TFLOPS: 69.75067381239072, Tokens per sec: 172849.85337812884, Loss: 2.505671501159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22660 , TFLOPS: 69.72867624365739, Tokens per sec: 172795.3409795754, Loss: 2.4837682247161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22661 , TFLOPS: 69.85812118550513, Tokens per sec: 173116.11980501164, Loss: 2.4802839756011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22662 , TFLOPS: 70.6338843522281, Tokens per sec: 175038.54638951863, Loss: 2.512044668197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22663 , TFLOPS: 69.57539620735157, Tokens per sec: 172415.49616441972, Loss: 2.488849639892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22664 , TFLOPS: 68.68025185522411, Tokens per sec: 170197.22985155712, Loss: 2.4737377166748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22665 , TFLOPS: 71.52662515690456, Tokens per sec: 177250.85644702503, Loss: 2.4765048027038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22666 , TFLOPS: 69.2438279397583, Tokens per sec: 171593.83347206368, Loss: 2.496878147125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22667 , TFLOPS: 69.72537276340204, Tokens per sec: 172787.1545915942, Loss: 2.4971609115600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22668 , TFLOPS: 69.78952914468576, Tokens per sec: 172946.14117182416, Loss: 2.4919722080230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22669 , TFLOPS: 69.6641463280456, Tokens per sec: 172635.4287401324, Loss: 2.4787492752075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22670 , TFLOPS: 69.9123437093768, Tokens per sec: 173250.4891922697, Loss: 2.506380796432495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22671 , TFLOPS: 70.62723124201399, Tokens per sec: 175022.0592495075, Loss: 2.506211519241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22672 , TFLOPS: 69.57039691927467, Tokens per sec: 172403.1073778481, Loss: 2.5101137161254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22673 , TFLOPS: 69.86392070747152, Tokens per sec: 173130.4916593144, Loss: 2.4961345195770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22674 , TFLOPS: 68.88048844131596, Tokens per sec: 170693.43816977364, Loss: 2.4924051761627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22675 , TFLOPS: 70.57165132062431, Tokens per sec: 174884.3260816701, Loss: 2.4966108798980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22676 , TFLOPS: 69.47622962388219, Tokens per sec: 172169.7504464816, Loss: 2.5009171962738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22677 , TFLOPS: 70.58306278734692, Tokens per sec: 174912.60495328554, Loss: 2.4926509857177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22678 , TFLOPS: 70.88268207033522, Tokens per sec: 175655.09454231928, Loss: 2.4779515266418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22679 , TFLOPS: 69.46001088632497, Tokens per sec: 172129.55862817334, Loss: 2.51346755027771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22680 , TFLOPS: 69.72493401879014, Tokens per sec: 172786.06733411428, Loss: 2.495086669921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22681 , TFLOPS: 69.83182730154846, Tokens per sec: 173050.96066405656, Loss: 2.486929416656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22682 , TFLOPS: 70.6919603954413, Tokens per sec: 175182.46522220524, Loss: 2.5177464485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22683 , TFLOPS: 69.52968888494117, Tokens per sec: 172302.22838440948, Loss: 2.490220308303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22684 , TFLOPS: 69.22157576440928, Tokens per sec: 171538.6901302693, Loss: 2.5026304721832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22685 , TFLOPS: 71.50876920578105, Tokens per sec: 177206.60743314555, Loss: 2.49831223487854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22686 , TFLOPS: 69.04907604084217, Tokens per sec: 171111.21681285676, Loss: 2.4947052001953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22687 , TFLOPS: 68.21216981586919, Tokens per sec: 169037.2709945423, Loss: 2.498819589614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22688 , TFLOPS: 70.64942211459412, Tokens per sec: 175077.05067628744, Loss: 2.4914278984069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22689 , TFLOPS: 70.68898051927754, Tokens per sec: 175175.080760811, Loss: 2.4974164962768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22690 , TFLOPS: 68.95196853140622, Tokens per sec: 170870.57370719925, Loss: 2.490889072418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22691 , TFLOPS: 69.47959706188692, Tokens per sec: 172178.0953288112, Loss: 2.4869728088378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22692 , TFLOPS: 70.74477196396018, Tokens per sec: 175313.33810666957, Loss: 2.4922971725463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22693 , TFLOPS: 69.83193962672658, Tokens per sec: 173051.23901822156, Loss: 2.495338201522827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22694 , TFLOPS: 69.40771592200419, Tokens per sec: 171999.9659458206, Loss: 2.4918675422668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22695 , TFLOPS: 70.81125409128438, Tokens per sec: 175478.08814178983, Loss: 2.5087766647338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22696 , TFLOPS: 69.75281179295204, Tokens per sec: 172855.15152947782, Loss: 2.4947643280029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22697 , TFLOPS: 70.68081847524847, Tokens per sec: 175154.85431658357, Loss: 2.484339952468872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22698 , TFLOPS: 69.56482978324459, Tokens per sec: 172389.3114014949, Loss: 2.501984119415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22699 , TFLOPS: 69.86833509701131, Tokens per sec: 173141.4310028792, Loss: 2.510141611099243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22700 , TFLOPS: 70.60823582046534, Tokens per sec: 174974.98650239152, Loss: 2.4780516624450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22701 , TFLOPS: 67.41366642990843, Tokens per sec: 167058.49164172207, Loss: 2.502045154571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22702 , TFLOPS: 70.59940826583475, Tokens per sec: 174953.1108495815, Loss: 2.4809157848358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22703 , TFLOPS: 70.2379637700189, Tokens per sec: 174057.41157255167, Loss: 2.482170820236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22704 , TFLOPS: 69.04371598092565, Tokens per sec: 171097.9339939243, Loss: 2.5054707527160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22705 , TFLOPS: 70.41877842095965, Tokens per sec: 174505.49019596097, Loss: 2.4986233711242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22706 , TFLOPS: 69.86348617293216, Tokens per sec: 173129.41483485795, Loss: 2.4702095985412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22707 , TFLOPS: 70.10401089481378, Tokens per sec: 173725.46159166604, Loss: 2.4855003356933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22708 , TFLOPS: 70.01007215988272, Tokens per sec: 173492.67105830097, Loss: 2.4885764122009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22709 , TFLOPS: 70.46449212634576, Tokens per sec: 174618.7737936881, Loss: 2.479929208755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22710 , TFLOPS: 68.90301856087927, Tokens per sec: 170749.27028794904, Loss: 2.4749464988708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22711 , TFLOPS: 70.71270179356912, Tokens per sec: 175233.86469161962, Loss: 2.485246181488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22712 , TFLOPS: 70.69375364690306, Tokens per sec: 175186.90909687045, Loss: 2.4808948040008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22713 , TFLOPS: 68.7836342150383, Tokens per sec: 170453.42272768376, Loss: 2.471799612045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22714 , TFLOPS: 69.60330341407978, Tokens per sec: 172484.65329692478, Loss: 2.5044991970062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22715 , TFLOPS: 70.66342875880468, Tokens per sec: 175111.7606836005, Loss: 2.4762771129608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22716 , TFLOPS: 68.70049284669471, Tokens per sec: 170247.38925815056, Loss: 2.513608455657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22717 , TFLOPS: 70.62135223572149, Tokens per sec: 175007.49042428867, Loss: 2.480990409851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22718 , TFLOPS: 70.72453197616665, Tokens per sec: 175263.1811872985, Loss: 2.4777543544769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22719 , TFLOPS: 69.71136741245549, Tokens per sec: 172752.44778913193, Loss: 2.4770925045013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22720 , TFLOPS: 69.5658280230017, Tokens per sec: 172391.78514957818, Loss: 2.4916250705718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22721 , TFLOPS: 68.97537720325657, Tokens per sec: 170928.58297472246, Loss: 2.4915201663970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22722 , TFLOPS: 70.72449027393091, Tokens per sec: 175263.07784456442, Loss: 2.5017452239990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22723 , TFLOPS: 70.4937798652668, Tokens per sec: 174691.3520370452, Loss: 2.4936776161193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22724 , TFLOPS: 68.84486882882544, Tokens per sec: 170605.1688461979, Loss: 2.485093116760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22725 , TFLOPS: 70.99609822864474, Tokens per sec: 175936.1522764315, Loss: 2.4709548950195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22726 , TFLOPS: 69.76232006441006, Tokens per sec: 172878.71407357333, Loss: 2.4973268508911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22727 , TFLOPS: 69.655503931285, Tokens per sec: 172614.01192892098, Loss: 2.503554582595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22728 , TFLOPS: 68.73896333099327, Tokens per sec: 170342.72335611607, Loss: 2.5055923461914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22729 , TFLOPS: 70.69581292358446, Tokens per sec: 175192.012211335, Loss: 2.5067522525787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22730 , TFLOPS: 69.64957925206188, Tokens per sec: 172599.32992114883, Loss: 2.4649133682250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22731 , TFLOPS: 68.99074699804926, Tokens per sec: 170966.67101934142, Loss: 2.505464553833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22732 , TFLOPS: 71.5798267370265, Tokens per sec: 177382.69582320502, Loss: 2.5002875328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22733 , TFLOPS: 69.1842430296622, Tokens per sec: 171446.1755010258, Loss: 2.4805922508239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22734 , TFLOPS: 69.24834940086167, Tokens per sec: 171605.03815075784, Loss: 2.478947162628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22735 , TFLOPS: 69.58940853933407, Tokens per sec: 172450.2202666581, Loss: 2.4824178218841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22736 , TFLOPS: 69.6341015668808, Tokens per sec: 172560.9745122623, Loss: 2.4946703910827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22737 , TFLOPS: 70.62820669767513, Tokens per sec: 175024.47653609113, Loss: 2.4829978942871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22738 , TFLOPS: 69.61847567208463, Tokens per sec: 172522.251823623, Loss: 2.4759926795959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22739 , TFLOPS: 70.59808851600033, Tokens per sec: 174949.84036410935, Loss: 2.4932491779327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22740 , TFLOPS: 70.08684110101096, Tokens per sec: 173682.91295121372, Loss: 2.489562511444092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22741 , TFLOPS: 68.76102702054477, Tokens per sec: 170397.39960934073, Loss: 2.450998306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22742 , TFLOPS: 70.48259051234352, Tokens per sec: 174663.62358789233, Loss: 2.478179454803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22743 , TFLOPS: 70.59059485253144, Tokens per sec: 174931.2702405383, Loss: 2.4958529472351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22744 , TFLOPS: 70.64158667337026, Tokens per sec: 175057.63358978965, Loss: 2.4884350299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22745 , TFLOPS: 70.13917665453263, Tokens per sec: 173812.60621807756, Loss: 2.518751859664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22746 , TFLOPS: 68.79046446693948, Tokens per sec: 170470.34884430925, Loss: 2.497098207473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22747 , TFLOPS: 69.8054756893312, Tokens per sec: 172985.65846611248, Loss: 2.505060911178589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22748 , TFLOPS: 69.76715467902869, Tokens per sec: 172890.69478117506, Loss: 2.4672224521636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22749 , TFLOPS: 70.64348915341463, Tokens per sec: 175062.34814491507, Loss: 2.4979474544525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22750 , TFLOPS: 69.69695011262382, Tokens per sec: 172716.7201319524, Loss: 2.48586106300354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22751 , TFLOPS: 69.69638264174687, Tokens per sec: 172715.3138766072, Loss: 2.514279842376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22752 , TFLOPS: 71.57498187328376, Tokens per sec: 177370.68971714476, Loss: 2.485036849975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22753 , TFLOPS: 67.86179909791214, Tokens per sec: 168169.01375892392, Loss: 2.509542465209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22754 , TFLOPS: 69.73829793333125, Tokens per sec: 172819.1845865039, Loss: 2.4913244247436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22755 , TFLOPS: 70.62489023373404, Tokens per sec: 175016.2579731064, Loss: 2.488190174102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22756 , TFLOPS: 70.56105049874027, Tokens per sec: 174858.0560772645, Loss: 2.4993736743927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22757 , TFLOPS: 68.74348098050298, Tokens per sec: 170353.9185892613, Loss: 2.4842629432678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22758 , TFLOPS: 70.59454610508189, Tokens per sec: 174941.06187962467, Loss: 2.5101735591888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22759 , TFLOPS: 70.63416361925857, Tokens per sec: 175039.23844398398, Loss: 2.483617067337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22760 , TFLOPS: 69.67541871444804, Tokens per sec: 172663.36295539347, Loss: 2.469966173171997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22761 , TFLOPS: 68.8161413486407, Tokens per sec: 170533.9789857069, Loss: 2.4788358211517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22762 , TFLOPS: 70.07609137252796, Tokens per sec: 173656.27393985246, Loss: 2.4914820194244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22763 , TFLOPS: 70.6176209896245, Tokens per sec: 174998.24398542964, Loss: 2.4940757751464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22764 , TFLOPS: 70.67837081286277, Tokens per sec: 175148.78873955953, Loss: 2.4780828952789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22765 , TFLOPS: 69.66893060031286, Tokens per sec: 172647.28469384174, Loss: 2.485448122024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22766 , TFLOPS: 69.7829979170943, Tokens per sec: 172929.95607037842, Loss: 2.486586093902588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22767 , TFLOPS: 70.59604536769305, Tokens per sec: 174944.77721753254, Loss: 2.49058198928833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22768 , TFLOPS: 68.41610237513152, Tokens per sec: 169542.6383414203, Loss: 2.4809064865112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22769 , TFLOPS: 70.65295486443337, Tokens per sec: 175085.80521955364, Loss: 2.490475654602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22770 , TFLOPS: 69.81683341510033, Tokens per sec: 173013.80416172516, Loss: 2.5151684284210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22771 , TFLOPS: 69.55452118072819, Tokens per sec: 172363.7655488729, Loss: 2.4750349521636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22772 , TFLOPS: 70.4436546064192, Tokens per sec: 174567.1361238678, Loss: 2.463315725326538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22773 , TFLOPS: 69.64498774685867, Tokens per sec: 172587.95166545923, Loss: 2.4777493476867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22774 , TFLOPS: 68.29507423116347, Tokens per sec: 169242.7172683168, Loss: 2.4962158203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22775 , TFLOPS: 69.70495403128488, Tokens per sec: 172736.55472409894, Loss: 2.4886300563812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22776 , TFLOPS: 70.7062352241643, Tokens per sec: 175217.83982028303, Loss: 2.497401714324951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22777 , TFLOPS: 68.90613562996037, Tokens per sec: 170756.99472850552, Loss: 2.4807543754577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22778 , TFLOPS: 70.6206073718035, Tokens per sec: 175005.64456944633, Loss: 2.474226474761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22779 , TFLOPS: 70.39100121387295, Tokens per sec: 174436.65521688818, Loss: 2.494901657104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22780 , TFLOPS: 68.81743725040164, Tokens per sec: 170537.19037302586, Loss: 2.511043071746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22781 , TFLOPS: 69.77336912241704, Tokens per sec: 172906.09485646884, Loss: 2.5114145278930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22782 , TFLOPS: 70.75478964130103, Tokens per sec: 175338.16301465844, Loss: 2.4987075328826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22783 , TFLOPS: 69.10311739446963, Tokens per sec: 171245.13724029216, Loss: 2.4836788177490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22784 , TFLOPS: 70.05990899589132, Tokens per sec: 173616.17222791095, Loss: 2.4735279083251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22785 , TFLOPS: 69.76348639643403, Tokens per sec: 172881.6043728108, Loss: 2.4738659858703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22786 , TFLOPS: 69.58244890644214, Tokens per sec: 172432.97352968506, Loss: 2.486924648284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22787 , TFLOPS: 69.51541359470826, Tokens per sec: 172266.8526426593, Loss: 2.49711537361145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22788 , TFLOPS: 68.90636677570048, Tokens per sec: 170757.5675331123, Loss: 2.4773173332214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22789 , TFLOPS: 70.7122629549747, Tokens per sec: 175232.7772012406, Loss: 2.490255355834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22790 , TFLOPS: 68.93656848879573, Tokens per sec: 170832.4107051557, Loss: 2.5012388229370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22791 , TFLOPS: 69.77335258894314, Tokens per sec: 172906.0538846992, Loss: 2.4846010208129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22792 , TFLOPS: 71.51507881593626, Tokens per sec: 177222.2433421711, Loss: 2.4930672645568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22793 , TFLOPS: 69.68614873478084, Tokens per sec: 172689.95312778748, Loss: 2.4751272201538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22794 , TFLOPS: 68.52923150876508, Tokens per sec: 169822.98479676712, Loss: 2.493713140487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22795 , TFLOPS: 70.19423121620129, Tokens per sec: 173949.0374866531, Loss: 2.4985580444335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22796 , TFLOPS: 70.61992774567587, Tokens per sec: 175003.9603810351, Loss: 2.5209109783172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22797 , TFLOPS: 69.67591173476139, Tokens per sec: 172664.58471404063, Loss: 2.4772024154663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22798 , TFLOPS: 69.58525954162707, Tokens per sec: 172439.93859329034, Loss: 2.49363112449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22799 , TFLOPS: 71.4628521058412, Tokens per sec: 177092.81980130848, Loss: 2.5044689178466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22800 , TFLOPS: 69.1289052259931, Tokens per sec: 171309.04232757143, Loss: 2.508145809173584 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/optimizer.pt +[rank0]:[2024-08-28 02:11:54,800] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006386265013134107, 'preprocessing_with_comm': 0.0013109520077705383, 'state_converting': 0.9906554550107103, : 0.9997384280140977}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0022800, took 4.48s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0020800 + eval ppl=9.590128898620605, eval loss=2.2607343196868896 +------------------------------------------------------------------ +iteration: 22801 , TFLOPS: 64.0249243292713, Tokens per sec: 158660.81541558407, Loss: 2.468465805053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22802 , TFLOPS: 70.72179040189761, Tokens per sec: 175256.38726425046, Loss: 2.5276777744293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22803 , TFLOPS: 69.87729190960286, Tokens per sec: 173163.62697115538, Loss: 2.4806699752807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22804 , TFLOPS: 70.5163559017631, Tokens per sec: 174747.29794243877, Loss: 2.4831626415252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22805 , TFLOPS: 69.42129380848203, Tokens per sec: 172033.61344424036, Loss: 2.4820661544799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22806 , TFLOPS: 70.64075008836372, Tokens per sec: 175055.56043998597, Loss: 2.484890937805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22807 , TFLOPS: 69.97315613117176, Tokens per sec: 173401.1890725189, Loss: 2.49746036529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22808 , TFLOPS: 69.8504311964539, Tokens per sec: 173097.06316502043, Loss: 2.502253293991089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22809 , TFLOPS: 69.13764572022849, Tokens per sec: 171330.70223513167, Loss: 2.4851765632629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22810 , TFLOPS: 69.72922005996305, Tokens per sec: 172796.68861628714, Loss: 2.4836132526397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22811 , TFLOPS: 70.54335797821116, Tokens per sec: 174814.21206240135, Loss: 2.501832962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22812 , TFLOPS: 69.71554111048003, Tokens per sec: 172762.79067260757, Loss: 2.492771863937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22813 , TFLOPS: 67.85667817524298, Tokens per sec: 168156.32354843355, Loss: 2.4980523586273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22814 , TFLOPS: 70.56720890111033, Tokens per sec: 174873.31727674152, Loss: 2.497865676879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22815 , TFLOPS: 69.79718503453964, Tokens per sec: 172965.11331024792, Loss: 2.480443239212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22816 , TFLOPS: 69.453915230229, Tokens per sec: 172114.4529208756, Loss: 2.498006820678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22817 , TFLOPS: 69.28373672836706, Tokens per sec: 171692.7318465521, Loss: 2.5097320079803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22818 , TFLOPS: 70.61930109041914, Tokens per sec: 175002.40746027714, Loss: 2.4938831329345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22819 , TFLOPS: 71.52656689730979, Tokens per sec: 177250.71207333135, Loss: 2.4906156063079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22820 , TFLOPS: 68.18642850516476, Tokens per sec: 168973.48119097727, Loss: 2.4660348892211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22821 , TFLOPS: 69.67646237567457, Tokens per sec: 172665.94926288162, Loss: 2.500434398651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22822 , TFLOPS: 70.61268921773836, Tokens per sec: 174986.0225114163, Loss: 2.4806442260742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22823 , TFLOPS: 68.98816168947822, Tokens per sec: 170960.2643399071, Loss: 2.5142295360565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22824 , TFLOPS: 69.49357975016062, Tokens per sec: 172212.7459706737, Loss: 2.4881439208984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22825 , TFLOPS: 70.6348089147874, Tokens per sec: 175040.83755738955, Loss: 2.4893898963928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22826 , TFLOPS: 70.62793140668691, Tokens per sec: 175023.7943346965, Loss: 2.5010416507720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22827 , TFLOPS: 69.69399160223772, Tokens per sec: 172709.3886173088, Loss: 2.5045599937438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22828 , TFLOPS: 68.6974716201687, Tokens per sec: 170239.9023260035, Loss: 2.4915621280670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22829 , TFLOPS: 69.82133981946755, Tokens per sec: 173024.97152816856, Loss: 2.4712765216827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22830 , TFLOPS: 69.75733706768769, Tokens per sec: 172866.36565877288, Loss: 2.485353946685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22831 , TFLOPS: 69.49914230212417, Tokens per sec: 172226.53058720508, Loss: 2.4835970401763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22832 , TFLOPS: 69.87920981705253, Tokens per sec: 173168.3797570906, Loss: 2.4643232822418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22833 , TFLOPS: 69.63629373682514, Tokens per sec: 172566.40695087187, Loss: 2.4952890872955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22834 , TFLOPS: 69.84924151615917, Tokens per sec: 173094.115006196, Loss: 2.481565237045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22835 , TFLOPS: 68.67015325797858, Tokens per sec: 170172.20441513037, Loss: 2.5034902095794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22836 , TFLOPS: 70.59236188071567, Tokens per sec: 174935.64913103302, Loss: 2.5074570178985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22837 , TFLOPS: 69.8117788991616, Tokens per sec: 173001.27851442804, Loss: 2.482484817504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22838 , TFLOPS: 69.20492240839641, Tokens per sec: 171497.42127955082, Loss: 2.483776330947876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22839 , TFLOPS: 71.50646665861842, Tokens per sec: 177200.9014676301, Loss: 2.464900493621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22840 , TFLOPS: 68.36686745909837, Tokens per sec: 169420.62879581645, Loss: 2.478189706802368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22841 , TFLOPS: 70.56856836638693, Tokens per sec: 174876.6861814544, Loss: 2.4836835861206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22842 , TFLOPS: 68.94843178335677, Tokens per sec: 170861.8092559277, Loss: 2.496701955795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22843 , TFLOPS: 69.74368942148972, Tokens per sec: 172832.54528808198, Loss: 2.494316339492798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22844 , TFLOPS: 70.03552596665563, Tokens per sec: 173555.7483954525, Loss: 2.508955478668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22845 , TFLOPS: 70.63365898961125, Tokens per sec: 175037.9879161283, Loss: 2.4975883960723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22846 , TFLOPS: 69.65563219433783, Tokens per sec: 172614.32977889525, Loss: 2.4872403144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22847 , TFLOPS: 69.83837193792627, Tokens per sec: 173067.178993952, Loss: 2.4818522930145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22848 , TFLOPS: 69.81960990859565, Tokens per sec: 173020.68461846258, Loss: 2.4983885288238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22849 , TFLOPS: 70.66870998331748, Tokens per sec: 175124.8481397181, Loss: 2.4857614040374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22850 , TFLOPS: 68.14696541777222, Tokens per sec: 168875.68731320061, Loss: 2.505880355834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22851 , TFLOPS: 70.6740113090517, Tokens per sec: 175137.98540887673, Loss: 2.48085618019104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22852 , TFLOPS: 69.9484449795026, Tokens per sec: 173339.95211652463, Loss: 2.487053155899048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22853 , TFLOPS: 68.86842613166354, Tokens per sec: 170663.5464376818, Loss: 2.480794668197632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22854 , TFLOPS: 70.64533660017898, Tokens per sec: 175066.9263215102, Loss: 2.4811432361602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22855 , TFLOPS: 68.99558811880718, Tokens per sec: 170978.66784987302, Loss: 2.4917004108428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22856 , TFLOPS: 70.59012455854354, Tokens per sec: 174930.10480022908, Loss: 2.47941255569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22857 , TFLOPS: 68.6429864790513, Tokens per sec: 170104.88214428083, Loss: 2.4889938831329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22858 , TFLOPS: 70.67738244096269, Tokens per sec: 175146.339445113, Loss: 2.4951276779174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22859 , TFLOPS: 70.98865398083298, Tokens per sec: 175917.70461030232, Loss: 2.470646619796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22860 , TFLOPS: 69.103431093886, Tokens per sec: 171245.91462200278, Loss: 2.468073844909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22861 , TFLOPS: 69.54148971705223, Tokens per sec: 172331.47214634967, Loss: 2.4825620651245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22862 , TFLOPS: 69.75500672885178, Tokens per sec: 172860.5908224295, Loss: 2.481079578399658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22863 , TFLOPS: 69.85193286942716, Tokens per sec: 173100.78447607113, Loss: 2.5020153522491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22864 , TFLOPS: 69.60073560338965, Tokens per sec: 172478.28997916696, Loss: 2.487582206726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22865 , TFLOPS: 70.69104504906034, Tokens per sec: 175180.19689303948, Loss: 2.4889683723449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22866 , TFLOPS: 70.59727081897368, Tokens per sec: 174947.81402080055, Loss: 2.4890103340148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22867 , TFLOPS: 69.93012868833787, Tokens per sec: 173294.5623293124, Loss: 2.4923205375671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22868 , TFLOPS: 69.58992721687183, Tokens per sec: 172451.50560673373, Loss: 2.4693548679351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22869 , TFLOPS: 70.68471263331462, Tokens per sec: 175164.5044692484, Loss: 2.4977519512176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22870 , TFLOPS: 68.81937465507536, Tokens per sec: 170541.99147523008, Loss: 2.484257459640503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22871 , TFLOPS: 70.67431297886358, Tokens per sec: 175138.73297990224, Loss: 2.505267381668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22872 , TFLOPS: 69.70492666483916, Tokens per sec: 172736.48690703177, Loss: 2.4804575443267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22873 , TFLOPS: 69.72190400267253, Tokens per sec: 172778.558620391, Loss: 2.480576276779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22874 , TFLOPS: 70.66018988111168, Tokens per sec: 175103.73438788537, Loss: 2.4759230613708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22875 , TFLOPS: 68.69062761307168, Tokens per sec: 170222.94212248747, Loss: 2.4965362548828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22876 , TFLOPS: 69.60912731087556, Tokens per sec: 172499.08555473993, Loss: 2.496452808380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22877 , TFLOPS: 69.68014336608394, Tokens per sec: 172675.07116261058, Loss: 2.473515033721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22878 , TFLOPS: 70.63553564523052, Tokens per sec: 175042.63847548357, Loss: 2.5078039169311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22879 , TFLOPS: 69.68637598014136, Tokens per sec: 172690.516266824, Loss: 2.491666793823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22880 , TFLOPS: 68.98771192030696, Tokens per sec: 170959.14976235462, Loss: 2.4730453491210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22881 , TFLOPS: 70.5779859524884, Tokens per sec: 174900.0239972466, Loss: 2.476616382598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22882 , TFLOPS: 69.966438195438, Tokens per sec: 173384.54128772893, Loss: 2.497981309890747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22883 , TFLOPS: 69.64025910407338, Tokens per sec: 172576.23356773425, Loss: 2.472261428833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22884 , TFLOPS: 69.88134276190573, Tokens per sec: 173173.6654294001, Loss: 2.486731767654419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22885 , TFLOPS: 70.02267067366327, Tokens per sec: 173523.89156328997, Loss: 2.5151073932647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22886 , TFLOPS: 71.55974319650724, Tokens per sec: 177332.92659741372, Loss: 2.478775978088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22887 , TFLOPS: 68.15756216485862, Tokens per sec: 168901.94721980873, Loss: 2.476787805557251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22888 , TFLOPS: 69.67289765930313, Tokens per sec: 172657.11550302725, Loss: 2.4744598865509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22889 , TFLOPS: 70.78155405123968, Tokens per sec: 175404.48817082957, Loss: 2.507450819015503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22890 , TFLOPS: 67.88015149863635, Tokens per sec: 168214.49302960234, Loss: 2.4714484214782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22891 , TFLOPS: 70.62639885034848, Tokens per sec: 175019.99649126484, Loss: 2.4938106536865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22892 , TFLOPS: 69.76636213888786, Tokens per sec: 172888.73077940056, Loss: 2.4790713787078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22893 , TFLOPS: 70.56543519377153, Tokens per sec: 174868.92183456602, Loss: 2.4960200786590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22894 , TFLOPS: 69.19690129562122, Tokens per sec: 171477.54407849535, Loss: 2.481043577194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22895 , TFLOPS: 69.81682097703248, Tokens per sec: 173013.77333882282, Loss: 2.4866039752960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22896 , TFLOPS: 69.93203488414092, Tokens per sec: 173299.28609249715, Loss: 2.493258237838745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22897 , TFLOPS: 69.7297934196697, Tokens per sec: 172798.10946480103, Loss: 2.5109574794769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22898 , TFLOPS: 69.56500960345369, Tokens per sec: 172389.75701578186, Loss: 2.4810588359832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22899 , TFLOPS: 70.56747097419371, Tokens per sec: 174873.9667227119, Loss: 2.4722023010253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22900 , TFLOPS: 69.78405365251118, Tokens per sec: 172932.57229904027, Loss: 2.480787754058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22901 , TFLOPS: 69.93117137622029, Tokens per sec: 173297.1462247466, Loss: 2.4959120750427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22902 , TFLOPS: 68.93949833879394, Tokens per sec: 170839.67119620086, Loss: 2.502974033355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22903 , TFLOPS: 70.6608174420299, Tokens per sec: 175105.28955297227, Loss: 2.4788033962249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22904 , TFLOPS: 69.61423226822842, Tokens per sec: 172511.73620141778, Loss: 2.470505475997925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22905 , TFLOPS: 68.63306139708517, Tokens per sec: 170080.28669783068, Loss: 2.4907889366149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22906 , TFLOPS: 71.52183605295143, Tokens per sec: 177238.98851986582, Loss: 2.478680372238159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22907 , TFLOPS: 68.918543927917, Tokens per sec: 170787.7438577289, Loss: 2.4862871170043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22908 , TFLOPS: 70.59457831645109, Tokens per sec: 174941.14170294593, Loss: 2.480046272277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22909 , TFLOPS: 69.23155844635677, Tokens per sec: 171563.42831581837, Loss: 2.4954099655151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22910 , TFLOPS: 69.69562547207713, Tokens per sec: 172713.43752675646, Loss: 2.4984381198883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22911 , TFLOPS: 69.94461431571558, Tokens per sec: 173330.45930967794, Loss: 2.4762139320373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22912 , TFLOPS: 68.77527355844705, Tokens per sec: 170432.7040996483, Loss: 2.5038866996765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22913 , TFLOPS: 70.68476426359712, Tokens per sec: 175164.63241477602, Loss: 2.4963343143463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22914 , TFLOPS: 69.78952499574682, Tokens per sec: 172946.13089029642, Loss: 2.4922361373901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22915 , TFLOPS: 69.83384350363362, Tokens per sec: 173055.95703492657, Loss: 2.492271661758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22916 , TFLOPS: 68.7729430489939, Tokens per sec: 170426.92884049655, Loss: 2.4907047748565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22917 , TFLOPS: 69.71429704075089, Tokens per sec: 172759.70773077314, Loss: 2.506089687347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22918 , TFLOPS: 70.66841391669637, Tokens per sec: 175124.11445401653, Loss: 2.4908061027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22919 , TFLOPS: 70.50740707726439, Tokens per sec: 174725.12176953655, Loss: 2.4958746433258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22920 , TFLOPS: 68.9505954598593, Tokens per sec: 170867.17108464966, Loss: 2.483330488204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22921 , TFLOPS: 70.56828780255937, Tokens per sec: 174875.99091338305, Loss: 2.5034635066986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22922 , TFLOPS: 69.24951421679884, Tokens per sec: 171607.92469296523, Loss: 2.485653877258301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22923 , TFLOPS: 70.59063074019794, Tokens per sec: 174931.3591741292, Loss: 2.4885010719299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22924 , TFLOPS: 68.7122711150916, Tokens per sec: 170276.5771047214, Loss: 2.4902751445770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22925 , TFLOPS: 70.59175965936167, Tokens per sec: 174934.15676017763, Loss: 2.4886367321014404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22926 , TFLOPS: 70.80614490825312, Tokens per sec: 175465.4270234154, Loss: 2.489903211593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22927 , TFLOPS: 68.72727849162777, Tokens per sec: 170313.76703697705, Loss: 2.4842987060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22928 , TFLOPS: 70.54865369304058, Tokens per sec: 174827.3354271198, Loss: 2.4856832027435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22929 , TFLOPS: 69.86250634677866, Tokens per sec: 173126.98671771283, Loss: 2.4996371269226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22930 , TFLOPS: 70.52213473671122, Tokens per sec: 174761.61853203145, Loss: 2.4770407676696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22931 , TFLOPS: 68.10066940350274, Tokens per sec: 168760.9606899138, Loss: 2.4801836013793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22932 , TFLOPS: 69.79036338943169, Tokens per sec: 172948.2085222042, Loss: 2.494100570678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22933 , TFLOPS: 70.67695372120639, Tokens per sec: 175145.27703032948, Loss: 2.4964852333068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22934 , TFLOPS: 69.75985749065589, Tokens per sec: 172872.6115445356, Loss: 2.4820659160614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22935 , TFLOPS: 68.70934968579361, Tokens per sec: 170269.337481099, Loss: 2.4877758026123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22936 , TFLOPS: 70.69502910623277, Tokens per sec: 175190.06982559292, Loss: 2.4924206733703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22937 , TFLOPS: 69.15351523876707, Tokens per sec: 171370.02865024205, Loss: 2.491344451904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22938 , TFLOPS: 69.58133330116699, Tokens per sec: 172430.20893691917, Loss: 2.4868175983428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22939 , TFLOPS: 70.18941787191035, Tokens per sec: 173937.10948926653, Loss: 2.4912447929382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22940 , TFLOPS: 69.89442206723382, Tokens per sec: 173206.07738880755, Loss: 2.49653697013855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22941 , TFLOPS: 70.58746394016397, Tokens per sec: 174923.5114947932, Loss: 2.4625089168548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22942 , TFLOPS: 68.87879546413924, Tokens per sec: 170689.24278583284, Loss: 2.488241672515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22943 , TFLOPS: 69.98065501438825, Tokens per sec: 173419.7721311995, Loss: 2.47602915763855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22944 , TFLOPS: 69.83346461902448, Tokens per sec: 173055.01811712753, Loss: 2.482917547225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22945 , TFLOPS: 70.56873298290935, Tokens per sec: 174877.09411933098, Loss: 2.4873898029327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22946 , TFLOPS: 69.54829328128707, Tokens per sec: 172348.3321279981, Loss: 2.475198745727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22947 , TFLOPS: 68.97152835096767, Tokens per sec: 170919.04509476453, Loss: 2.4930763244628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22948 , TFLOPS: 70.56452642239154, Tokens per sec: 174866.66979897695, Loss: 2.485854387283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22949 , TFLOPS: 68.86726683554963, Tokens per sec: 170660.67357420453, Loss: 2.4885740280151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22950 , TFLOPS: 70.55744397905123, Tokens per sec: 174849.11872418414, Loss: 2.491556167602539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22951 , TFLOPS: 68.8666546402955, Tokens per sec: 170659.1564869259, Loss: 2.485908269882202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22952 , TFLOPS: 69.88863547043604, Tokens per sec: 173191.7375644964, Loss: 2.485002279281616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22953 , TFLOPS: 70.50272788775257, Tokens per sec: 174713.52622247097, Loss: 2.5063509941101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22954 , TFLOPS: 69.18900225398171, Tokens per sec: 171457.9693831614, Loss: 2.4969162940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22955 , TFLOPS: 70.59068671344897, Tokens per sec: 174931.49788201123, Loss: 2.4892985820770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22956 , TFLOPS: 69.98062826343183, Tokens per sec: 173419.70583938266, Loss: 2.492673397064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22957 , TFLOPS: 68.05492188794975, Tokens per sec: 168647.59330686866, Loss: 2.485172748565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22958 , TFLOPS: 70.60495127756211, Tokens per sec: 174966.84704325502, Loss: 2.4780187606811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22959 , TFLOPS: 70.65216218203925, Tokens per sec: 175083.84086525984, Loss: 2.4954583644866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22960 , TFLOPS: 70.6054517213786, Tokens per sec: 174968.0871981607, Loss: 2.504002094268799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22961 , TFLOPS: 69.08885153144635, Tokens per sec: 171209.78486020592, Loss: 2.5037736892700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22962 , TFLOPS: 69.78837616346762, Tokens per sec: 172943.2839573542, Loss: 2.4968292713165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22963 , TFLOPS: 69.80621352660927, Tokens per sec: 172987.486908167, Loss: 2.4934868812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22964 , TFLOPS: 69.46054363029548, Tokens per sec: 172130.87882641886, Loss: 2.481898784637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22965 , TFLOPS: 70.55819923692873, Tokens per sec: 174850.9903364031, Loss: 2.4961624145507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22966 , TFLOPS: 70.74561846400412, Tokens per sec: 175315.43582702815, Loss: 2.4927120208740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22967 , TFLOPS: 69.79193264131304, Tokens per sec: 172952.0973012336, Loss: 2.4716553688049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22968 , TFLOPS: 69.0579650005377, Tokens per sec: 171133.24463418205, Loss: 2.476186513900757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22969 , TFLOPS: 69.50479221030801, Tokens per sec: 172240.5316820734, Loss: 2.509242057800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22970 , TFLOPS: 70.66174764215798, Tokens per sec: 175107.59469135964, Loss: 2.5015830993652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22971 , TFLOPS: 69.83451742343983, Tokens per sec: 173057.62708244476, Loss: 2.4813201427459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22972 , TFLOPS: 69.44935403151054, Tokens per sec: 172103.14976799372, Loss: 2.481295108795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22973 , TFLOPS: 70.98885289701123, Tokens per sec: 175918.19754650447, Loss: 2.5013818740844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22974 , TFLOPS: 68.70309747144505, Tokens per sec: 170253.843805205, Loss: 2.510197162628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22975 , TFLOPS: 70.63561180988968, Tokens per sec: 175042.8272198992, Loss: 2.486379861831665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22976 , TFLOPS: 68.76840049994472, Tokens per sec: 170415.6719035496, Loss: 2.470561981201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22977 , TFLOPS: 69.85873978824685, Tokens per sec: 173117.6527707503, Loss: 2.498823881149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22978 , TFLOPS: 70.10080173786595, Tokens per sec: 173717.50894723146, Loss: 2.4908180236816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22979 , TFLOPS: 69.61077318429406, Tokens per sec: 172503.1642103789, Loss: 2.4842000007629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22980 , TFLOPS: 70.60636132137661, Tokens per sec: 174970.3412871565, Loss: 2.4816365242004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22981 , TFLOPS: 69.63310264885938, Tokens per sec: 172558.4990833653, Loss: 2.488469362258911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22982 , TFLOPS: 69.68945023298338, Tokens per sec: 172698.1346040234, Loss: 2.4683785438537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22983 , TFLOPS: 68.87388070882133, Tokens per sec: 170677.0634807501, Loss: 2.490912437438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22984 , TFLOPS: 69.93351939424721, Tokens per sec: 173302.96487207283, Loss: 2.4926414489746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22985 , TFLOPS: 70.65122134461718, Tokens per sec: 175081.50936648616, Loss: 2.4779653549194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22986 , TFLOPS: 69.65388048263763, Tokens per sec: 172609.9888443352, Loss: 2.506530523300171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22987 , TFLOPS: 69.70103727759349, Tokens per sec: 172726.84857698562, Loss: 2.468456983566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22988 , TFLOPS: 70.550192466105, Tokens per sec: 174831.14867628328, Loss: 2.5149478912353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22989 , TFLOPS: 69.1938964067866, Tokens per sec: 171470.09763294735, Loss: 2.5000975131988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22990 , TFLOPS: 69.6114519034226, Tokens per sec: 172504.8461511465, Loss: 2.479365825653076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22991 , TFLOPS: 69.80162711518474, Tokens per sec: 172976.12127542213, Loss: 2.503901481628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22992 , TFLOPS: 70.59018874725663, Tokens per sec: 174930.26386693108, Loss: 2.466017246246338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22993 , TFLOPS: 70.06397823083624, Tokens per sec: 173626.2562403677, Loss: 2.4941961765289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22994 , TFLOPS: 68.6610794822267, Tokens per sec: 170149.7185992583, Loss: 2.4695229530334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22995 , TFLOPS: 70.55121662858247, Tokens per sec: 174833.68666372323, Loss: 2.495589256286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22996 , TFLOPS: 69.780331564435, Tokens per sec: 172923.34855476566, Loss: 2.481553077697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22997 , TFLOPS: 69.72250536652999, Tokens per sec: 172780.04886627547, Loss: 2.480731248855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22998 , TFLOPS: 69.61576456438458, Tokens per sec: 172515.5334000891, Loss: 2.5014946460723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22999 , TFLOPS: 70.6124072008433, Tokens per sec: 174985.32364248356, Loss: 2.4667434692382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23000 , TFLOPS: 70.64224172841607, Tokens per sec: 175059.2568883534, Loss: 2.493521213531494 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023000, took 4.41s +[rank0]:[2024-08-28 02:37:18,700] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006362203013850376, 'preprocessing_with_comm': 0.0012946369824931026, 'state_converting': 1.007295179006178, : 1.0163192369800527}) +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021000 + eval ppl=9.732975006103516, eval loss=2.275519609451294 +------------------------------------------------------------------ +iteration: 23001 , TFLOPS: 63.283534508182264, Tokens per sec: 156823.56976809454, Loss: 2.4891719818115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23002 , TFLOPS: 69.76417316203293, Tokens per sec: 172883.30625361836, Loss: 2.4974217414855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23003 , TFLOPS: 70.62580349094043, Tokens per sec: 175018.5211250674, Loss: 2.4775424003601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23004 , TFLOPS: 68.72466446789396, Tokens per sec: 170307.28919820616, Loss: 2.495636224746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23005 , TFLOPS: 69.47921685988904, Tokens per sec: 172177.15314637776, Loss: 2.4969124794006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23006 , TFLOPS: 69.87394904654143, Tokens per sec: 173155.3429882424, Loss: 2.491882085800171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23007 , TFLOPS: 69.81007449969985, Tokens per sec: 172997.05482480675, Loss: 2.4759743213653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23008 , TFLOPS: 69.5183891218033, Tokens per sec: 172274.2263265822, Loss: 2.4844846725463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23009 , TFLOPS: 68.90703531702911, Tokens per sec: 170759.22425217056, Loss: 2.485100269317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23010 , TFLOPS: 70.56978430390056, Tokens per sec: 174879.69940855974, Loss: 2.4949676990509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23011 , TFLOPS: 69.74305086958917, Tokens per sec: 172830.96288613012, Loss: 2.515516996383667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23012 , TFLOPS: 70.4010298590375, Tokens per sec: 174461.50730435236, Loss: 2.4894704818725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23013 , TFLOPS: 70.87517001771145, Tokens per sec: 175636.47884839665, Loss: 2.4956912994384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23014 , TFLOPS: 68.9082253517361, Tokens per sec: 170762.17328926764, Loss: 2.5031566619873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23015 , TFLOPS: 70.63610775939796, Tokens per sec: 175044.05623741407, Loss: 2.4798331260681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23016 , TFLOPS: 68.81489920702978, Tokens per sec: 170530.90082196184, Loss: 2.4915366172790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23017 , TFLOPS: 69.85639385766977, Tokens per sec: 173111.83929635474, Loss: 2.484712600708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23018 , TFLOPS: 70.73247617947412, Tokens per sec: 175282.86779822066, Loss: 2.5005035400390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23019 , TFLOPS: 68.93307388777662, Tokens per sec: 170823.75069886824, Loss: 2.4905920028686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23020 , TFLOPS: 70.57669946163381, Tokens per sec: 174896.83593119026, Loss: 2.500181198120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23021 , TFLOPS: 69.64206245679878, Tokens per sec: 172580.7024744435, Loss: 2.4617486000061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23022 , TFLOPS: 69.82195722748364, Tokens per sec: 173026.5015332455, Loss: 2.4774160385131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23023 , TFLOPS: 69.5267561620599, Tokens per sec: 172294.9607740436, Loss: 2.502955913543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23024 , TFLOPS: 68.97424639232393, Tokens per sec: 170925.7807006623, Loss: 2.473771572113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23025 , TFLOPS: 70.56027228107271, Tokens per sec: 174856.12756815652, Loss: 2.502134323120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23026 , TFLOPS: 69.93108759717892, Tokens per sec: 173296.93861105305, Loss: 2.4832000732421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23027 , TFLOPS: 68.76860486314608, Tokens per sec: 170416.17833807503, Loss: 2.488665819168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23028 , TFLOPS: 70.55562780174127, Tokens per sec: 174844.6180367416, Loss: 2.492379903793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23029 , TFLOPS: 69.68245881901466, Tokens per sec: 172680.8091100411, Loss: 2.4918901920318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23030 , TFLOPS: 70.07910020403806, Tokens per sec: 173663.73015578464, Loss: 2.5225167274475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23031 , TFLOPS: 68.75374564796195, Tokens per sec: 170379.35556597108, Loss: 2.478102445602417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23032 , TFLOPS: 70.64731124854401, Tokens per sec: 175071.8197176842, Loss: 2.46525239944458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23033 , TFLOPS: 70.68053927335598, Tokens per sec: 175154.16242353735, Loss: 2.4889702796936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23034 , TFLOPS: 68.77618528781457, Tokens per sec: 170434.96346545627, Loss: 2.4907402992248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23035 , TFLOPS: 71.48736235444974, Tokens per sec: 177153.55889738724, Loss: 2.491427183151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23036 , TFLOPS: 69.01014535127226, Tokens per sec: 171014.74227553303, Loss: 2.4935386180877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23037 , TFLOPS: 70.67481170842332, Tokens per sec: 175139.96888669085, Loss: 2.507188320159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23038 , TFLOPS: 68.8995784065904, Tokens per sec: 170740.74520665692, Loss: 2.495939254760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23039 , TFLOPS: 69.78590071172016, Tokens per sec: 172937.1495152305, Loss: 2.4953393936157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23040 , TFLOPS: 71.54029215258892, Tokens per sec: 177284.7247678747, Loss: 2.486083507537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23041 , TFLOPS: 69.07636996888631, Tokens per sec: 171178.85417322454, Loss: 2.4832346439361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23042 , TFLOPS: 68.74633273054317, Tokens per sec: 170360.9855400071, Loss: 2.4991016387939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23043 , TFLOPS: 70.04722958392603, Tokens per sec: 173584.75124830843, Loss: 2.4853017330169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23044 , TFLOPS: 68.74901365235068, Tokens per sec: 170367.62915957405, Loss: 2.498662233352661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23045 , TFLOPS: 70.66758532713662, Tokens per sec: 175122.0611178108, Loss: 2.495790481567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23046 , TFLOPS: 68.91556852062972, Tokens per sec: 170780.37047070285, Loss: 2.5344760417938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23047 , TFLOPS: 70.70369266321009, Tokens per sec: 175211.5390741523, Loss: 2.524322986602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23048 , TFLOPS: 70.17116400716166, Tokens per sec: 173891.8744015901, Loss: 2.47457218170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23049 , TFLOPS: 69.4728761501753, Tokens per sec: 172161.44016922024, Loss: 2.487338066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23050 , TFLOPS: 69.68320860395741, Tokens per sec: 172682.66715972486, Loss: 2.479844808578491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23051 , TFLOPS: 69.68591783130952, Tokens per sec: 172689.38092354944, Loss: 2.4976398944854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23052 , TFLOPS: 70.51509609479565, Tokens per sec: 174744.17600199423, Loss: 2.4708945751190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23053 , TFLOPS: 69.63969109050284, Tokens per sec: 172574.8259675346, Loss: 2.500864267349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23054 , TFLOPS: 69.89810324672025, Tokens per sec: 173215.19975709033, Loss: 2.4645121097564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23055 , TFLOPS: 70.89472908643971, Tokens per sec: 175684.94837531456, Loss: 2.4554061889648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23056 , TFLOPS: 68.87567342236103, Tokens per sec: 170681.50602238515, Loss: 2.47934889793396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23057 , TFLOPS: 70.63222115976666, Tokens per sec: 175034.42481538342, Loss: 2.474637508392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23058 , TFLOPS: 69.66982375691812, Tokens per sec: 172649.49803429886, Loss: 2.489478588104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23059 , TFLOPS: 69.711849715596, Tokens per sec: 172753.64298944423, Loss: 2.4802775382995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23060 , TFLOPS: 70.47123160888545, Tokens per sec: 174635.4749738367, Loss: 2.477383613586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23061 , TFLOPS: 68.45434060452435, Tokens per sec: 169637.39688614442, Loss: 2.5027589797973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23062 , TFLOPS: 70.52904506897704, Tokens per sec: 174778.74309662206, Loss: 2.4813947677612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23063 , TFLOPS: 70.12790654586425, Tokens per sec: 173784.67764729235, Loss: 2.474224090576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23064 , TFLOPS: 68.68061062280404, Tokens per sec: 170198.1189171417, Loss: 2.4984164237976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23065 , TFLOPS: 69.95615478210736, Tokens per sec: 173359.05785668408, Loss: 2.4790430068969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23066 , TFLOPS: 69.91557940715339, Tokens per sec: 173258.50760780164, Loss: 2.4878711700439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23067 , TFLOPS: 70.6257790988743, Tokens per sec: 175018.46067884055, Loss: 2.4591546058654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23068 , TFLOPS: 69.61951318815608, Tokens per sec: 172524.8229027399, Loss: 2.477565050125122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23069 , TFLOPS: 69.69776639152863, Tokens per sec: 172718.74296099902, Loss: 2.4938595294952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23070 , TFLOPS: 69.82138486458128, Tokens per sec: 173025.0831549225, Loss: 2.5027318000793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23071 , TFLOPS: 68.70785905536957, Tokens per sec: 170265.64353470173, Loss: 2.504610776901245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23072 , TFLOPS: 70.64918173810966, Tokens per sec: 175076.4549968793, Loss: 2.4803953170776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23073 , TFLOPS: 70.66608671217503, Tokens per sec: 175118.3473848499, Loss: 2.4881606101989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23074 , TFLOPS: 69.73345788442714, Tokens per sec: 172807.19041214406, Loss: 2.4825448989868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23075 , TFLOPS: 69.60933239097373, Tokens per sec: 172499.59376581467, Loss: 2.4759840965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23076 , TFLOPS: 68.86044586458642, Tokens per sec: 170643.77045676572, Loss: 2.4912214279174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23077 , TFLOPS: 70.70154966194362, Tokens per sec: 175206.2284809429, Loss: 2.4807233810424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23078 , TFLOPS: 69.66716962058383, Tokens per sec: 172642.9207920853, Loss: 2.5138115882873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23079 , TFLOPS: 69.53379790721013, Tokens per sec: 172312.41099423816, Loss: 2.500356674194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23080 , TFLOPS: 70.95534142163255, Tokens per sec: 175835.15241892237, Loss: 2.4926490783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23081 , TFLOPS: 69.0005091900477, Tokens per sec: 170990.86280650774, Loss: 2.50687313079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23082 , TFLOPS: 70.35191539940352, Tokens per sec: 174339.796262975, Loss: 2.466409206390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23083 , TFLOPS: 69.92910756171166, Tokens per sec: 173292.03186504627, Loss: 2.4988529682159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23084 , TFLOPS: 70.6441881230957, Tokens per sec: 175064.08026878262, Loss: 2.496281147003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23085 , TFLOPS: 69.89491785014806, Tokens per sec: 173207.305993484, Loss: 2.4984114170074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23086 , TFLOPS: 68.20440164573081, Tokens per sec: 169018.0206132048, Loss: 2.499666929244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23087 , TFLOPS: 70.56297451544366, Tokens per sec: 174862.82400260342, Loss: 2.4781079292297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23088 , TFLOPS: 69.8961773350216, Tokens per sec: 173210.4271357444, Loss: 2.481809377670288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23089 , TFLOPS: 69.84221342633991, Tokens per sec: 173076.69862541585, Loss: 2.498067855834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23090 , TFLOPS: 69.73163617489423, Tokens per sec: 172802.67601524366, Loss: 2.4839987754821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23091 , TFLOPS: 69.06850723204134, Tokens per sec: 171159.3694451707, Loss: 2.517878770828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23092 , TFLOPS: 70.56885192377284, Tokens per sec: 174877.3888678931, Loss: 2.490889549255371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23093 , TFLOPS: 69.59106035903403, Tokens per sec: 172454.31365783757, Loss: 2.5061943531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23094 , TFLOPS: 69.88127061348598, Tokens per sec: 173173.48663766816, Loss: 2.523324489593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23095 , TFLOPS: 70.62057137798348, Tokens per sec: 175005.5553727953, Loss: 2.518733501434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23096 , TFLOPS: 69.7793647189119, Tokens per sec: 172920.95260505262, Loss: 2.4843380451202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23097 , TFLOPS: 69.87241779453447, Tokens per sec: 173151.54837708783, Loss: 2.481156826019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23098 , TFLOPS: 69.70290986100655, Tokens per sec: 172731.48904496524, Loss: 2.508100986480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23099 , TFLOPS: 70.52212382121309, Tokens per sec: 174761.5914822246, Loss: 2.487304925918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23100 , TFLOPS: 70.61124704319677, Tokens per sec: 174982.44864403357, Loss: 2.4751224517822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23101 , TFLOPS: 67.10881531037523, Tokens per sec: 166303.03698539574, Loss: 2.4855947494506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23102 , TFLOPS: 71.53886324563899, Tokens per sec: 177281.18377904574, Loss: 2.4568326473236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23103 , TFLOPS: 69.19306136717002, Tokens per sec: 171468.02831279024, Loss: 2.496629476547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23104 , TFLOPS: 70.56166562587205, Tokens per sec: 174859.5804300589, Loss: 2.5112760066986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23105 , TFLOPS: 69.62999112132896, Tokens per sec: 172550.7883753819, Loss: 2.49516224861145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23106 , TFLOPS: 68.99077135258264, Tokens per sec: 170966.73137255802, Loss: 2.479599952697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23107 , TFLOPS: 71.45905505178871, Tokens per sec: 177083.41028308638, Loss: 2.4716687202453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23108 , TFLOPS: 68.96422698122805, Tokens per sec: 170900.95149623745, Loss: 2.498356342315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23109 , TFLOPS: 70.66935394810868, Tokens per sec: 175126.44395540925, Loss: 2.4997451305389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23110 , TFLOPS: 70.06459349328455, Tokens per sec: 173627.78092849135, Loss: 2.4807610511779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23111 , TFLOPS: 68.92189205500799, Tokens per sec: 170796.04088548702, Loss: 2.503755807876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23112 , TFLOPS: 69.48193654613186, Tokens per sec: 172183.89282848564, Loss: 2.4846017360687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23113 , TFLOPS: 70.66063914538493, Tokens per sec: 175104.84771424503, Loss: 2.4763383865356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23114 , TFLOPS: 70.52617613780465, Tokens per sec: 174771.63356913804, Loss: 2.4931750297546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23115 , TFLOPS: 70.0644804928898, Tokens per sec: 173627.50090106513, Loss: 2.491454601287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23116 , TFLOPS: 68.69528876176192, Tokens per sec: 170234.4929624098, Loss: 2.4891505241394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23117 , TFLOPS: 69.93682695937673, Tokens per sec: 173311.16138282217, Loss: 2.493847370147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23118 , TFLOPS: 69.72166878571798, Tokens per sec: 172777.97572686634, Loss: 2.491628408432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23119 , TFLOPS: 70.61834016039391, Tokens per sec: 175000.02616981944, Loss: 2.4811103343963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23120 , TFLOPS: 69.29908271922429, Tokens per sec: 171730.76090239678, Loss: 2.5025887489318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23121 , TFLOPS: 69.77586428391875, Tokens per sec: 172912.27814153483, Loss: 2.4889326095581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23122 , TFLOPS: 70.89978417224098, Tokens per sec: 175697.4754347938, Loss: 2.473860740661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23123 , TFLOPS: 67.96483495147129, Tokens per sec: 168424.34795437977, Loss: 2.461773633956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23124 , TFLOPS: 70.60676372371353, Tokens per sec: 174971.33848447568, Loss: 2.5037295818328857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23125 , TFLOPS: 69.67731840376416, Tokens per sec: 172668.07059478687, Loss: 2.487220525741577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23126 , TFLOPS: 69.51478902815613, Tokens per sec: 172265.30489794145, Loss: 2.4841763973236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23127 , TFLOPS: 69.57578306713835, Tokens per sec: 172416.4548455874, Loss: 2.4885382652282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23128 , TFLOPS: 68.93150310054845, Tokens per sec: 170819.85811508037, Loss: 2.496192455291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23129 , TFLOPS: 70.65573971242354, Tokens per sec: 175092.70637964964, Loss: 2.499084949493408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23130 , TFLOPS: 68.98099696870443, Tokens per sec: 170942.50937257055, Loss: 2.4877066612243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23131 , TFLOPS: 69.81022209269224, Tokens per sec: 172997.42057650062, Loss: 2.481210231781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23132 , TFLOPS: 70.83482701803597, Tokens per sec: 175536.50445105252, Loss: 2.493405818939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23133 , TFLOPS: 69.86981421118156, Tokens per sec: 173145.09641073094, Loss: 2.4805707931518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23134 , TFLOPS: 70.20902949097666, Tokens per sec: 173985.70924171104, Loss: 2.5125796794891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23135 , TFLOPS: 69.80095260550478, Tokens per sec: 172974.44976613176, Loss: 2.4884369373321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23136 , TFLOPS: 69.6922222548877, Tokens per sec: 172705.00397966575, Loss: 2.5081381797790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23137 , TFLOPS: 69.85335764050116, Tokens per sec: 173104.315215744, Loss: 2.4872055053710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23138 , TFLOPS: 68.89467460802912, Tokens per sec: 170728.5930536241, Loss: 2.478581666946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23139 , TFLOPS: 70.65490480094701, Tokens per sec: 175090.63737703767, Loss: 2.518611192703247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23140 , TFLOPS: 70.66878661541016, Tokens per sec: 175125.03804248545, Loss: 2.5029377937316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23141 , TFLOPS: 69.80343939617958, Tokens per sec: 172980.61230736657, Loss: 2.4869394302368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23142 , TFLOPS: 70.25302955148128, Tokens per sec: 174094.74623864825, Loss: 2.4825501441955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23143 , TFLOPS: 68.95882597879454, Tokens per sec: 170887.56721723767, Loss: 2.471864938735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23144 , TFLOPS: 70.57239888685203, Tokens per sec: 174886.1786331336, Loss: 2.4859328269958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23145 , TFLOPS: 68.70706224089133, Tokens per sec: 170263.6689406483, Loss: 2.4825968742370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23146 , TFLOPS: 70.64531533372183, Tokens per sec: 175066.87362088665, Loss: 2.4882121086120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23147 , TFLOPS: 70.05878139348833, Tokens per sec: 173613.37790494066, Loss: 2.506155252456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23148 , TFLOPS: 68.95265882800861, Tokens per sec: 170872.28433822238, Loss: 2.4771993160247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23149 , TFLOPS: 69.65058499246308, Tokens per sec: 172601.82225665444, Loss: 2.489429473876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23150 , TFLOPS: 69.8110713305992, Tokens per sec: 172999.52508158534, Loss: 2.4802486896514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23151 , TFLOPS: 69.80647319921408, Tokens per sec: 172988.130405487, Loss: 2.4725544452667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23152 , TFLOPS: 70.75023351352726, Tokens per sec: 175326.87242813615, Loss: 2.498290538787842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23153 , TFLOPS: 68.76425353308356, Tokens per sec: 170405.39526284154, Loss: 2.4968740940093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23154 , TFLOPS: 70.66088708766006, Tokens per sec: 175105.4621425159, Loss: 2.4819414615631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23155 , TFLOPS: 69.66676345488781, Tokens per sec: 172641.9142687477, Loss: 2.4990639686584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23156 , TFLOPS: 68.72391204612454, Tokens per sec: 170305.42461417546, Loss: 2.504246711730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23157 , TFLOPS: 70.72391168220683, Tokens per sec: 175261.644030535, Loss: 2.4837892055511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23158 , TFLOPS: 69.38788304761222, Tokens per sec: 171950.8178983058, Loss: 2.5025713443756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23159 , TFLOPS: 70.6198377115232, Tokens per sec: 175003.73726648669, Loss: 2.458026170730591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23160 , TFLOPS: 70.49027362023696, Tokens per sec: 174682.6631756165, Loss: 2.5190372467041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23161 , TFLOPS: 70.5871868209775, Tokens per sec: 174922.8247629219, Loss: 2.490508794784546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23162 , TFLOPS: 69.95694388463608, Tokens per sec: 173361.0133396771, Loss: 2.49660062789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23163 , TFLOPS: 69.76539625059668, Tokens per sec: 172886.33720181367, Loss: 2.507138967514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23164 , TFLOPS: 68.99560792709258, Tokens per sec: 170978.71693698628, Loss: 2.499438524246216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23165 , TFLOPS: 69.87612223947065, Tokens per sec: 173160.72839971795, Loss: 2.4767119884490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23166 , TFLOPS: 70.55556173895381, Tokens per sec: 174844.45432587687, Loss: 2.5062503814697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23167 , TFLOPS: 69.05157885537358, Tokens per sec: 171117.41906297338, Loss: 2.499634027481079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23168 , TFLOPS: 68.69631832220759, Tokens per sec: 170237.0443266096, Loss: 2.486135244369507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23169 , TFLOPS: 71.49058556837993, Tokens per sec: 177161.54637657257, Loss: 2.5029494762420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23170 , TFLOPS: 68.92699948919922, Tokens per sec: 170808.6976700432, Loss: 2.495671033859253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23171 , TFLOPS: 69.6713950283761, Tokens per sec: 172653.3918180616, Loss: 2.496046543121338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23172 , TFLOPS: 70.16165715312951, Tokens per sec: 173868.31537003198, Loss: 2.490096092224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23173 , TFLOPS: 69.64403740072147, Tokens per sec: 172585.59660304198, Loss: 2.4846389293670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23174 , TFLOPS: 71.50700304606893, Tokens per sec: 177202.2306948204, Loss: 2.4869136810302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23175 , TFLOPS: 68.109529737566, Tokens per sec: 168782.91757376897, Loss: 2.472810745239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23176 , TFLOPS: 69.75045623757377, Tokens per sec: 172849.3142037646, Loss: 2.4715189933776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23177 , TFLOPS: 70.56978628868123, Tokens per sec: 174879.70432706492, Loss: 2.5031421184539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23178 , TFLOPS: 69.14796136236079, Tokens per sec: 171356.26553269732, Loss: 2.5113189220428467 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/optimizer.pt +[rank0]:[2024-08-28 02:59:55,699] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006350181007292122, 'preprocessing_with_comm': 0.0013361950113903731, 'state_converting': 0.9863133459875826, : 0.9953841809765436}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0023178, took 4.46s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5/iter_0021200 \ No newline at end of file